diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 3560a93af3..7024351758 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -98,6 +98,8 @@ Below are the dev container defaults. The field name to change these defaults is - Plugin loader: enabled; reads `.devcontainer/config/plugin-config.py` if present - If `plugin-config.py` is missing: plugin is enabled with empty config (features won’t work until configured) + + ## πŸ”§ Configuration ### NetBox Version and Environment (use .devcontainer/.env) diff --git a/.devcontainer/scripts/diagnose.sh b/.devcontainer/scripts/diagnose.sh index dd50b5703b..be7596d699 100755 --- a/.devcontainer/scripts/diagnose.sh +++ b/.devcontainer/scripts/diagnose.sh @@ -1,8 +1,10 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script echo "πŸ” DevContainer Startup Diagnostics" echo "==================================" +PLUGIN_WS_DIR="${PLUGIN_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}" echo "πŸ“ Current working directory: $(pwd)" echo "πŸ‘€ Current user: $(whoami)" echo "πŸ†” User ID: $(id)" @@ -26,18 +28,20 @@ echo " - Redis: $(timeout 3 bash -c 'cat < /dev/null > /dev/tcp/redis/6379' 2>/ echo "" echo "πŸ—‚οΈ File System:" echo " - NetBox venv: $(test -f /opt/netbox/venv/bin/activate && echo 'Exists' || echo 'Missing')" -echo " - Plugin directory: $(test -d /workspaces/netbox-librenms-plugin && echo 'Exists' || echo 'Missing')" -echo " - Setup script: $(test -f /workspaces/netbox-librenms-plugin/.devcontainer/scripts/setup.sh && echo 'Exists' || echo 'Missing')" -echo " - Start script: $(test -f /workspaces/netbox-librenms-plugin/.devcontainer/scripts/start-netbox.sh && echo 'Exists' || echo 'Missing')" -echo " - Start script executable: $(test -x /workspaces/netbox-librenms-plugin/.devcontainer/scripts/start-netbox.sh && echo 'Yes' || echo 'No')" -echo " - Plugin config: $(test -f /workspaces/netbox-librenms-plugin/.devcontainer/plugin-config.py && echo 'Found' || echo 'Missing (using defaults)')" +echo " - Plugin directory: $(test -d "$PLUGIN_WS_DIR" && echo 'Exists' || echo 'Missing')" +echo " - Setup script: $(test -f "$PLUGIN_WS_DIR/.devcontainer/scripts/setup.sh" && echo 'Exists' || echo 'Missing')" +echo " - Start script: $(test -f "$PLUGIN_WS_DIR/.devcontainer/scripts/start-netbox.sh" && echo 'Exists' || echo 'Missing')" +echo " - Start script executable: $(test -x "$PLUGIN_WS_DIR/.devcontainer/scripts/start-netbox.sh" && echo 'Yes' || echo 'No')" +echo " - Plugin config: $(test -f "$PLUGIN_WS_DIR/.devcontainer/config/plugin-config.py" && echo 'Found' || echo 'Missing (using defaults)')" echo " - NetBox config path: /opt/netbox/netbox/netbox/configuration.py" echo "" echo "πŸš€ Process Status:" if [ -f /tmp/netbox.pid ]; then PID=$(cat /tmp/netbox.pid) - if kill -0 $PID 2>/dev/null; then + if [ -z "$PID" ]; then + echo " - NetBox server: PID file exists but is empty" + elif kill -0 "$PID" 2>/dev/null; then echo " - NetBox server: Running (PID: $PID)" else echo " - NetBox server: PID file exists but process not running" diff --git a/.devcontainer/scripts/load-aliases.sh b/.devcontainer/scripts/load-aliases.sh index fc8a979d0c..7d30a23303 100755 --- a/.devcontainer/scripts/load-aliases.sh +++ b/.devcontainer/scripts/load-aliases.sh @@ -1,67 +1,225 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Quick alias loader for current session # Usage: source .devcontainer/scripts/load-aliases.sh export PATH="/opt/netbox/venv/bin:$PATH" export DEBUG="${DEBUG:-True}" -PLUGIN_DIR="/workspaces/netbox-librenms-plugin" +PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -alias netbox-run-bg="$PLUGIN_DIR/.devcontainer/scripts/start-netbox.sh --background" -alias netbox-run="$PLUGIN_DIR/.devcontainer/scripts/start-netbox.sh" +# Clean up empty CA bundle vars (Compose/devcontainer inject "" when host var is +# unset, which breaks requests/curl). When setup.sh has installed custom CAs +# into the system trust store, point to it instead. +for _ca_var in REQUESTS_CA_BUNDLE SSL_CERT_FILE CURL_CA_BUNDLE; do + _val="${!_ca_var}" + if [ -z "$_val" ]; then + if [ -f /etc/ssl/certs/ca-certificates.crt ]; then + declare -x "$_ca_var=/etc/ssl/certs/ca-certificates.crt" + else + unset "$_ca_var" + fi + fi +done +unset _ca_var _val + +# Load shared process management helpers +if ! source "$PLUGIN_DIR/.devcontainer/scripts/process-helpers.sh"; then + printf '%s\n' "Failed to load process-helpers.sh" >&2 + return 1 +fi + +netbox-run-bg() { "$PLUGIN_DIR/.devcontainer/scripts/start-netbox.sh" --background; } +netbox-run() { "$PLUGIN_DIR/.devcontainer/scripts/start-netbox.sh"; } # Robust stop command that kills both tracked and orphaned processes -alias netbox-stop='echo "πŸ›‘ Stopping NetBox and RQ workers..."; \ - if [ -f /tmp/netbox.pid ]; then \ - PID=$(cat /tmp/netbox.pid 2>/dev/null); \ - if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then \ - kill "$PID" 2>/dev/null || kill -9 "$PID" 2>/dev/null; \ - echo " Stopped NetBox (PID: $PID)"; \ - fi; \ - rm -f /tmp/netbox.pid; \ - fi; \ - if [ -f /tmp/rqworker.pid ]; then \ - PID=$(cat /tmp/rqworker.pid 2>/dev/null); \ - if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then \ - kill "$PID" 2>/dev/null || kill -9 "$PID" 2>/dev/null; \ - echo " Stopped RQ worker (PID: $PID)"; \ - fi; \ - rm -f /tmp/rqworker.pid; \ - fi; \ - if pgrep -f "python.*rqworker" >/dev/null 2>&1; then \ - ORPHAN_COUNT=$(pgrep -cf "python.*rqworker" 2>/dev/null || echo 0); \ - pkill -9 -f "python.*rqworker" 2>/dev/null; \ - echo " Killed $ORPHAN_COUNT orphaned RQ worker(s)"; \ - fi; \ - if pgrep -f "python.*runserver.*8000" >/dev/null 2>&1; then \ - pkill -9 -f "python.*runserver.*8000" 2>/dev/null; \ - echo " Killed orphaned NetBox server(s)"; \ - fi; \ - echo "βœ… All processes stopped"' - -alias netbox-restart="netbox-stop && sleep 1 && netbox-run-bg" -alias netbox-reload="cd $PLUGIN_DIR && (command -v uv >/dev/null 2>&1 && uv pip install -e . || pip install -e .) && netbox-restart" +netbox-stop() { + echo "πŸ›‘ Stopping NetBox and RQ workers..." + if [ -f /tmp/netbox.pid ]; then + local PID + PID=$(cat /tmp/netbox.pid 2>/dev/null) + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + if is_expected_pid "$PID" "python.*runserver.*8000"; then + graceful_kill_pid "$PID" + echo " Stopped NetBox (PID: $PID)" + else + echo " Skipping stale /tmp/netbox.pid (PID $PID is not NetBox runserver)" + fi + fi + rm -f /tmp/netbox.pid + fi + if [ -f /tmp/rqworker.pid ]; then + local PID + PID=$(cat /tmp/rqworker.pid 2>/dev/null) + if [ -n "$PID" ] && kill -0 "$PID" 2>/dev/null; then + if is_expected_pid "$PID" "python.*rqworker"; then + graceful_kill_pid "$PID" + echo " Stopped RQ worker (PID: $PID)" + else + echo " Skipping stale /tmp/rqworker.pid (PID $PID is not rqworker)" + fi + fi + rm -f /tmp/rqworker.pid + fi + if pgrep -f "python.*rqworker" >/dev/null 2>&1; then + local ORPHAN_COUNT + ORPHAN_COUNT=$(pgrep -cf "python.*rqworker" 2>/dev/null || echo 0) + graceful_kill_pattern "python.*rqworker" + echo " Killed $ORPHAN_COUNT orphaned RQ worker(s)" + fi + if pgrep -f "python.*runserver.*8000" >/dev/null 2>&1; then + graceful_kill_pattern "python.*runserver.*8000" + echo " Killed orphaned NetBox server(s)" + fi + echo "βœ… All processes stopped" +} + +netbox-restart() { + netbox-stop && sleep 1 && netbox-run-bg +} + +netbox-reload() { + cd "$PLUGIN_DIR" || return 1 + if command -v uv >/dev/null 2>&1; then + uv pip install -e . || return 1 + else + pip install -e . || return 1 + fi + netbox-restart +} alias netbox-logs="tail -f /tmp/netbox.log" -alias netbox-status="[ -f /tmp/netbox.pid ] && kill -0 \$(cat /tmp/netbox.pid) 2>/dev/null && echo 'NetBox is running (PID: '\$(cat /tmp/netbox.pid)')' || echo 'NetBox is not running'; [ -f /tmp/rqworker.pid ] && kill -0 \$(cat /tmp/rqworker.pid) 2>/dev/null && echo 'RQ worker is running (PID: '\$(cat /tmp/rqworker.pid)')' || echo 'RQ worker is not running'" alias rq-logs="tail -f /tmp/rqworker.log" -alias rq-status="[ -f /tmp/rqworker.pid ] && kill -0 \$(cat /tmp/rqworker.pid) 2>/dev/null && echo 'RQ worker is running (PID: '\$(cat /tmp/rqworker.pid)')' || echo 'RQ worker is not running'" -alias netbox-shell="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell" -alias netbox-test="cd $PLUGIN_DIR && source /opt/netbox/venv/bin/activate && python -m pytest" -alias netbox-manage="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py" -alias plugin-install="cd $PLUGIN_DIR && (command -v uv >/dev/null 2>&1 && uv pip install -e . || pip install -e .)" -alias ruff-check="cd $PLUGIN_DIR && ruff check ." -alias ruff-format="cd $PLUGIN_DIR && ruff format ." -alias ruff-fix="cd $PLUGIN_DIR && ruff check --fix ." -alias diagnose="$PLUGIN_DIR/.devcontainer/scripts/diagnose.sh" -alias plugins-install='if [ -f "$PLUGIN_DIR/.devcontainer/extra-requirements.txt" ]; then source /opt/netbox/venv/bin/activate && pip install -r "$PLUGIN_DIR/.devcontainer/extra-requirements.txt"; else echo "No .devcontainer/extra-requirements.txt found"; fi' + +netbox-status() { + local PID + if [ -f /tmp/netbox.pid ]; then + PID=$(cat /tmp/netbox.pid 2>/dev/null) + if [ -n "$PID" ] && is_expected_pid "$PID" "python.*runserver.*8000"; then + echo "NetBox is running (PID: $PID)" + else + echo "NetBox is not running" + fi + else + echo "NetBox is not running" + fi + if [ -f /tmp/rqworker.pid ]; then + PID=$(cat /tmp/rqworker.pid 2>/dev/null) + if [ -n "$PID" ] && is_expected_pid "$PID" "python.*rqworker"; then + echo "RQ worker is running (PID: $PID)" + else + echo "RQ worker is not running" + fi + else + echo "RQ worker is not running" + fi +} + +rq-status() { + local PID + if [ -f /tmp/rqworker.pid ]; then + PID=$(cat /tmp/rqworker.pid 2>/dev/null) + if [ -n "$PID" ] && is_expected_pid "$PID" "python.*rqworker"; then + echo "RQ worker is running (PID: $PID)" + else + echo "RQ worker is not running" + fi + else + echo "RQ worker is not running" + fi +} + +netbox-shell() { + cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell +} + +netbox-test() { + cd "$PLUGIN_DIR" && source /opt/netbox/venv/bin/activate && python -m pytest "$@" +} + +netbox-manage() { + cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py "$@" +} + +plugin-install() { + cd "$PLUGIN_DIR" || return 1 + if command -v uv >/dev/null 2>&1; then + uv pip install -e . + else + pip install -e . + fi +} + +plugins-install() { + if [ -f "$PLUGIN_DIR/.devcontainer/extra-requirements.txt" ]; then + source /opt/netbox/venv/bin/activate && pip install -r "$PLUGIN_DIR/.devcontainer/extra-requirements.txt" + else + echo "No .devcontainer/extra-requirements.txt found" + fi +} + +ruff-check() { cd "$PLUGIN_DIR" && command ruff check .; } +ruff-format() { cd "$PLUGIN_DIR" && command ruff format .; } +ruff-fix() { cd "$PLUGIN_DIR" && command ruff check --fix .; } + +diagnose() { "$PLUGIN_DIR/.devcontainer/scripts/diagnose.sh"; } # RQ job inspection commands -alias rq-stats="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py rqstats" -alias rq-jobs="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell -c \"from django_rq import get_queue; q = get_queue('default'); print(f'Jobs in queue: {len(q)}'); [print(f' {job.id[:8]}: {job.func_name} - {job.get_status()}') for job in q.jobs[:10]]\"" -alias rq-failed="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell -c \"from django_rq import get_failed_queue; q = get_failed_queue(); print(f'Failed jobs: {len(q)}'); [print(f' {job.id[:8]}: {job.func_name}') for job in q.jobs[:10]]\"" -alias rq-recent="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell -c \"from core.models import Job; jobs = Job.objects.all().order_by('-created')[:10]; [print(f'{j.id}: {j.name[:50]} - {getattr(j.status, \\\"value\\\", j.status)} ({j.user})') for j in jobs]\"" +rq-stats() { + cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py rqstats +} + +rq-jobs() { + cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell -c \ + "from django_rq import get_queue; q = get_queue('default'); print(f'Jobs in queue: {len(q)}'); [print(f' {job.id[:8]}: {job.func_name} - {job.get_status()}') for job in q.jobs[:10]]" +} + +rq-failed() { + cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell -c \ + "from django_rq import get_failed_queue; q = get_failed_queue(); print(f'Failed jobs: {len(q)}'); [print(f' {job.id[:8]}: {job.func_name}') for job in q.jobs[:10]]" +} + +rq-recent() { + cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell -c \ + "from core.models import Job; jobs = Job.objects.all().order_by('-created')[:10]; [print(f'{j.id}: {j.name[:50]} - {getattr(j.status, \"value\", j.status)} ({j.user})') for j in jobs]" +} # Help -alias dev-help='echo "🎯 NetBox LibreNMS Plugin Development Commands:"; echo ""; echo "πŸ“Š NetBox Server Management:"; echo " netbox-run-bg : Start NetBox in background"; echo " netbox-run : Start NetBox in foreground (for debugging)"; echo " netbox-stop : Stop NetBox and RQ worker"; echo " netbox-restart : Restart NetBox and RQ worker"; echo " netbox-reload : Reinstall plugin and restart NetBox"; echo " netbox-status : Check if NetBox and RQ worker are running"; echo " netbox-logs : View NetBox server logs"; echo ""; echo "βš™οΈ Background Jobs (RQ Worker):"; echo " rq-status : Check if RQ worker is running"; echo " rq-logs : View RQ worker logs"; echo " rq-stats : Show RQ queue statistics"; echo " rq-jobs : List jobs in default queue"; echo " rq-failed : List failed jobs"; echo " rq-recent : Show recent NetBox jobs"; echo ""; echo "πŸ› οΈ Development Tools:"; echo " netbox-shell : Open NetBox Django shell"; echo " netbox-test : Run plugin tests"; echo " netbox-manage : Run Django management commands"; echo " plugin-install : Reinstall plugin in development mode"; echo ""; echo "🧹 Code Quality:"; echo " ruff-check : Check code with Ruff"; echo " ruff-format : Format code with Ruff"; echo " ruff-fix : Auto-fix code issues with Ruff"; echo ""; echo "πŸ”Ž Diagnostics:"; echo " diagnose : Run startup diagnostics"; echo " dev-help : Show this help message"; echo ""; echo "πŸ“– NetBox available at: http://localhost:8000 (admin/admin)"; echo ""' +dev-help() { + echo "🎯 NetBox LibreNMS Plugin Development Commands:" + echo "" + echo "πŸ“Š NetBox Server Management:" + echo " netbox-run-bg : Start NetBox in background" + echo " netbox-run : Start NetBox in foreground (for debugging)" + echo " netbox-stop : Stop NetBox and RQ worker" + echo " netbox-restart : Restart NetBox and RQ worker" + echo " netbox-reload : Reinstall plugin and restart NetBox" + echo " netbox-status : Check if NetBox and RQ worker are running" + echo " netbox-logs : View NetBox server logs" + echo "" + echo "βš™οΈ Background Jobs (RQ Worker):" + echo " rq-status : Check if RQ worker is running" + echo " rq-logs : View RQ worker logs" + echo " rq-stats : Show RQ queue statistics" + echo " rq-jobs : List jobs in default queue" + echo " rq-failed : List failed jobs" + echo " rq-recent : Show recent NetBox jobs" + echo "" + echo "πŸ› οΈ Development Tools:" + echo " netbox-shell : Open NetBox Django shell" + echo " netbox-test : Run plugin tests" + echo " netbox-manage : Run Django management commands" + echo " plugin-install : Reinstall plugin in development mode" + echo "" + echo "🧹 Code Quality:" + echo " ruff-check : Check code with Ruff" + echo " ruff-format : Format code with Ruff" + echo " ruff-fix : Auto-fix code issues with Ruff" + echo "" + echo "πŸ”Ž Diagnostics:" + echo " diagnose : Run startup diagnostics" + echo " dev-help : Show this help message" + echo "" + echo "πŸ“– NetBox available at: http://localhost:8000 (admin/admin)" +} echo "βœ… Aliases loaded! Try: rq-status, rq-stats, rq-recent, dev-help" diff --git a/.devcontainer/scripts/process-helpers.sh b/.devcontainer/scripts/process-helpers.sh new file mode 100755 index 0000000000..a45655c72f --- /dev/null +++ b/.devcontainer/scripts/process-helpers.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Shared process management helpers. +# Sourced by load-aliases.sh and start-netbox.sh. + +# Graceful termination: SIGTERM, wait, then SIGKILL if still alive. +graceful_kill_pid() { + local pid="$1" + kill -15 "$pid" 2>/dev/null + sleep 2 + kill -0 "$pid" 2>/dev/null && kill -9 "$pid" 2>/dev/null +} + +graceful_kill_pattern() { + local pattern="$1" + pkill -15 -f "$pattern" 2>/dev/null + sleep 2 + pgrep -f "$pattern" >/dev/null 2>&1 && pkill -9 -f "$pattern" 2>/dev/null +} + +# Verify a PID matches the expected process before killing it +is_expected_pid() { + local pid="$1" pattern="$2" + ps -p "$pid" -o args= 2>/dev/null | grep -Eq "$pattern" +} diff --git a/.devcontainer/scripts/setup.sh b/.devcontainer/scripts/setup.sh index 73372ded5a..7f4278fd46 100755 --- a/.devcontainer/scripts/setup.sh +++ b/.devcontainer/scripts/setup.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script set -e echo "πŸš€ Setting up NetBox LibreNMS Plugin development environment..." @@ -28,6 +29,13 @@ detect_plugin_workspace() { fi } +# Clean up empty CA bundle vars (Compose injects "" when host var is unset) +for _ca_var in REQUESTS_CA_BUNDLE SSL_CERT_FILE CURL_CA_BUNDLE; do + _val="${!_ca_var}" + [ -z "$_val" ] && unset "$_ca_var" +done +unset _ca_var _val + # Configure proxy for apt and pip if proxy environment variables are set if [ -n "$HTTP_PROXY" ] || [ -n "$HTTPS_PROXY" ]; then echo "🌐 Configuring proxy settings..." @@ -186,7 +194,7 @@ if [ -f "$CONF_FILE" ]; then echo "import importlib.util, os"; echo "PLUGINS = ['netbox_librenms_plugin']"; echo "PLUGINS_CONFIG = {'netbox_librenms_plugin': {}}"; - echo "_pc_path = '/workspaces/netbox-librenms-plugin/.devcontainer/config/plugin-config.py'"; + echo "_pc_path = '$PLUGIN_WS_DIR/.devcontainer/config/plugin-config.py'"; echo "if os.path.isfile(_pc_path):"; echo " _spec = importlib.util.spec_from_file_location('workspace_plugin_config', _pc_path)"; echo " _mod = importlib.util.module_from_spec(_spec)"; @@ -200,7 +208,7 @@ if [ -f "$CONF_FILE" ]; then echo " print('ℹ️ plugin-config.py not found; using defaults')"; echo "# Import optional extra NetBox configuration (uppercase settings)"; - echo "_xc_path = '/workspaces/netbox-librenms-plugin/.devcontainer/config/extra-configuration.py'"; + echo "_xc_path = '$PLUGIN_WS_DIR/.devcontainer/config/extra-configuration.py'"; echo "if os.path.isfile(_xc_path):"; echo " _xc_spec = importlib.util.spec_from_file_location('workspace_extra_configuration', _xc_path)"; echo " _xc_mod = importlib.util.module_from_spec(_xc_spec)"; @@ -213,7 +221,7 @@ if [ -f "$CONF_FILE" ]; then echo " print(f'⚠️ Failed to apply extra-configuration.py: {e}')"; echo "# Import Codespaces configuration when applicable (uppercase settings)"; - echo "_cs_path = '/workspaces/netbox-librenms-plugin/.devcontainer/config/codespaces-configuration.py'"; + echo "_cs_path = '$PLUGIN_WS_DIR/.devcontainer/config/codespaces-configuration.py'"; echo "if os.environ.get('CODESPACES') == 'true' and os.path.isfile(_cs_path):"; echo " _cs_spec = importlib.util.spec_from_file_location('workspace_codespaces_configuration', _cs_path)"; echo " _cs_mod = importlib.util.module_from_spec(_cs_spec)"; @@ -251,15 +259,17 @@ echo "πŸ—ƒοΈ Applying database migrations..." python manage.py migrate 2>&1 | grep -E "(Operations to perform|Running migrations|Apply all migrations|No migrations to apply|\s+Applying|\s+OK)" || true echo "πŸ” Creating superuser (if not exists)..." +echo " Credentials are read from environment variables (see .devcontainer/.env)" python manage.py shell -c " +import os from django.contrib.auth import get_user_model User = get_user_model() -username = '${SUPERUSER_NAME:-admin}' -email = '${SUPERUSER_EMAIL:-admin@example.com}' -password = '${SUPERUSER_PASSWORD:-admin}' +username = (os.environ.get('SUPERUSER_NAME') or '').strip() or 'admin' +email = (os.environ.get('SUPERUSER_EMAIL') or '').strip() or 'admin@example.com' +password = (os.environ.get('SUPERUSER_PASSWORD') or '').strip() or 'admin' if not User.objects.filter(username=username).exists(): User.objects.create_superuser(username, email, password) - print(f'Created superuser: {username}/{password}') + print(f'Created superuser: {username}') else: print(f'Superuser {username} already exists') " 2>/dev/null || true diff --git a/.devcontainer/scripts/start-netbox.sh b/.devcontainer/scripts/start-netbox.sh index 0ab8acaf46..d5e4796600 100755 --- a/.devcontainer/scripts/start-netbox.sh +++ b/.devcontainer/scripts/start-netbox.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Check if we should run in background or foreground BACKGROUND=false @@ -18,31 +19,35 @@ if [ "$CODESPACES" = "true" ] && [ -n "$CODESPACE_NAME" ]; then echo "πŸ”— GitHub Codespaces detected" else ACCESS_URL="http://localhost:8000" - echo "πŸ› Debug: ACCESS_URL is set to: $ACCESS_URL" fi -# Kill any orphaned RQ workers (not tracked by PID file) +# Load shared process management helpers +if ! source "$(dirname "$0")/process-helpers.sh"; then + echo "ERROR: Failed to load process-helpers.sh" >&2 + exit 1 +fi + +# Kill any orphaned processes (not tracked by PID file) echo "🧹 Cleaning up orphaned processes..." -ORPHAN_RQ_PIDS=$(pgrep -f "python.*rqworker" 2>/dev/null) -if [ -n "$ORPHAN_RQ_PIDS" ]; then +if pgrep -f "python.*rqworker" >/dev/null 2>&1; then echo " Found orphaned RQ workers, killing..." - pkill -9 -f "python.*rqworker" 2>/dev/null - sleep 1 + graceful_kill_pattern "python.*rqworker" fi -# Kill any orphaned NetBox runserver processes -ORPHAN_NETBOX_PIDS=$(pgrep -f "python.*runserver.*8000" 2>/dev/null) -if [ -n "$ORPHAN_NETBOX_PIDS" ]; then +if pgrep -f "python.*runserver.*8000" >/dev/null 2>&1; then echo " Found orphaned NetBox servers, killing..." - pkill -9 -f "python.*runserver.*8000" 2>/dev/null - sleep 1 + graceful_kill_pattern "python.*runserver.*8000" fi # Stop any tracked processes from PID files if [ -f /tmp/netbox.pid ]; then OLD_PID=$(cat /tmp/netbox.pid 2>/dev/null) if [ -n "$OLD_PID" ] && kill -0 "$OLD_PID" 2>/dev/null; then - kill "$OLD_PID" 2>/dev/null || kill -9 "$OLD_PID" 2>/dev/null + if is_expected_pid "$OLD_PID" "python.*runserver.*8000"; then + graceful_kill_pid "$OLD_PID" + else + echo "⚠️ Skipping stale /tmp/netbox.pid (PID $OLD_PID is not NetBox runserver)" + fi fi rm -f /tmp/netbox.pid fi @@ -50,7 +55,11 @@ fi if [ -f /tmp/rqworker.pid ]; then OLD_PID=$(cat /tmp/rqworker.pid 2>/dev/null) if [ -n "$OLD_PID" ] && kill -0 "$OLD_PID" 2>/dev/null; then - kill "$OLD_PID" 2>/dev/null || kill -9 "$OLD_PID" 2>/dev/null + if is_expected_pid "$OLD_PID" "python.*rqworker"; then + graceful_kill_pid "$OLD_PID" + else + echo "⚠️ Skipping stale /tmp/rqworker.pid (PID $OLD_PID is not rqworker)" + fi fi rm -f /tmp/rqworker.pid fi diff --git a/.devcontainer/scripts/welcome.sh b/.devcontainer/scripts/welcome.sh index 3d972d5753..e273313766 100755 --- a/.devcontainer/scripts/welcome.sh +++ b/.devcontainer/scripts/welcome.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Ensure aliases are available in the postAttach terminal session source "$(dirname "$0")/load-aliases.sh" 2>/dev/null @@ -6,7 +7,8 @@ source "$(dirname "$0")/load-aliases.sh" 2>/dev/null echo "" echo "🎯 NetBox LibreNMS Plugin Development Environment" -if [ ! -f "/workspaces/netbox-librenms-plugin/.devcontainer/config/plugin-config.py" ]; then +PLUGIN_WS_DIR="${PLUGIN_DIR:-$(cd "$(dirname "$0")/../.." && pwd)}" +if [ ! -f "$PLUGIN_WS_DIR/.devcontainer/config/plugin-config.py" ]; then echo "" echo "⚠️ Plugin configuration not found: .devcontainer/config/plugin-config.py" echo " Create it first: cp .devcontainer/config/plugin-config.py.example .devcontainer/config/plugin-config.py" @@ -43,7 +45,7 @@ if [ -n "$CODESPACES" ]; then echo " πŸ’‘ Click the link in the Ports panel or look for the 'Open in Browser' button" else echo "πŸ–₯️ Local Development Environment:" - echo " NetBox will be available at: http://localhost:8000 (paste into you browser)" + echo " NetBox will be available at: http://localhost:8000 (paste into your browser)" fi echo "" diff --git a/.github/workflows/lint-format.yaml b/.github/workflows/lint-format.yaml index c5ba8432de..afd4dd86e8 100644 --- a/.github/workflows/lint-format.yaml +++ b/.github/workflows/lint-format.yaml @@ -2,13 +2,7 @@ name: Lint and Format on: push: - branches: - - master - - develop pull_request: - branches: - - master - - develop jobs: format-and-lint: @@ -20,8 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.9' - cache: 'pip' + python-version: '3.12' - name: Install dependencies run: | @@ -29,22 +22,7 @@ jobs: pip install ruff - name: Run Ruff linting - run: | - echo "::group::Ruff Linting" - ruff check . --output-format=github - echo "::endgroup::" + run: ruff check . - name: Run Ruff formatting check - run: | - echo "::group::Ruff Formatting" - ruff format --check . - echo "::endgroup::" - - - name: Report formatting issues - if: failure() - run: | - echo "::error::Formatting or linting issues detected!" - echo "To fix locally, run:" - echo " ruff check --fix ." - echo " ruff format ." - echo "Then commit and push the changes." + run: ruff format --check . diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 26afa7d787..4fe8085fe0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,17 +39,17 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@main + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: path: netbox-librenms-plugin - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@main + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - name: Checkout NetBox - uses: actions/checkout@main + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: "netbox-community/netbox" path: netbox diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..8714ac5342 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,28 @@ +# Contrib: Example Mapping Files + +This directory contains example YAML mapping files for bulk import into the +NetBox LibreNMS Plugin. Each file can be imported via the plugin's bulk import +feature in the NetBox UI. + +## How to Import + +1. Navigate to the mapping page (e.g., **LibreNMS β†’ Device Type Mappings**) +2. Click the **Import** button (upload icon) in the top right +3. Select **YAML** format +4. Paste the contents of the relevant YAML file +5. Click **Submit** + +## Available Mappings + +| File | Description | +|------|-------------| +| `interface_type_mappings.yaml` | Maps LibreNMS interface types + speeds to NetBox interface types | +| `device_type_mappings.yaml` | Maps LibreNMS hardware strings to NetBox device types | +| `module_type_mappings.yaml` | Maps LibreNMS inventory model names to NetBox module types (incl. transceivers) | +| `module_bay_mappings.yaml` | Maps LibreNMS inventory container names to NetBox module bay names | + +## Customisation + +These files are **examples** β€” adjust values to match the device types, module +types, and interface types defined in your NetBox instance. The `netbox_*` +fields must reference objects that already exist in your NetBox. diff --git a/contrib/device_type_mappings.yaml b/contrib/device_type_mappings.yaml new file mode 100644 index 0000000000..2dec241524 --- /dev/null +++ b/contrib/device_type_mappings.yaml @@ -0,0 +1,73 @@ +# Device Type Mappings +# +# Maps LibreNMS hardware strings to NetBox device types. +# Import via: LibreNMS Plugin > Device Type Mappings > Import +# +# Fields: +# librenms_hardware β€” Hardware string exactly as shown in LibreNMS +# netbox_device_type β€” NetBox DeviceType (matched by model name or ID) +# description β€” Optional note +# +# The librenms_hardware value is matched case-insensitively. +# These mappings are checked BEFORE the built-in part_number/model fallback. + +# Juniper β€” LibreNMS reports verbose marketing names +- librenms_hardware: "Juniper MX480 Internet Backbone Router" + netbox_device_type: "MX480" + description: "Juniper MX480 chassis" + +- librenms_hardware: "Juniper MX960 Internet Backbone Router" + netbox_device_type: "MX960" + description: "Juniper MX960 chassis" + +- librenms_hardware: "Juniper MX304 Edge Router" + netbox_device_type: "MX304" + description: "Juniper MX304 edge router" + +- librenms_hardware: "JNP10008 [PTX10008]" + netbox_device_type: "PTX10008" + description: "Juniper PTX10008 core router" + +- librenms_hardware: "JNP7100-32C [ACX7100-32C]" + netbox_device_type: "ACX7100-32C" + description: "Juniper ACX7100-32C" + +- librenms_hardware: "JNP7024 [ACX7024]" + netbox_device_type: "ACX7024" + description: "Juniper ACX7024" + +- librenms_hardware: "Juniper JNP10008 Internet Backbone Router" + netbox_device_type: "PTX10008" + description: "Juniper PTX10008 (alternate hardware string)" + +- librenms_hardware: "Juniper VRR Internet Backbone Router" + netbox_device_type: "VRR" + description: "Juniper Virtual Route Reflector" + +# Nokia β€” model string matches directly in most cases +- librenms_hardware: "7750 SR-7s" + netbox_device_type: "7750 SR-7s" + description: "Nokia 7750 SR-7s service router" + +# Cisco β€” often matches by part_number but not always +- librenms_hardware: "WS-C4900M" + netbox_device_type: "WS-C4900M" + description: "Cisco Catalyst 4900M" + +# Cisco IOS XR +- librenms_hardware: "8201-SYS" + netbox_device_type: "8201" + description: "Cisco 8201 (hardware string differs from model)" + +# UfiSpace β€” LibreNMS reports SONiC/ONIE platform names +- librenms_hardware: "x86-64-ufispace-s9610-36d-r0" + netbox_device_type: "S9610-36D" + description: "UfiSpace S9610-36D" + +- librenms_hardware: "x86-64-ufispace-s9610-46dx-r0" + netbox_device_type: "S9610-46DX" + description: "UfiSpace S9610-46DX" + +- librenms_hardware: "x86-64-ufispace-s9700-53dx-r9" + netbox_device_type: "S9700-53DX" + description: "UfiSpace S9700-53DX" diff --git a/contrib/interface_name_rules.yaml b/contrib/interface_name_rules.yaml new file mode 100644 index 0000000000..52da69dff5 --- /dev/null +++ b/contrib/interface_name_rules.yaml @@ -0,0 +1,200 @@ +# Interface Name Rules +# +# Post-install interface rename rules for module types where NetBox's +# position-based naming can't produce the correct interface name. +# +# Covers two scenarios: +# 1. Converter offset β€” e.g., GLC-T inside CVR-X2-SFP needs port numbering +# that accounts for the converter's position in the parent module bay. +# 2. Breakout channels β€” e.g., QSFP+ 4x10G produces multiple sub-interfaces +# from a single physical port. +# +# Template variables: +# {slot} β€” Top-level slot/module bay position +# {bay_position} β€” Position of the bay this module is installed into (raw) +# {bay_position_num} β€” Numeric suffix of bay position (e.g., "swp1" β†’ "1") +# {parent_bay_position} β€” Position of the parent module's bay +# {sfp_slot} β€” Numeric sub-bay index within the parent module +# {base} β€” Original interface name from the NetBox module template +# {channel} β€” Breakout channel number (iterated) +# +# Arithmetic expressions are supported inside braces: +# {8 + ({parent_bay_position} - 1) * 2 + {sfp_slot}} +# +# Bulk import via: LibreNMS Plugin > Settings > Interface Name Rules > Import + +# --- Converter Offset Examples --- + +# SFP-1G-T (1G copper SFP, covers GLC-T/GLC-TE) in CVR-X2-SFP converter +# X2 bays are numbered 1-N; each converter holds 2 SFP slots +# Resulting interface: GigabitEthernet/ +- module_type: SFP-1G-T + parent_module_type: CVR-X2-SFP + name_template: "GigabitEthernet{slot}/{8 + ({parent_bay_position} - 1) * 2 + {sfp_slot}}" + channel_count: 0 + channel_start: 0 + description: "SFP-1G-T in CVR-X2-SFP: offset port numbering for X2-to-SFP conversion" + +# --- Breakout Channel Examples --- + +# QSFP-4X10G-LR breakout β€” Juniper-style (channels start at 0) +- module_type: QSFP-4X10G-LR + name_template: "{base}:{channel}" + channel_count: 4 + channel_start: 0 + description: "QSFP+ 4x10G-LR breakout with Juniper-style channel numbering (0-3)" + +# QSFP-4X10G-SR breakout β€” Juniper-style (channels start at 0) +- module_type: QSFP-4X10G-SR + name_template: "{base}:{channel}" + channel_count: 4 + channel_start: 0 + description: "QSFP+ 4x10G-SR breakout with Juniper-style channel numbering (0-3)" + +# --- Commented Examples --- + +# QSFP+ 4x10G breakout β€” Cisco-style (channels start at 1) +# - module_type: QSFP-4X10G-LR +# name_template: "{base}:{channel}" +# channel_count: 4 +# channel_start: 1 +# description: "QSFP+ 4x10G breakout with Cisco-style channel numbering (1-4)" + +# --- UfiSpace/Arcos Breakout Rules --- +# UfiSpace switches use swpNsC naming for breakout interfaces. +# bay_position_num extracts the numeric suffix from the bay name (e.g., "swp1" β†’ "1"). +# Channels start at 1, with 2 channels per 100G QSFP28 (2x100G breakout). + +# S9610-36D breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# S9610-46DX breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# S9700-53DX breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# --- Juniper ACX7024 Platform-Specific Rules --- +# These rules are scoped to the ACX7024 device type and use bay_position +# to generate Juniper-style interface names with FPC/PIC/port notation. + +# 100GE QSFP28 transceivers -> et-0/0/{port} +- module_type: QSFP-100G-LR4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +- module_type: QSFP-100G-SR4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +- module_type: QSFP-100G-SWDM4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +# 10GE SFP+ transceivers -> xe-0/0/{port} +- module_type: SFP-10G-SR + device_type: ACX7024 + name_template: "xe-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 10GE SFP+ naming" + +- module_type: SFP-10G-LR + device_type: ACX7024 + name_template: "xe-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 10GE SFP+ naming" + +# 1GE SFP transceivers -> ge-0/0/{port} +- module_type: SFP-1G-T + device_type: ACX7024 + name_template: "ge-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 1GE SFP naming" + +- module_type: SFP-1G-LX + device_type: ACX7024 + name_template: "ge-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 1GE SFP naming" diff --git a/contrib/interface_type_mappings.yaml b/contrib/interface_type_mappings.yaml new file mode 100644 index 0000000000..19db2a1fcf --- /dev/null +++ b/contrib/interface_type_mappings.yaml @@ -0,0 +1,70 @@ +# Interface Type Mappings +# +# Maps LibreNMS interface types (and optional speeds) to NetBox interface types. +# Import via: LibreNMS Plugin > Interface Mappings > Import +# +# Fields: +# librenms_type β€” IANA ifType string from LibreNMS (e.g. ethernetCsmacd) +# librenms_speed β€” Speed in Kbps (optional, null matches any speed) +# netbox_type β€” NetBox InterfaceTypeChoices slug +# description β€” Optional note +# +# Common NetBox interface type slugs: +# 1000base-t, 10gbase-t, 10gbase-x-sfpp, 25gbase-x-sfp28, +# 40gbase-x-qsfpp, 100gbase-x-qsfp28, 400gbase-x-qsfpdd, +# ieee802.11ax, lag, virtual, other + +- librenms_type: ethernetCsmacd + librenms_speed: 1000000 + netbox_type: 1000base-t + description: "1G Ethernet copper" + +- librenms_type: ethernetCsmacd + librenms_speed: 10000000 + netbox_type: 10gbase-x-sfpp + description: "10G Ethernet SFP+" + +- librenms_type: ethernetCsmacd + librenms_speed: 25000000 + netbox_type: 25gbase-x-sfp28 + description: "25G Ethernet SFP28" + +- librenms_type: ethernetCsmacd + librenms_speed: 40000000 + netbox_type: 40gbase-x-qsfpp + description: "40G Ethernet QSFP+" + +- librenms_type: ethernetCsmacd + librenms_speed: 100000000 + netbox_type: 100gbase-x-qsfp28 + description: "100G Ethernet QSFP28" + +- librenms_type: ethernetCsmacd + librenms_speed: 400000000 + netbox_type: 400gbase-x-qsfpdd + description: "400G Ethernet QSFP-DD" + +- librenms_type: ieee8023adLag + librenms_speed: + netbox_type: lag + description: "LACP/LAG aggregation" + +- librenms_type: propVirtual + librenms_speed: + netbox_type: virtual + description: "Virtual/loopback interface" + +- librenms_type: softwareLoopback + librenms_speed: + netbox_type: virtual + description: "Software loopback" + +- librenms_type: tunnel + librenms_speed: + netbox_type: virtual + description: "Tunnel interface" + +- librenms_type: l2vlan + librenms_speed: + netbox_type: virtual + description: "VLAN interface" diff --git a/contrib/module_bay_mappings.yaml b/contrib/module_bay_mappings.yaml new file mode 100644 index 0000000000..64c063176a --- /dev/null +++ b/contrib/module_bay_mappings.yaml @@ -0,0 +1,216 @@ +# Module Bay Mappings - Map LibreNMS inventory container names to NetBox module bay names +# +# These mappings replace heuristic matching between LibreNMS inventory and NetBox module bays. +# Import via: LibreNMS Plugin β†’ Module Bay Mappings β†’ Import +# +# Fields: +# librenms_name: LibreNMS entPhysicalName or container name (exact match or regex) +# librenms_class: Optional entPhysicalClass filter (powerSupply, fan, module, etc.) +# Leave empty for class-independent mappings +# netbox_bay_name: Target NetBox module bay name (supports \1, \2 backreferences with regex) +# is_regex: Set to true to treat librenms_name as a Python regex pattern +# description: Optional description +# +# Regex patterns use Python re.fullmatch() β€” the pattern must match the entire string. +# Backreferences (\1, \2) in netbox_bay_name reference capture groups in the pattern. + +# ─── Regex Patterns ────────────────────────────────────────────────────────── +# These patterns replace many individual exact-match entries. + +# Arcos/UfiSpace: sfpN β†’ Transceiver N (covers sfp0 through sfp53+) +- librenms_name: "^sfp(\\d+)$" + netbox_bay_name: "Transceiver \\1" + is_regex: true + description: "Arcos sfpN β†’ Transceiver N" + +# Cisco X2: Port Container slot/port β†’ X2 Port port +- librenms_name: "^Port Container (\\d+)/(\\d+)$" + netbox_bay_name: "X2 Port \\2" + is_regex: true + description: "Cisco X2 Port Container β†’ X2 Port N" + +# Cisco modules: Linecard/Supervisor(slot N) β†’ Slot N +- librenms_name: "^Linecard\\(slot (\\d+)\\)$" + librenms_class: "module" + netbox_bay_name: "Slot \\1" + is_regex: true + description: "Cisco Linecard slot β†’ Slot N" +- librenms_name: "^Supervisor\\(slot (\\d+)\\)$" + librenms_class: "module" + netbox_bay_name: "Slot \\1" + is_regex: true + description: "Cisco Supervisor slot β†’ Slot N" + +# Generic power supplies and fans +- librenms_name: "^Power Supply (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PS\\1" + is_regex: true + description: "Power Supply N β†’ PSN" +- librenms_name: "^FanTray (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan Tray \\1" + is_regex: true + description: "FanTray N β†’ Fan Tray N" + +# Nokia 7750 SR chassis fans and power modules +- librenms_name: "^Chassis 1 Fan (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan \\1" + is_regex: true + description: "Nokia chassis fan β†’ Fan N" +- librenms_name: "^Chassis 1 PowShelf 1 PM (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PM \\1" + is_regex: true + description: "Nokia power module β†’ PM N" + +# Nokia MDA and XIOM sub-module bays +# Bay names resolve from {module}/N templates: IOM Slot 1 pos=1 β†’ bay {module}/1 = 1/1 +- librenms_name: "^MDA (\\d+)/(\\d+)$" + librenms_class: "mdaModule" + netbox_bay_name: "\\1/\\2" + is_regex: true + description: "Nokia MDA N/M β†’ N/M (matches {module}/M on IOM)" +- librenms_name: "^XIOM (\\d+)/x(\\d+)$" + librenms_class: "xioModule" + netbox_bay_name: "\\1/x\\2" + is_regex: true + description: "Nokia XIOM N/xM β†’ N/xM (matches {module}/xM on IOM)" +- librenms_name: "^MDA (\\d+)/x(\\d+)/(\\d+)$" + librenms_class: "mdaModule" + netbox_bay_name: "x\\2/\\3" + is_regex: true + description: "Nokia MDA in XIOM N/xP/Q β†’ xP/Q (matches {module}/Q on XIOM)" + +# Nokia transceiver connector bays +# LibreNMS ifName "1/1/c1" (slot/mda/connector) β†’ NetBox bay "1/c1" +# ({module} on MDA resolves to position, stripping the slot prefix) +- librenms_name: "(\\d+)/(\\d+)/(c\\d+)" + librenms_class: "port" + netbox_bay_name: "\\2/\\3" + is_regex: true + description: "Nokia transceiver slot/mda/cN β†’ mda-pos/cN" +# LibreNMS ifName "2/x1/1/c2" (slot/xiom/mda/connector) β†’ NetBox bay "1/c2" +- librenms_name: "(\\d+)/x(\\d+)/(\\d+)/(c\\d+)" + librenms_class: "port" + netbox_bay_name: "\\3/\\4" + is_regex: true + description: "Nokia XIOM transceiver slot/xiom/mda/cN β†’ mda-pos/cN" + +# Juniper MX transceiver bays +# LibreNMS entPhysicalDescr format: "SFP+-10G-SR @ {fpc}/{pic}/{port}" +# NetBox MPC-3D-16XGE-SFPP bay format: "Transceiver {pic}/{port}" +- librenms_name: "[^@]+ @ \\d+/(\\d+)/(\\d+)" + librenms_class: "port" + netbox_bay_name: "Transceiver \\1/\\2" + is_regex: true + description: "Juniper MX SFP+ @ fpc/pic/port β†’ Transceiver pic/port" + +# ─── Exact Match Entries ───────────────────────────────────────────────────── +# These are for special cases where names don't follow a regex pattern. + +# Nokia CPM slots +- librenms_name: "Slot A" + librenms_class: "cpmModule" + netbox_bay_name: "Slot A" + description: "Nokia CPM slot A" +- librenms_name: "Slot B" + librenms_class: "cpmModule" + netbox_bay_name: "Slot B" + description: "Nokia CPM slot B" +- librenms_name: "SR-7s 2 CPM mini" + librenms_class: "cpmCarrier" + netbox_bay_name: "CMA" + description: "Nokia CMA2-7s CPM carrier bracket" + +# Juniper fixed-form devices +- librenms_name: "PSM 0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Juniper PSU slot 0" +- librenms_name: "PSM 1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Juniper PSU slot 1" + +# Juniper chassis devices (PTX10008 etc.): PSM β†’ PEM +# Regex runs after exact matches, so PSM 0/1 β†’ PSU 0/1 above takes priority for ACX +- librenms_name: "^PSM (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PEM \\1" + is_regex: true + description: "Juniper chassis PSM N β†’ PEM N" + +# Juniper FPC container: "FPC: @ N/*/*" β†’ FPC N +- librenms_name: "^FPC: .+ @ (\\d+)/\\*/\\*$" + librenms_class: "container" + netbox_bay_name: "FPC \\1" + is_regex: true + description: "Juniper FPC container description β†’ FPC N" + +# Juniper transceivers: " @ slot/pic/port" description β†’ Transceiver slot/pic/port +- librenms_name: "^.+ @ (\\d+/\\d+/\\d+)$" + librenms_class: "port" + netbox_bay_name: "Transceiver \\1" + is_regex: true + description: "Juniper transceiver description β†’ Transceiver slot/pic/port" + +# Juniper fan trays: "Fan Tray N" β†’ "Fan N" (ACX7100, etc.) +# Runs after exact match, so "Fan Tray 0" β†’ "Fan Tray" (ACX7024) still works +- librenms_name: "^Fan Tray (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan \\1" + is_regex: true + description: "Juniper Fan Tray N β†’ Fan N (ACX7100 etc.)" + +# Juniper MX304: PEM β†’ PSU (MX304 bays are named PSU, not PEM) +- librenms_name: "PEM 0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Juniper MX304 PEM 0 β†’ PSU 0" +- librenms_name: "PEM 1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Juniper MX304 PEM 1 β†’ PSU 1" + +- librenms_name: "Fan Tray 0" + librenms_class: "fan" + netbox_bay_name: "Fan Tray" + description: "Juniper single fan tray (ACX7024)" + +# Juniper PTX10008: SIB β†’ CB (Switch Interface Board β†’ Component Board slot) +- librenms_name: "SIB 0" + librenms_class: "container" + netbox_bay_name: "CB 0" + description: "Juniper PTX10008 SIB 0 β†’ CB 0" +- librenms_name: "SIB 1" + librenms_class: "container" + netbox_bay_name: "CB 1" + description: "Juniper PTX10008 SIB 1 β†’ CB 1" +- librenms_name: "SIB 2" + librenms_class: "container" + netbox_bay_name: "CB 2" + description: "Juniper PTX10008 SIB 2 β†’ CB 2" +- librenms_name: "SIB 3" + librenms_class: "container" + netbox_bay_name: "CB 3" + description: "Juniper PTX10008 SIB 3 β†’ CB 3" +- librenms_name: "SIB 4" + librenms_class: "container" + netbox_bay_name: "CB 4" + description: "Juniper PTX10008 SIB 4 β†’ CB 4" +- librenms_name: "SIB 5" + librenms_class: "container" + netbox_bay_name: "CB 5" + description: "Juniper PTX10008 SIB 5 β†’ CB 5" + +# Arcos power supplies +- librenms_name: "psu0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Arcos PSU slot 0" +- librenms_name: "psu1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Arcos PSU slot 1" diff --git a/contrib/module_type_mappings.yaml b/contrib/module_type_mappings.yaml new file mode 100644 index 0000000000..e70d726f1b --- /dev/null +++ b/contrib/module_type_mappings.yaml @@ -0,0 +1,332 @@ +# Module Type Mappings +# +# Maps LibreNMS inventory model names (entPhysicalModelName) to NetBox module types. +# Import via: LibreNMS Plugin > Module Type Mappings > Import +# +# Fields: +# librenms_model β€” Model name from LibreNMS SNMP inventory +# netbox_module_type β€” NetBox ModuleType (matched by model name or ID) +# description β€” Optional note +# +# These mappings are checked FIRST. If no mapping exists, the plugin falls back +# to exact model name and part_number matching against NetBox module types. + +# ─── Cisco Catalyst 4900M ──────────────────────────────────────────────────── + +- librenms_model: "WS-X4908-10GE" + netbox_module_type: "WS-X4908-10GE" + description: "Cisco 8-port 10G X2 line card" + +- librenms_model: "WS-X4992" + netbox_module_type: "WS-X4992" + description: "Cisco 48-port 10/100/1000 line card" + +- librenms_model: "PWR-C49M-1000AC" + netbox_module_type: "PWR-C49M-1000AC" + description: "Cisco 1000W AC power supply" + +- librenms_model: "CVR-X2-SFP" + netbox_module_type: "CVR-X2-SFP" + description: "Cisco X2-to-SFP converter" + +# ─── Juniper Backplane ─────────────────────────────────────────────────────── + +- librenms_model: "710-017414" + netbox_module_type: "MX480-CHASSIS-BP" + description: "Juniper MX480 backplane (matched by part number)" + +- librenms_model: "CHAS-BP-MX480-S" + netbox_module_type: "MX480-CHASSIS-BP" + description: "Juniper MX480 backplane (matched by name)" + +# ─── Juniper FPC / Line Card Mappings ──────────────────────────────────────── +# Juniper FPCs use 750-xxxxxx part numbers as entPhysicalModelName. + +- librenms_model: "750-018124" + netbox_module_type: "DPCE-R-4XGE-XFP" + description: "Juniper DPCE 4-port 10G XFP DPC" + +- librenms_model: "750-022765" + netbox_module_type: "DPCE-R-20GE-2XGE" + description: "Juniper DPCE 20x1G + 2x10G combo DPC" + +- librenms_model: "750-028467" + netbox_module_type: "MPC-3D-16XGE-SFPP" + description: "Juniper MPC 16-port 10G SFP+" + +- librenms_model: "750-056519" + netbox_module_type: "MPC7E-MRATE" + description: "Juniper MPC7E 12-port QSFP+/QSFP28 multirate" + +- librenms_model: "750-062581" + netbox_module_type: "MPC-3D-16XGE-SFPP" + description: "Juniper MPC 16-port 10G SFP+ (variant PN)" + +# ─── Juniper Power Supply Mappings ─────────────────────────────────────────── + +- librenms_model: "740-029970" + netbox_module_type: "PWR-MX480-2520-AC" + description: "Juniper MX480 2520W AC PSU" + +- librenms_model: "740-063046" + netbox_module_type: "PWR-MX480-2520-AC" + description: "Juniper MX480 2520W AC PSU (variant PN)" + +- librenms_model: "740-027760" + netbox_module_type: "PWR-MX960-4100-AC" + description: "Juniper MX960 4100W AC PSU" + +- librenms_model: "740-110419" + netbox_module_type: "JNP-PWR2200-AC" + description: "Juniper MX304 2200W AC PSU" + +# Removed: JPSU-1600W-1UACAFO β€” exact model match, no mapping needed + +# ─── Juniper Fan Tray Mappings ─────────────────────────────────────────────── + +- librenms_model: "740-031521" + netbox_module_type: "FFANTRAY-MX960-HC" + description: "Juniper MX960 high-capacity fan tray" + +- librenms_model: "760-126744" + netbox_module_type: "JNP-FAN-2RU" + description: "Juniper MX304 2RU fan tray" + +# Removed: JNP7100-FAN1RU-AO β€” exact model match, no mapping needed + +# ─── Nokia 7750 SR-7s Module Mappings ──────────────────────────────────────── +# Nokia 3HE part numbers are handled by NormalizationRule: +# 1. Strip extra text (e.g. "3HE10550AARA01 NOK IPU3BFUEAA" β†’ "3HE10550AARA01") +# 2. Strip revision suffix (e.g. "3HE10550AARA01" β†’ "3HE10550AA") +# The normalized value matches the part_number field on NetBox ModuleTypes. +# No explicit Nokia mappings are needed. + +# ─── Transceiver Mappings: Juniper Part Numbers ───────────────────────────── +# Juniper-qualified optics use 740-xxxxxx part numbers regardless of OEM vendor. + +- librenms_model: "740-013111" + netbox_module_type: "SFP-1G-T" + description: "Juniper SFP 1000BASE-T copper" + +- librenms_model: "740-021308" + netbox_module_type: "SFP-10G-SR" + description: "Juniper SFP+ 10G-SR" + +- librenms_model: "740-031850" + netbox_module_type: "SFP-1G-LX" + description: "Juniper SFP 1000BASE-LX 10km" + +- librenms_model: "740-031980" + netbox_module_type: "SFP-10G-SR" + description: "Juniper SFP+ 10G-SR" + +- librenms_model: "740-031981" + netbox_module_type: "SFP-10G-LR" + description: "Juniper SFP+ 10G-LR" + +- librenms_model: "740-047682" + netbox_module_type: "CFP-100G-LR4" + description: "Juniper CFP 100G-LR4" + +- librenms_model: "740-054050" + netbox_module_type: "QSFP-4X10G-LR" + description: "Juniper QSFP+ 4x10G-LR" + +- librenms_model: "740-054053" + netbox_module_type: "QSFP-4X10G-SR" + description: "Juniper QSFP+ 4x10G-SR" + +- librenms_model: "740-058732" + netbox_module_type: "QSFP-100G-LR4" + description: "Juniper QSFP28 100G-LR4" + +- librenms_model: "740-061405" + netbox_module_type: "QSFP-100G-SR4" + description: "Juniper QSFP28 100G-SR4" + +- librenms_model: "740-061409" + netbox_module_type: "QSFP-100G-LR4" + description: "Juniper QSFP28 100G-LR4" + +- librenms_model: "740-079871" + netbox_module_type: "QSFP28-DD-2X100G-LR4" + description: "Juniper QSFP-DD 2x100G-LR4" + +- librenms_model: "740-082823" + netbox_module_type: "QSFP-DD-400G-LR8" + description: "Juniper QSFP-DD 400G-LR8" + +- librenms_model: "740-085349" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "Juniper QSFP-DD 400G-FR4" + +- librenms_model: "740-085351" + netbox_module_type: "QSFP-DD-400G-DR4" + description: "Juniper QSFP-DD 400G-DR4" + +- librenms_model: "740-096176" + netbox_module_type: "QSFP-DD-400G-LR4" + description: "Juniper QSFP-DD 400G-LR4 (10km variant)" + +- librenms_model: "740-131169" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Juniper QSFP-DD 400G-ZR-M" + +- librenms_model: "740-151745" + netbox_module_type: "QSFP-DD-400G-ZR-M-HP" + description: "Juniper QSFP-DD 400G-ZR-M high-power" + +- librenms_model: "740-172665" + netbox_module_type: "QSFP-100G-ZR" + description: "Juniper QSFP28 100G-ZR" + +# ─── Transceiver Mappings: Finisar / II-VI / Coherent ──────────────────────── +# These are BASE part numbers (after normalization strips customer suffixes). +# See contrib/normalization_rules.yaml for the Finisar suffix-stripping rule. + +- librenms_model: "FTLC1154RDPL" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4" + +- librenms_model: "FTLC1151RDPL" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (variant)" + +- librenms_model: "FTLX1474D3BCL" + netbox_module_type: "SFP-10G-LR" + description: "Finisar SFP+ 10G-LR" + +- librenms_model: "FTCD3323R1PCL" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Finisar/II-VI QSFP-DD 400G-ZR-M coherent" + +- librenms_model: "FTLC9152RGPL" + netbox_module_type: "QSFP-100G-SWDM4" + description: "Finisar QSFP28 100G-SWDM4" + +# ─── Transceiver Mappings: Cisco / Cisco-branded OEM ───────────────────────── + +- librenms_model: "X2-10GB-LR" + netbox_module_type: "X2-10GB-LR" + description: "Cisco X2 10G-LR" + +- librenms_model: "X2-10GB-SR" + netbox_module_type: "X2-10GB-SR" + description: "Cisco X2 10G-SR" + +- librenms_model: "GLC-T" + netbox_module_type: "SFP-1G-T" + description: "Cisco SFP 1000BASE-T copper" + +- librenms_model: "GLC-TE" + netbox_module_type: "SFP-1G-T" + description: "Cisco SFP 1000BASE-T copper (extended temp)" + +- librenms_model: "SPP5200LR-C5" + netbox_module_type: "SFP-10G-LR" + description: "Cisco-branded Sumitomo SFP+ 10G-LR" + +- librenms_model: "SPP5310LR-C5" + netbox_module_type: "SFP-10G-LR" + description: "Cisco-branded Sumitomo SFP+ 10G-LR" + +- librenms_model: "SFBR-709SMZ-CS1" + netbox_module_type: "SFP-10G-SR" + description: "Cisco-branded Avago/Broadcom SFP+ 10G-SR" + +- librenms_model: "DP04QSDD-HE0" + netbox_module_type: "QSFP-DD-400G-ZR+" + description: "Cisco/Acacia QSFP-DD 400G-ZR+ coherent" + +- librenms_model: "QDD-400G-ZRP-S" + netbox_module_type: "QSFP-DD-400G-ZR+" + description: "Cisco QSFP-DD 400G-ZR+" + +- librenms_model: "QDD-400G-ZR4-S" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Cisco QSFP-DD 400G-ZR" + +# ─── Transceiver Mappings: Ciena ───────────────────────────────────────────── + +- librenms_model: "180-3530-900" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Ciena WaveLogic 5 Nano QSFP-DD 400ZR" + +- librenms_model: "176-3360-900" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Ciena QSFP-DD 400G-ZR-M coherent" + +- librenms_model: "176-3530-901" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Ciena QSFP-DD 400G-ZR coherent" + +- librenms_model: "176-3590-900" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Ciena QSFP-DD 400G-ZR-M coherent" + +# ─── Transceiver Mappings: T1 Nexus ───────────────────────────────────────── + +- librenms_model: "T1-QDD-400G-LR4" + netbox_module_type: "QSFP-DD-400G-LR4" + description: "T1 Nexus QSFP-DD 400G-LR4" + +- librenms_model: "T1-QDD-400G-FR4" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "T1 Nexus QSFP-DD 400G-FR4" + +- librenms_model: "T1-QSFP28-LR4" + netbox_module_type: "QSFP-100G-LR4" + description: "T1 Nexus QSFP28 100G-LR4" + +- librenms_model: "100G-LR4_A3" + netbox_module_type: "QSFP-100G-LR4" + description: "T1 Nexus QSFP28 100G-LR4 (rev A3)" + +# ─── Transceiver Mappings: Innolight ──────────────────────────────────────── + +- librenms_model: "T-DQ4CNT-NCN" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "Innolight QSFP-DD 400G-FR4" + +# ─── Transceiver Mappings: FS.com ──────────────────────────────────────────── + +- librenms_model: "Q28-PC03" + netbox_module_type: "QSFP28-100G-CU3M" + description: "FS.com QSFP28 100G passive DAC 3m" + +# ─── Transceiver Mappings: ProLabs ─────────────────────────────────────────── + +- librenms_model: "Q28LR431-10-IN" + netbox_module_type: "QSFP-100G-LR4" + description: "ProLabs QSFP28 100G-LR4 10km" + +# ─── Transceiver Mappings: Arcos Fixed-Port Part Numbers ───────────────────── + +- librenms_model: "SP7041-TE" + netbox_module_type: "SFP-1G-T" + description: "SFP 1000BASE-T copper (Arcos platform)" + +# ─── Transceiver Mappings: LeGrand Innolight ───────────────────────────────── + +- librenms_model: "LGI-FTLC9152RGPL" + netbox_module_type: "QSFP-100G-SWDM4" + description: "LeGrand-branded Finisar QSFP28 100G-SWDM4" + +# ─── Transceiver Mappings: Additional Finisar Variants ────────────────────── +# Some transceivers have customer-code suffixes that normalization may not handle. +# Add direct mappings as fallback. + +- librenms_model: "FTLC1151RDPL-CN" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (CN customer code)" + +- librenms_model: "FTLC1154RDPL-A5" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (A5 customer code)" + +# ─── Unknown / Unidentified Part Numbers ───────────────────────────────────── +# These are mapped based on port context (QSFP28 100G slot) when vendor is unknown. + +- librenms_model: "1F3QAA" + netbox_module_type: "QSFP-100G-LR4" + description: "Unknown QSFP28 100G (mapped by port context)" diff --git a/contrib/normalization_rules.yaml b/contrib/normalization_rules.yaml new file mode 100644 index 0000000000..c3d2081bea --- /dev/null +++ b/contrib/normalization_rules.yaml @@ -0,0 +1,61 @@ +# Normalization Rules β€” Examples +# +# Regex-based string transformations applied before module type, device type, +# or module bay matching. Rules run in priority order (lower first); each +# rule's output feeds the next. +# +# Import via: LibreNMS β†’ Normalization Rules β†’ Import β†’ YAML +# +# Fields: +# scope β€” module_type, device_type, or module_bay +# manufacturer β€” Optional manufacturer name (must exist in NetBox). +# When set, the rule only fires for that manufacturer. +# match_pattern β€” Python regex (re.sub pattern) +# replacement β€” Replacement string (supports \1, \2 back-references) +# priority β€” Lower values run first (default 100) +# description β€” Optional note + +# ── Nokia revision suffix stripping ────────────────────────────────────────── +# Nokia ENTITY-MIB reports module/transceiver models with 4-char revision +# suffixes (e.g. 3HE16474AARA01). NetBox module types use the base part +# number (3HE16474AA). This rule strips the suffix before matching. +# +# Captures the 10-char base (3HE + 5 alnum + 2 quality-tier letters), +# discards the 2-letter revision code + 2-digit build number. +- scope: module_type + manufacturer: Nokia + match_pattern: "^(3HE\\w{5}[A-Z]{2})[A-Z]{2}\\d{2}$" + replacement: "\\1" + priority: 100 + description: "Strip Nokia revision suffixes (e.g. RA01, RB01, RG01) from ENTITY-MIB model strings" + +# ── Finisar / II-VI / Coherent suffix stripping ───────────────────────────── +# Finisar part numbers have customer-specific suffixes after a hyphen: +# FTLC1154RDPL-A5 (original Finisar) +# FTLC1154RDPL-C (Prolabs compatible) +# FTLX1474D3BCL-C1 (Cisco-coded Finisar) +# This rule strips everything after the last hyphen for FT... models. +- scope: module_type + match_pattern: "^(FT[A-Z0-9]+)-[A-Z0-9]+$" + replacement: "\\1" + priority: 100 + description: "Strip Finisar/II-VI customer suffixes (-A5, -C, -CN, -C1, etc.)" + +# ── Prolabs LGI- prefix stripping ─────────────────────────────────────────── +# Prolabs-compatible optics sometimes prepend LGI- to the OEM part number: +# LGI-FTLC9152RGPL β†’ FTLC9152RGPL +- scope: module_type + match_pattern: "^LGI-(.+)$" + replacement: "\\1" + priority: 50 + description: "Strip Prolabs LGI- prefix from OEM part numbers" + +# ── Nokia transceiver model field cleanup ──────────────────────────────────── +# Nokia transceiver API sometimes returns model strings with trailing vendor +# info: "3HE10550AARA01 NOK IPU3BFUEAA" β€” extract just the part number. +- scope: module_type + manufacturer: Nokia + match_pattern: "^(3HE\\w+)\\s+.*$" + replacement: "\\1" + priority: 50 + description: "Extract Nokia part number from transceiver model field (strip trailing vendor/oui info)" diff --git a/docs/usage_tips/custom_field.md b/docs/usage_tips/custom_field.md index 032812ed82..7ed27a2f97 100644 --- a/docs/usage_tips/custom_field.md +++ b/docs/usage_tips/custom_field.md @@ -4,6 +4,9 @@ To enhance device identification and synchronization between NetBox and LibreNMS, this plugin supports using a custom field `librenms_id` on Device, Virtual Machine and Interface objects. While the plugin works without it, using this custom field is recommended for LibreNMS API lookups, and to assist with matching the remote device and remote interfaces for cable creation in Netbox. It can also be entered manually if no primary IP or FQDN is available. +!!! info "Automatic Creation" + As of version 0.4.2, the plugin **automatically creates** the `librenms_id` custom field when migrations are run. You no longer need to create it manually. The field is created for Device, Virtual Machine, Interface, and VM Interface objects. + For the Device and Virtual Machine objects the plugin will automatically populate the LibreNMS ID custom field when opening the LibreNMS Sync page if the device has been found in LibreNMS. For the Interface object, the plugin will automatically populate the LibreNMS ID custom field when the interface data is synced from LibreNMS. @@ -15,7 +18,10 @@ For the Interface object, the plugin will automatically populate the LibreNMS ID - **Efficient Synchronization:** Enhances the reliability of API lookups. - **Cable creation:** Allows better device identification for the creation of cables between NetBox devices. -## Suggested Custom Field Setup +## Manual Custom Field Setup (Legacy) + +!!! note + This section is only needed if you are running an older version of the plugin that does not auto-create the field, or if you need to recreate it after deletion. Follow these steps to create the `librenms_id` custom field in NetBox: diff --git a/docs/usage_tips/permissions.md b/docs/usage_tips/permissions.md index 9f9ecfb3f3..39c5225d2a 100644 --- a/docs/usage_tips/permissions.md +++ b/docs/usage_tips/permissions.md @@ -26,7 +26,7 @@ A user needs both tiers of permissions to complete an action. For example, to vi The Plugin also enforces Netbox object permissions so the following permission would also be required: -2. **Tier 2: Object permission**: User needs `dcim.add_device` (to create the device in NetBox) +1. **Tier 2: Object permission**: User needs `dcim.add_device` (to create the device in NetBox) If either permission is missing, the operation fails with an appropriate error message. diff --git a/netbox_librenms_plugin/__init__.py b/netbox_librenms_plugin/__init__.py index f1720c85d6..d0499c53f3 100644 --- a/netbox_librenms_plugin/__init__.py +++ b/netbox_librenms_plugin/__init__.py @@ -28,6 +28,7 @@ def ready(self): super().ready() from django.conf import settings + from django.db.models.signals import post_migrate plugin_config = getattr(settings, "PLUGINS_CONFIG", {}).get(self.name, {}) @@ -37,6 +38,12 @@ def ready(self): else: self._validate_legacy_config(plugin_config) + # Auto-create the librenms_id custom field after migrations complete + post_migrate.connect( + _ensure_librenms_id_custom_field, + dispatch_uid="netbox_librenms_plugin_ensure_cf", + ) + def _validate_multi_server_config(self, servers_config): """Validate multi-server configuration.""" if not servers_config or not isinstance(servers_config, dict): @@ -61,4 +68,61 @@ def _validate_legacy_config(self, plugin_config): ) +def _ensure_librenms_id_custom_field(sender, **kwargs): + """ + Auto-create the 'librenms_id' custom field if it doesn't exist. + Runs after migrations via post_migrate signal to ensure tables exist. + Uses dispatch_uid to avoid duplicate connections. + """ + # Only run once per migrate invocation (post_migrate fires per-app). + # The _executed flag is intentionally never reset: migrations are expected to + # run in short-lived CLI processes (manage.py migrate) where the flag is + # naturally cleared on exit. Long-running processes (e.g. gunicorn workers) + # should not rely on this handler re-executing after startup. + if getattr(_ensure_librenms_id_custom_field, "_executed", False): + return + _ensure_librenms_id_custom_field._executed = True # not reset; see comment above + + import logging + + try: + from django.contrib.contenttypes.models import ContentType + + from extras.models import CustomField + + cf, created = CustomField.objects.get_or_create( + name="librenms_id", + defaults={ + "type": "integer", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Ensure the field is assigned to the required object types + from dcim.models import Device, Interface + from virtualization.models import VirtualMachine, VMInterface + + required_models = [Device, VirtualMachine, Interface, VMInterface] + current_types = set(cf.object_types.values_list("pk", flat=True)) + + for model in required_models: + ct = ContentType.objects.get_for_model(model) + if ct.pk not in current_types: + cf.object_types.add(ct) + + if created: + logging.getLogger("netbox_librenms_plugin").info( + "Auto-created 'librenms_id' custom field for Device, VirtualMachine, Interface, VMInterface" + ) + except Exception as e: + # Don't break startup if custom field creation fails (e.g., during initial migration), + # but log the error so it's not silently swallowed. + logging.getLogger("netbox_librenms_plugin").exception("Failed to auto-create 'librenms_id' custom field: %s", e) + + config = LibreNMSSyncConfig diff --git a/netbox_librenms_plugin/api/serializers.py b/netbox_librenms_plugin/api/serializers.py index 6bcd0aef20..bcde788d2b 100644 --- a/netbox_librenms_plugin/api/serializers.py +++ b/netbox_librenms_plugin/api/serializers.py @@ -1,6 +1,12 @@ from netbox.api.serializers import NetBoxModelSerializer -from netbox_librenms_plugin.models import InterfaceTypeMapping +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) class InterfaceTypeMappingSerializer(NetBoxModelSerializer): @@ -11,3 +17,51 @@ class Meta: model = InterfaceTypeMapping fields = ["id", "librenms_type", "librenms_speed", "netbox_type", "description"] + + +class DeviceTypeMappingSerializer(NetBoxModelSerializer): + """Serialize DeviceTypeMapping model for REST API.""" + + class Meta: + """Meta options for DeviceTypeMappingSerializer.""" + + model = DeviceTypeMapping + fields = ["id", "librenms_hardware", "netbox_device_type", "description"] + + +class ModuleTypeMappingSerializer(NetBoxModelSerializer): + """Serialize ModuleTypeMapping model for REST API.""" + + class Meta: + """Meta options for ModuleTypeMappingSerializer.""" + + model = ModuleTypeMapping + fields = ["id", "librenms_model", "netbox_module_type", "description"] + + +class ModuleBayMappingSerializer(NetBoxModelSerializer): + """Serialize ModuleBayMapping model for REST API.""" + + class Meta: + """Meta options for ModuleBayMappingSerializer.""" + + model = ModuleBayMapping + fields = ["id", "librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class NormalizationRuleSerializer(NetBoxModelSerializer): + """Serialize NormalizationRule model for REST API.""" + + class Meta: + """Meta options for NormalizationRuleSerializer.""" + + model = NormalizationRule + fields = [ + "id", + "scope", + "manufacturer", + "match_pattern", + "replacement", + "priority", + "description", + ] diff --git a/netbox_librenms_plugin/api/urls.py b/netbox_librenms_plugin/api/urls.py index 230aa078d0..c032e7b2f5 100644 --- a/netbox_librenms_plugin/api/urls.py +++ b/netbox_librenms_plugin/api/urls.py @@ -7,6 +7,10 @@ router = NetBoxRouter() router.register("interface-type-mappings", views.InterfaceTypeMappingViewSet) +router.register("device-type-mappings", views.DeviceTypeMappingViewSet) +router.register("module-type-mappings", views.ModuleTypeMappingViewSet) +router.register("module-bay-mappings", views.ModuleBayMappingViewSet) +router.register("normalization-rules", views.NormalizationRuleViewSet) urlpatterns = [ path("jobs//sync-status/", views.sync_job_status, name="sync_job_status"), diff --git a/netbox_librenms_plugin/api/views.py b/netbox_librenms_plugin/api/views.py index 768c67f5fe..287a3858c1 100644 --- a/netbox_librenms_plugin/api/views.py +++ b/netbox_librenms_plugin/api/views.py @@ -11,9 +11,21 @@ from rq.job import Job as RQJob from netbox_librenms_plugin.constants import PERM_CHANGE_PLUGIN, PERM_VIEW_PLUGIN -from netbox_librenms_plugin.models import InterfaceTypeMapping - -from .serializers import InterfaceTypeMappingSerializer +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) + +from .serializers import ( + DeviceTypeMappingSerializer, + InterfaceTypeMappingSerializer, + ModuleBayMappingSerializer, + ModuleTypeMappingSerializer, + NormalizationRuleSerializer, +) logger = logging.getLogger(__name__) @@ -22,8 +34,8 @@ class LibreNMSPluginPermission(BasePermission): """ Permission class for LibreNMS plugin API endpoints. - - GET requests require view_librenmssettings - - All other requests require change_librenmssettings + - Safe requests (GET, HEAD, OPTIONS) require netbox_librenms_plugin.view_librenmssettings + - All other requests require netbox_librenms_plugin.change_librenmssettings """ def has_permission(self, request, view): @@ -41,6 +53,42 @@ class InterfaceTypeMappingViewSet(NetBoxModelViewSet): serializer_class = InterfaceTypeMappingSerializer +class DeviceTypeMappingViewSet(NetBoxModelViewSet): + """API viewset for DeviceTypeMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = DeviceTypeMapping.objects.all() + serializer_class = DeviceTypeMappingSerializer + + +class ModuleTypeMappingViewSet(NetBoxModelViewSet): + """API viewset for ModuleTypeMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = ModuleTypeMapping.objects.all() + serializer_class = ModuleTypeMappingSerializer + + +class ModuleBayMappingViewSet(NetBoxModelViewSet): + """API viewset for ModuleBayMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = ModuleBayMapping.objects.all() + serializer_class = ModuleBayMappingSerializer + + +class NormalizationRuleViewSet(NetBoxModelViewSet): + """API viewset for NormalizationRule CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = NormalizationRule.objects.all() + serializer_class = NormalizationRuleSerializer + + @api_view(["POST"]) @permission_classes([LibreNMSPluginPermission]) def sync_job_status(request, job_pk): diff --git a/netbox_librenms_plugin/filters.py b/netbox_librenms_plugin/filters.py index 9ec162a64c..134bd8962d 100644 --- a/netbox_librenms_plugin/filters.py +++ b/netbox_librenms_plugin/filters.py @@ -1,6 +1,6 @@ import django_filters -from .models import InterfaceTypeMapping +from .models import DeviceTypeMapping, InterfaceTypeMapping, ModuleBayMapping, ModuleTypeMapping, NormalizationRule class InterfaceTypeMappingFilterSet(django_filters.FilterSet): @@ -11,3 +11,43 @@ class Meta: model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] + + +class DeviceTypeMappingFilterSet(django_filters.FilterSet): + """Filter set for DeviceTypeMapping model.""" + + class Meta: + """Meta options for DeviceTypeMappingFilterSet.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "description"] + + +class ModuleTypeMappingFilterSet(django_filters.FilterSet): + """Filter set for ModuleTypeMapping model.""" + + class Meta: + """Meta options for ModuleTypeMappingFilterSet.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "description"] + + +class ModuleBayMappingFilterSet(django_filters.FilterSet): + """Filter set for ModuleBayMapping model.""" + + class Meta: + """Meta options for ModuleBayMappingFilterSet.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex"] + + +class NormalizationRuleFilterSet(django_filters.FilterSet): + """Filter set for NormalizationRule model.""" + + class Meta: + """Meta options for NormalizationRuleFilterSet.""" + + model = NormalizationRule + fields = ["scope", "manufacturer"] diff --git a/netbox_librenms_plugin/forms.py b/netbox_librenms_plugin/forms.py index e22cf32222..445e170868 100644 --- a/netbox_librenms_plugin/forms.py +++ b/netbox_librenms_plugin/forms.py @@ -2,7 +2,7 @@ import logging from dcim.choices import InterfaceTypeChoices -from dcim.models import Device, DeviceRole, DeviceType, Location, Rack, Site +from dcim.models import Device, DeviceRole, DeviceType, Location, Manufacturer, Rack, Site from django import forms from django.http import QueryDict from django.utils.translation import gettext_lazy as _ @@ -12,10 +12,22 @@ NetBoxModelImportForm, ) from netbox.plugins import get_plugin_config -from utilities.forms.fields import CSVChoiceField, DynamicModelMultipleChoiceField +from utilities.forms.fields import ( + CSVChoiceField, + CSVModelChoiceField, + DynamicModelChoiceField, + DynamicModelMultipleChoiceField, +) from virtualization.models import Cluster, VirtualMachine -from .models import InterfaceTypeMapping, LibreNMSSettings +from .models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + LibreNMSSettings, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) logger = logging.getLogger(__name__) @@ -47,6 +59,52 @@ def _get_librenms_server_choices(): return choices +def _get_librenms_poller_group_choices(): + """ + Helper function to get poller group choices from LibreNMS API. + Shared between AddToLIbreSNMPV1V2 and AddToLIbreSNMPV3 forms. + Results are cached to avoid repeated API calls on every form instantiation. + """ + from django.core.cache import cache + + from .librenms_api import LibreNMSAPI + + choices = [("0", "Default (0)")] + + try: + api = LibreNMSAPI() + server_id = api.librenms_url.rstrip("/") + cache_key = f"librenms_poller_group_choices_{server_id}" + except Exception: + cache_key = "librenms_poller_group_choices" + cached_choices = cache.get(cache_key) + if cached_choices: + return cached_choices + + try: + api = LibreNMSAPI() + success, poller_groups = api.get_poller_groups() + + if success and poller_groups: + for group in poller_groups: + group_id = str(group.get("id", "")) + group_name = group.get("group_name", "") + group_descr = group.get("descr", "") + + if group_id: + if group_descr and group_descr != group_name: + label = f"{group_name} - {group_descr} ({group_id})" + else: + label = f"{group_name} ({group_id})" + choices.append((group_id, label)) + + cache.set(cache_key, choices, timeout=api.cache_timeout) + except Exception: + logger.exception("Failed to load LibreNMS poller groups") + + return choices + + class ServerConfigForm(NetBoxModelForm): """ Form for selecting the active LibreNMS server from configured servers. @@ -59,12 +117,14 @@ class ServerConfigForm(NetBoxModelForm): ) class Meta: + """Meta options for ServerConfigForm.""" + model = LibreNMSSettings fields = ["selected_server"] def __init__(self, *args, **kwargs): + """Initialize form and populate server choices.""" super().__init__(*args, **kwargs) - # Get available servers from configuration self.fields["selected_server"].choices = _get_librenms_server_choices() @@ -101,6 +161,8 @@ class ImportSettingsForm(NetBoxModelForm): ) class Meta: + """Meta options for ImportSettingsForm.""" + model = LibreNMSSettings fields = [ "vc_member_name_pattern", @@ -183,6 +245,8 @@ class InterfaceTypeMappingForm(NetBoxModelForm): """ class Meta: + """Meta options for InterfaceTypeMappingForm.""" + model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] @@ -200,6 +264,8 @@ class InterfaceTypeMappingImportForm(NetBoxModelImportForm): ) class Meta: + """Meta options for InterfaceTypeMappingImportForm.""" + model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] @@ -230,6 +296,163 @@ class InterfaceTypeMappingFilterForm(NetBoxModelFilterSetForm): model = InterfaceTypeMapping +class DeviceTypeMappingForm(NetBoxModelForm): + """Form for creating and editing device type mappings between LibreNMS and NetBox.""" + + netbox_device_type = forms.ModelChoiceField( + queryset=DeviceType.objects.all(), + label="NetBox Device Type", + widget=forms.Select(attrs={"class": "form-select"}), + ) + + class Meta: + """Meta options for DeviceTypeMappingForm.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "netbox_device_type", "description"] + + +class DeviceTypeMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing device type mappings.""" + + class Meta: + """Meta options for DeviceTypeMappingImportForm.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "netbox_device_type", "description"] + + +class DeviceTypeMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering device type mappings.""" + + librenms_hardware = forms.CharField(required=False, label="LibreNMS Hardware") + description = forms.CharField( + required=False, + label="Description", + help_text="Filter by description (partial match)", + ) + + model = DeviceTypeMapping + + +class ModuleTypeMappingForm(NetBoxModelForm): + """Form for creating and editing module type mappings between LibreNMS and NetBox.""" + + class Meta: + """Meta options for ModuleTypeMappingForm.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "netbox_module_type", "description"] + + +class ModuleTypeMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing module type mappings.""" + + class Meta: + """Meta options for ModuleTypeMappingImportForm.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "netbox_module_type", "description"] + + +class ModuleTypeMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering module type mappings.""" + + librenms_model = forms.CharField(required=False, label="LibreNMS Model") + description = forms.CharField( + required=False, + label="Description", + help_text="Filter by description (partial match)", + ) + + model = ModuleTypeMapping + + +class ModuleBayMappingForm(NetBoxModelForm): + """Form for creating and editing module bay mappings between LibreNMS and NetBox.""" + + class Meta: + """Meta options for ModuleBayMappingForm.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class ModuleBayMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing module bay mappings.""" + + class Meta: + """Meta options for ModuleBayMappingImportForm.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class ModuleBayMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering module bay mappings.""" + + librenms_name = forms.CharField(required=False, label="LibreNMS Name") + librenms_class = forms.CharField(required=False, label="LibreNMS Class") + netbox_bay_name = forms.CharField(required=False, label="NetBox Bay Name") + is_regex = forms.NullBooleanField(required=False, label="Regex") + + model = ModuleBayMapping + + +class NormalizationRuleForm(NetBoxModelForm): + """Form for creating and editing normalization rules.""" + + manufacturer = DynamicModelChoiceField( + queryset=Manufacturer.objects.all(), + required=False, + help_text="Optional: scope this rule to a specific manufacturer", + ) + + class Meta: + """Meta options for NormalizationRuleForm.""" + + model = NormalizationRule + fields = ["scope", "manufacturer", "match_pattern", "replacement", "priority", "description"] + + +class NormalizationRuleImportForm(NetBoxModelImportForm): + """Form for bulk importing normalization rules.""" + + scope = CSVChoiceField( + choices=NormalizationRule.SCOPE_CHOICES, + help_text="Scope: module_type, device_type, or module_bay", + ) + manufacturer = CSVModelChoiceField( + queryset=Manufacturer.objects.all(), + to_field_name="name", + required=False, + help_text="Optional manufacturer name (must already exist in NetBox)", + ) + + class Meta: + """Meta options for NormalizationRuleImportForm.""" + + model = NormalizationRule + fields = ["scope", "manufacturer", "match_pattern", "replacement", "priority", "description"] + + +class NormalizationRuleFilterForm(NetBoxModelFilterSetForm): + """Form for filtering normalization rules.""" + + scope = forms.ChoiceField( + required=False, + choices=[("", "---------")] + NormalizationRule.SCOPE_CHOICES, + label="Scope", + ) + manufacturer_id = DynamicModelChoiceField( + queryset=Manufacturer.objects.all(), + required=False, + label="Manufacturer", + ) + + model = NormalizationRule + + class AddToLIbreSNMPV1V2(forms.Form): """ Form for adding devices to LibreNMS using SNMPv1 or SNMPv2c authentication. @@ -285,38 +508,9 @@ class AddToLIbreSNMPV1V2(forms.Form): ) def __init__(self, *args, **kwargs): + """Initialize form and populate poller group choices.""" super().__init__(*args, **kwargs) - # Populate poller groups from LibreNMS API - self.fields["poller_group"].choices = self._get_poller_group_choices() - - def _get_poller_group_choices(self): - """Get poller group choices from LibreNMS API.""" - from .librenms_api import LibreNMSAPI - - choices = [("0", "Default (0)")] - - try: - api = LibreNMSAPI() - success, poller_groups = api.get_poller_groups() - - if success and poller_groups: - for group in poller_groups: - group_id = str(group.get("id", "")) - group_name = group.get("group_name", "") - group_descr = group.get("descr", "") - - if group_id: - # Format: "Group Name (ID)" or "Group Name - Description (ID)" - if group_descr and group_descr != group_name: - label = f"{group_name} - {group_descr} ({group_id})" - else: - label = f"{group_name} ({group_id})" - choices.append((group_id, label)) - except Exception: - # If API call fails, just use default option - pass - - return choices + self.fields["poller_group"].choices = _get_librenms_poller_group_choices() class AddToLIbreSNMPV3(forms.Form): @@ -412,38 +606,9 @@ class AddToLIbreSNMPV3(forms.Form): ) def __init__(self, *args, **kwargs): + """Initialize form and populate poller group choices.""" super().__init__(*args, **kwargs) - # Populate poller groups from LibreNMS API - self.fields["poller_group"].choices = self._get_poller_group_choices() - - def _get_poller_group_choices(self): - """Get poller group choices from LibreNMS API.""" - from .librenms_api import LibreNMSAPI - - choices = [("0", "Default (0)")] - - try: - api = LibreNMSAPI() - success, poller_groups = api.get_poller_groups() - - if success and poller_groups: - for group in poller_groups: - group_id = str(group.get("id", "")) - group_name = group.get("group_name", "") - group_descr = group.get("descr", "") - - if group_id: - # Format: "Group Name (ID)" or "Group Name - Description (ID)" - if group_descr and group_descr != group_name: - label = f"{group_name} - {group_descr} ({group_id})" - else: - label = f"{group_name} ({group_id})" - choices.append((group_id, label)) - except Exception: - # If API call fails, just use default option - pass - - return choices + self.fields["poller_group"].choices = _get_librenms_poller_group_choices() class DeviceStatusFilterForm(NetBoxModelFilterSetForm): @@ -452,6 +617,7 @@ class DeviceStatusFilterForm(NetBoxModelFilterSetForm): """ def __init__(self, *args, **kwargs): + """Initialize form and remove saved filter field.""" super().__init__(*args, **kwargs) # Remove the saved filter field if it exists if "filter_id" in self.fields: @@ -611,7 +777,9 @@ def _populate_librenms_locations(self): try: # Use caching to avoid repeated API calls - cache_key = "librenms_locations_choices" + api = LibreNMSAPI() + server_id = api.librenms_url.rstrip("/") + cache_key = f"librenms_locations_choices:{server_id}" cached_choices = cache.get(cache_key) if cached_choices: @@ -619,7 +787,6 @@ def _populate_librenms_locations(self): return # Fetch locations from LibreNMS - api = LibreNMSAPI() success, locations = api.get_locations() if success and locations: diff --git a/netbox_librenms_plugin/import_utils.py b/netbox_librenms_plugin/import_utils.py deleted file mode 100644 index cb9c9a99e7..0000000000 --- a/netbox_librenms_plugin/import_utils.py +++ /dev/null @@ -1,2334 +0,0 @@ -""" -Utilities for importing devices from LibreNMS to NetBox. - -This module provides functions for: -- Validating LibreNMS devices for import -- Retrieving filtered LibreNMS devices -- Importing single and multiple devices -- Smart matching of NetBox objects -- Permission checking for import operations -""" - -import logging -from typing import List - -from core.choices import JobStatusChoices -from dcim.models import Device, DeviceRole, DeviceType, Rack, Site, VirtualChassis -from django.core.cache import cache -from django.core.exceptions import PermissionDenied -from django.db import transaction -from django.utils import timezone -from virtualization.models import Cluster - -from .librenms_api import LibreNMSAPI -from .utils import ( - find_matching_platform, - find_matching_site, - match_librenms_hardware_to_device_type, -) - -logger = logging.getLogger(__name__) - - -# ============================================================================= -# Permission Check Helpers -# ============================================================================= - - -def check_user_permissions(user, permissions): - """ - Check if user has all required permissions. - - Args: - user: The user object to check permissions for - permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) - - Returns: - tuple: (has_all_permissions: bool, missing_permissions: list[str]) - - Raises: - PermissionDenied: If user is None (no user context available) - """ - if user is None: - raise PermissionDenied("No user context available for permission check") - - missing = [perm for perm in permissions if not user.has_perm(perm)] - return (len(missing) == 0, missing) - - -def require_permissions(user, permissions, action_description="perform this action"): - """ - Require user has all permissions, raising PermissionDenied if not. - - Args: - user: The user object to check permissions for - permissions: List of permission strings - action_description: Human-readable description for error message - - Raises: - PermissionDenied: If user lacks any required permission - """ - has_perms, missing = check_user_permissions(user, permissions) - if not has_perms: - missing_str = ", ".join(missing) - raise PermissionDenied( - f"You do not have permission to {action_description}. Missing permissions: {missing_str}" - ) - - -def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: - """ - Generate a consistent cache metadata key from filter parameters. - - Args: - server_key: LibreNMS server identifier - filters: Filter dictionary - vc_enabled: Whether VC detection is enabled - - Returns: - str: Consistent cache key for metadata - """ - # Sort filter items to ensure consistent key generation - filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) - return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" - - -def get_active_cached_searches(server_key: str) -> list[dict]: - """ - Retrieve all active cached searches for a server and enrich with display-friendly values. - - Enriches raw filter IDs with human-readable names by looking up location names - from cached choices and converting type codes to display names. - - Args: - server_key: LibreNMS server identifier - - Returns: - List of dicts containing cache metadata with enriched display_filters - """ - from datetime import datetime, timezone - - cache_index_key = f"librenms_cache_index_{server_key}" - cache_index = cache.get(cache_index_key, []) - - active_searches = [] - valid_cache_keys = [] - - # Get location and type choices for enriching display - location_choices = {} - type_choices = { - "": "All Types", - "network": "Network", - "server": "Server", - "storage": "Storage", - "wireless": "Wireless", - "firewall": "Firewall", - "power": "Power", - "appliance": "Appliance", - "printer": "Printer", - "loadbalancer": "Load Balancer", - "other": "Other", - } - - # Get cached location choices for enrichment - location_cache_key = "librenms_locations_choices" - cached_locations = cache.get(location_cache_key) - if cached_locations: - location_choices = dict(cached_locations) - - for cache_key in cache_index: - metadata = cache.get(cache_key) - if metadata: - # Cache still exists, calculate time remaining - cached_at = datetime.fromisoformat(metadata.get("cached_at")) - cache_timeout = metadata.get("cache_timeout", 300) - now = datetime.now(timezone.utc) - age_seconds = (now - cached_at).total_seconds() - remaining_seconds = max(0, cache_timeout - age_seconds) - - if remaining_seconds > 0: - # Add remaining time and cache key - metadata["remaining_seconds"] = int(remaining_seconds) - metadata["cache_key"] = cache_key - - # Enrich filters with human-readable display values - if "filters" in metadata: - display_filters = metadata["filters"].copy() - # Convert location ID to location name - if "location" in display_filters and display_filters["location"] in location_choices: - display_filters["location"] = location_choices[display_filters["location"]] - # Convert type code to display name - if "type" in display_filters and display_filters["type"] in type_choices: - display_filters["type"] = type_choices[display_filters["type"]] - metadata["display_filters"] = display_filters - else: - # Fallback if filters key missing - metadata["display_filters"] = {} - - active_searches.append(metadata) - valid_cache_keys.append(cache_key) - - # Clean up index if any keys have expired - if len(valid_cache_keys) < len(cache_index): - cache.set(cache_index_key, valid_cache_keys, timeout=3600) - - # Sort by most recent first - active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) - - return active_searches - - -def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: - """ - Generate a consistent cache key for validated device data. - - This ensures both synchronous and background job processing use the same - cache keys, avoiding duplicate validation work and cache entries. - - Args: - server_key: LibreNMS server key - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - device_id: LibreNMS device ID - vc_enabled: Whether virtual chassis detection was enabled - - Returns: - str: Cache key for the validated device - - Example: - >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) - >>> key - 'validated_device_default_-1234567890_123_vc' - """ - # Sort filters for consistent hashing - filter_hash = hash(str(sorted(filters.items()))) - vc_part = "vc" if vc_enabled else "novc" - return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" - - -def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: - """ - Generate cache key for raw LibreNMS device data. - - This key is used to cache raw device data (without validation metadata) - to avoid redundant API calls when users interact with dropdowns during - the import workflow. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server identifier for multi-server setups - - Returns: - str: Cache key for the device data - - Example: - >>> get_import_device_cache_key(123, "production") - 'import_device_data_production_123' - """ - return f"import_device_data_{server_key}_{device_id}" - - -def _determine_device_name( - libre_device: dict, - use_sysname: bool = True, - strip_domain: bool = False, - device_id: int | str = None, -) -> str: - """ - Determine the device/VM name from LibreNMS data. - - Centralized logic for building device names with consistent handling of: - - sysName vs hostname preference - - Domain stripping (avoiding IP addresses) - - Fallback to device_id when name is missing - - Args: - libre_device: Device data from LibreNMS - use_sysname: If True, prefer sysName; if False, use hostname - strip_domain: If True, strip domain suffix (e.g., '.example.com') - device_id: LibreNMS device ID for fallback name generation - - Returns: - str: The determined device name - - Example: - >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, - ... use_sysname=True, strip_domain=True) - 'router' - """ - # Determine base name based on use_sysname preference - if use_sysname: - name = libre_device.get("sysName") or libre_device.get("hostname") - else: - name = libre_device.get("hostname") or libre_device.get("sysName") - - # Fallback to device_id if no name found - if not name: - if device_id is not None: - name = f"device-{device_id}" - else: - name = libre_device.get("device_id", "unknown") - name = f"device-{name}" - - # Strip domain if requested (but not for IP addresses) - if strip_domain and name and "." in name: - try: - from ipaddress import ip_address - - ip_address(name) - # It's a valid IP address, don't strip - except ValueError: - # Not an IP, safe to strip domain - name = name.split(".")[0] - - return name - - -def empty_virtual_chassis_data() -> dict: - """Public helper for callers that need a blank VC payload.""" - - return { - "is_stack": False, - "member_count": 0, - "members": [], - "detection_error": None, - } - - -def _clone_virtual_chassis_data(data: dict | None) -> dict: - """Return a defensive copy of cached VC data to avoid shared references.""" - - if not data: - return empty_virtual_chassis_data() - - members = [] - for idx, member in enumerate(data.get("members", [])): - member_copy = member.copy() - raw_position = member_copy.get("position", idx) - try: - member_copy["position"] = int(raw_position) - except (TypeError, ValueError): - member_copy["position"] = idx - members.append(member_copy) - - member_count = data.get("member_count") or len(members) - - return { - "is_stack": bool(data.get("is_stack")), - "member_count": member_count, - "members": members, - "detection_error": data.get("detection_error"), - } - - -_VC_CACHE_VERSION = "v1" - - -def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: - server_key = getattr(api, "server_key", "default") - return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" - - -def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: - """Fetch (and cache) virtual chassis data for a LibreNMS device.""" - - if not api or device_id is None: - return empty_virtual_chassis_data() - - cache_key = _vc_cache_key(api, device_id) - if not force_refresh: - cached = cache.get(cache_key) - if cached is not None: - return _clone_virtual_chassis_data(cached) - - detection_data = detect_virtual_chassis_from_inventory(api, device_id) - if detection_data and "detection_error" not in detection_data: - detection_data["detection_error"] = None - - cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() - - cache_timeout = getattr(api, "cache_timeout", 300) or 300 - cache.set(cache_key, cache_value, timeout=cache_timeout) - return _clone_virtual_chassis_data(cache_value) - - -def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: - """ - Pre-warm the virtual chassis cache for multiple devices. - - This eliminates the 0.5-1s delay when rendering the import table - by proactively fetching VC data before validation. - - Args: - api: LibreNMSAPI instance - device_ids: List of LibreNMS device IDs to prefetch VC data for - force_refresh: When True, bypass cache and fetch fresh data - - Example: - >>> # Before rendering import table - >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) - >>> # Now all validate_device_for_import() calls hit cache instantly - """ - if not api or not device_ids: - return - - logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") - - for idx, device_id in enumerate(device_ids): - # This populates the cache if empty, or skips if already cached - try: - get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) - except (BrokenPipeError, ConnectionError, IOError, OSError) as e: - logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") - # Stop processing if connection is broken - return - except Exception as e: - # Log but continue for other errors - logger.warning(f"Error prefetching VC data for device {device_id}: {e}") - - logger.debug(f"VC cache warming complete for {len(device_ids)} devices") - - -def get_device_count_for_filters( - api: LibreNMSAPI, - filters: dict, - clear_cache: bool = False, - show_disabled: bool = True, -) -> int: - """ - Get count of LibreNMS devices matching filters. - - This is a lightweight function to determine device count for background job - decision making. Uses the same caching as get_librenms_devices_for_import(). - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname keys - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - - Returns: - int: Count of devices matching filters - """ - devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) - - # Filter out disabled devices if requested - if not show_disabled: - devices = [d for d in devices if d.get("status") == 1] - - return len(devices) - - -def get_librenms_devices_for_import( - api: LibreNMSAPI = None, - filters: dict = None, - server_key: str = None, - *, - force_refresh: bool = False, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Retrieve LibreNMS devices based on filters. - - Args: - api: LibreNMSAPI instance (if not provided, creates one with server_key) - filters: Dict containing filter parameters: - - location: LibreNMS location/site filter - - type: Device type filter - - os: Operating system filter - - hostname: Hostname filter (partial match) - - sysname: System name filter (partial match) - - status: Device status filter (1=up, 0=down) - - disabled: Include disabled devices (0=active only, 1=all) - server_key: Key for specific server configuration (used if api not provided) - force_refresh: When True, bypass the cache and fetch fresh data - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from - existing cache; from_cache=False means data was just fetched from LibreNMS. - """ - try: - # Use provided API instance or create a new one - if api is None: - api = LibreNMSAPI(server_key=server_key) - - # Build LibreNMS API filters using the type/query format - # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters - # NOTE: API only supports ONE type/query pair, so we'll use the most - # specific filter for the API and apply others client-side - api_filters = {} - client_filters = {} # Filters to apply after fetching from API - - if filters: - # Check for status filter first - it has special handling - if filters.get("status") is not None: - # Status filter uses special types that don't need query param - if filters["status"] == 1: - api_filters["type"] = "up" - elif filters["status"] == 0: - api_filters["type"] = "down" - - # Save ALL other filters for client-side filtering when status is used - if filters.get("location"): - client_filters["location"] = filters["location"] - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - else: - # Priority order for type/query filters: location > type > os > hostname > sysname - # Note: When sysname is combined with other filters, it's applied client-side for partial matching - # When sysname is alone, it uses API exact match (type=sysName) - # Note: hardware is always applied client-side for partial matching - # Use first available for API, save others for client-side filtering - if filters.get("location"): - api_filters["type"] = "location_id" - api_filters["query"] = filters["location"] - # Save remaining filters for client-side - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("type"): - api_filters["type"] = "type" - api_filters["query"] = filters["type"] - # Save remaining filters for client-side - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("os"): - api_filters["type"] = "os" - api_filters["query"] = filters["os"] - # Save remaining filters for client-side - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hostname"): - api_filters["type"] = "hostname" - api_filters["query"] = filters["hostname"] - # Save sysname and hardware for client-side - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("sysname"): - # sysname-only filter: Use API exact match (type=sysName&query=) - # This is safe - returns empty if no exact match found - api_filters["type"] = "sysName" - api_filters["query"] = filters["sysname"] - # Save hardware for client-side - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hardware"): - # hardware-only filter: apply client-side for partial matching - client_filters["hardware"] = filters["hardware"] - - # Note: disabled filter isn't directly supported by LibreNMS API - # We'll filter client-side if needed - - # Use caching to avoid repeated API calls - # Include both API and client filters in cache key - cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" - from_cache = False - - if force_refresh: - cache.delete(cache_key) - else: - cached_result = cache.get(cache_key) - if cached_result is not None: - # No need to deepcopy - cached data isn't mutated - devices = cached_result - from_cache = True - if return_cache_status: - return devices, from_cache - return devices - - success, devices = api.list_devices(api_filters if api_filters else None) - - if not success: - logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") - if return_cache_status: - return [], False - return [] - - # Apply client-side filters if any - if client_filters: - devices = _apply_client_filters(devices, client_filters) - - # Cache using configured timeout (default 300s) - # No need to deepcopy - Django's cache backend handles serialization - cache.set(cache_key, devices, timeout=api.cache_timeout) - - if return_cache_status: - return devices, from_cache - return devices - - except Exception: - logger.exception("Error retrieving LibreNMS devices for import") - if return_cache_status: - return [], False - return [] - - -def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: - """ - Apply client-side filters to device list. - - Args: - devices: List of device dicts from LibreNMS - filters: Dict of filters to apply (location, type, os, hostname, sysname) - - Returns: - Filtered list of devices - """ - filtered = devices - - if filters.get("location"): - location_id = str(filters["location"]) - filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] - - if filters.get("type"): - device_type = filters["type"].lower() - filtered = [d for d in filtered if d.get("type", "").lower() == device_type] - - if filters.get("os"): - os_filter = filters["os"].lower() - filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] - - if filters.get("hostname"): - hostname_filter = filters["hostname"].lower() - filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] - - if filters.get("sysname"): - sysname_filter = filters["sysname"].lower() - filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] - - if filters.get("hardware"): - hardware_filter = filters["hardware"].lower() - filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] - - return filtered - - -def validate_device_for_import( - libre_device: dict, - import_as_vm: bool = False, - api: "LibreNMSAPI" = None, - *, - include_vc_detection: bool = True, - force_vc_refresh: bool = False, -) -> dict: - """ - Validate if a LibreNMS device can be imported to NetBox. - - Performs comprehensive validation: - - Checks if device already exists in NetBox - - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) - OR (Cluster for VMs) - - Provides smart matching for missing objects - - Detects virtual chassis/stack configuration (if API provided) - - Returns detailed validation status - - Args: - libre_device: Device data from LibreNMS - import_as_vm: If True, validate for VM import instead of device import - api: Optional LibreNMSAPI instance for virtual chassis detection - include_vc_detection: Skip VC detection when False to speed up bulk operations - force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS - - Returns: - dict: Validation result with structure: - { - 'is_ready': bool, # Can import without user intervention - 'can_import': bool, # Can import (possibly after configuration) - 'import_as_vm': bool, # Whether importing as VM - 'existing_device': Device or VirtualMachine or None, - 'issues': List[str], # Blocking issues - 'warnings': List[str], # Non-blocking warnings - 'site': { # Only for devices - 'found': bool, - 'site': Site or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[Site] # Alternative suggestions - }, - 'device_type': { # Only for devices - 'found': bool, - 'device_type': DeviceType or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[dict] # Device types for user selection - }, - 'device_role': { # Only for devices - 'found': bool, # Always False - requires manual selection - 'role': DeviceRole or None, - 'available_roles': List[DeviceRole] # All roles for user selection - }, - 'cluster': { # Only for VMs - 'found': bool, # Always False - requires manual selection - 'cluster': Cluster or None, - 'available_clusters': List[Cluster] # All clusters for user selection - }, - 'platform': { - 'found': bool, - 'platform': Platform or None, - 'match_type': str # 'exact' or None - } - } - - Example: - >>> validation = validate_device_for_import(libre_device) - >>> if validation['is_ready']: - ... import_single_device(libre_device['device_id']) - """ - result = { - "is_ready": False, - "can_import": False, - "import_as_vm": import_as_vm, - "existing_device": None, - "existing_match_type": None, # Track how existing device was matched - "issues": [], - "warnings": [], - "virtual_chassis": empty_virtual_chassis_data(), - "site": { - "found": False, - "site": None, - "match_type": None, - "suggestions": [], - }, - "device_type": { - "found": False, - "device_type": None, - "match_type": None, - "suggestions": [], - }, - "device_role": { - "found": False, - "role": None, - "available_roles": [], - }, - "cluster": { - "found": False, - "cluster": None, - "available_clusters": [], - }, - "platform": {"found": False, "platform": None, "match_type": None}, - "rack": { - "found": False, - "rack": None, - "available_racks": [], - }, - } - - try: - # 1. Check if device/VM already exists in NetBox - # Always check both Devices AND VMs to properly detect existing objects - librenms_id = libre_device.get("device_id") - hostname = libre_device.get("hostname", "") - logger.debug( - f"Checking for existing device/VM: " - f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " - f"hostname={hostname}" - ) - - from virtualization.models import VirtualMachine - - # Check for existing VM first (by librenms_id custom field) - # Always query with int to match custom field type - try: - existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_vm = None - - if existing_vm: - logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_vm - result["existing_match_type"] = "librenms_id" - result["import_as_vm"] = True # Force VM mode since VM exists - result["warnings"].append(f"VM already imported to NetBox as '{existing_vm.name}'") - result["can_import"] = False - return result - - # Check for existing Device (by librenms_id custom field) - # Always query with int to match custom field type - try: - existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_device = None - - if existing_device: - logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_device - result["existing_match_type"] = "librenms_id" - result["warnings"].append(f"Device already imported to NetBox as '{existing_device.name}'") - result["can_import"] = False - return result - - # Check by hostname/name - Check both VMs and Devices for conflicts - existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() - existing_device = Device.objects.filter(name__iexact=hostname).first() - - # If BOTH exist with same hostname, it's ambiguous - don't match either - if existing_vm and existing_device: - logger.warning( - f"Hostname conflict: Both VM '{existing_vm.name}' and Device " - f"'{existing_device.name}' exist with hostname '{hostname}'" - ) - result["warnings"].append( - f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " - f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." - ) - # Don't set existing_device, don't block import - let user proceed as new - # This allows them to import and then resolve the conflict manually - elif existing_vm: - logger.info(f"Found existing VM by hostname: {existing_vm.name}") - result["existing_device"] = existing_vm - result["existing_match_type"] = "hostname" - result["import_as_vm"] = True # Force VM mode since VM exists - result["warnings"].append( - f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - elif existing_device: - logger.info(f"Found existing device by hostname: {existing_device.name}") - result["existing_device"] = existing_device - result["existing_match_type"] = "hostname" - result["warnings"].append( - f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - - # Check by primary IP (weaker match, IP could be reassigned) - only for devices - primary_ip = libre_device.get("ip") - if primary_ip and not import_as_vm: - from ipam.models import IPAddress - - existing_ip = IPAddress.objects.filter(address__startswith=primary_ip).first() - if existing_ip and existing_ip.assigned_object: - device = existing_ip.assigned_object.device if hasattr(existing_ip.assigned_object, "device") else None - if device: - result["existing_device"] = device - result["existing_match_type"] = "primary_ip" - result["warnings"].append( - f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - - # Validate based on import type (Device or VM) - if import_as_vm: - # 2. For VMs: Validate Cluster (required) - Must be manually selected - from virtualization.models import Cluster - - result["cluster"]["found"] = False - result["issues"].append("Cluster must be manually selected before importing as VM") - # Provide list of available clusters for user selection (cached) - cache_key = "librenms_import_all_clusters" - all_clusters = cache.get(cache_key) - if all_clusters is None: - all_clusters = list(Cluster.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_clusters, cache_timeout) - result["cluster"]["available_clusters"] = all_clusters - - # Skip device-specific validations for VMs - result["site"]["found"] = True # Not required for VMs - result["device_type"]["found"] = True # Not required for VMs - result["device_role"]["found"] = True # Not required for VMs - - else: - # 2. For Devices: Validate Site (required) - location = libre_device.get("location", "") - site_match = find_matching_site(location) - result["site"] = site_match - - if not site_match["found"]: - result["issues"].append(f"No matching site found for location: '{location}'") - # Get alternative suggestions - if location: - all_sites = Site.objects.all()[:10] # Limit for performance - result["site"]["suggestions"] = list(all_sites) - - # 3. Validate DeviceType (required) - hardware = libre_device.get("hardware", "") - dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match - - if not dt_match["matched"]: - result["issues"].append(f"No matching device type found for hardware: '{hardware}'") - # Get some device types for user to choose from - all_device_types = DeviceType.objects.all()[:10] - result["device_type"]["suggestions"] = [ - { - "device_type": dt, - "similarity": 0.0, # No fuzzy matching, just showing options - "match_field": None, - } - for dt in all_device_types - ] - else: - # Rename 'matched' to 'found' for consistency - result["device_type"]["found"] = dt_match["matched"] - result["device_type"]["device_type"] = dt_match["device_type"] - result["device_type"]["match_type"] = dt_match["match_type"] - - # 4. DeviceRole (required) - Must be manually selected by user - logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") - result["device_role"]["found"] = False - result["issues"].append("Device role must be manually selected before import") - logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") - # Provide list of available roles for user selection (cached) - cache_key = "librenms_import_all_roles" - all_roles = cache.get(cache_key) - if all_roles is None: - all_roles = list(DeviceRole.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_roles, cache_timeout) - result["device_role"]["available_roles"] = all_roles - - # 4b. Rack (optional) - Provide available racks for the matched site - if site_match["found"] and site_match["site"]: - site = site_match["site"] - # Use cache to optimize rack lookups per site - cache_key = f"librenms_import_racks_site_{site.pk}" - available_racks = cache.get(cache_key) - - if available_racks is None: - from dcim.models import Rack - from django.db.models import Q - - # Query racks for this site - include both: - # 1. Racks assigned to locations within the site - # 2. Racks directly assigned to the site (without location) - available_racks = list( - Rack.objects.filter(Q(location__site=site) | Q(site=site)) - .select_related("location", "site") - .order_by("location__name", "name") - ) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, available_racks, cache_timeout) - - result["rack"]["available_racks"] = available_racks - # Rack is optional, don't add to issues - result["rack"]["found"] = True # Mark as "found" even if None (optional field) - - # Skip VM-specific validations for devices - result["cluster"]["found"] = True # Not required for devices - - # 5. Match Platform (optional - same for both devices and VMs) - os = libre_device.get("os", "") - platform_match = find_matching_platform(os) - result["platform"] = platform_match - - if not platform_match["found"] and os: - result["warnings"].append(f"No matching platform found for OS: '{os}'") - - # 6. Additional validations - if not hostname: - result["issues"].append("Device has no hostname") - - # Serial number check - serial = libre_device.get("serial", "") - if serial and serial != "-": - existing_serial = Device.objects.filter(serial=serial).first() - if existing_serial: - result["warnings"].append(f"Serial number {serial} already exists on device: {existing_serial.name}") - - # 7. Virtual chassis detection (only for devices, not VMs) - if include_vc_detection and not import_as_vm and api is not None: - device_id = libre_device.get("device_id") - if device_id: - try: - logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") - vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) - logger.debug( - f"VC detection result: is_stack={vc_detection.get('is_stack')}, " - f"member_count={vc_detection.get('member_count')}, " - f"members={len(vc_detection.get('members', []))}" - ) - if vc_detection: - result["virtual_chassis"] = vc_detection - if vc_detection["is_stack"]: - logger.debug( - f"Virtual chassis CONFIRMED for device {hostname}: " - f"{vc_detection['member_count']} members" - ) - except Exception as e: - logger.exception(f"Exception during VC detection for device {hostname}: {e}") - result["virtual_chassis"]["detection_error"] = str(e) - else: - logger.debug(f"No device_id found for {hostname}") - - # 8. Determine if device/VM is ready to import - result["can_import"] = len(result["issues"]) == 0 - - if import_as_vm: - # For VMs: only cluster is required - result["is_ready"] = result["can_import"] and result["cluster"]["found"] - else: - # For Devices: site, device_type, and device_role are required - result["is_ready"] = ( - result["can_import"] - and result["site"]["found"] - and result["device_type"]["found"] - and result["device_role"]["found"] - ) - - logger.debug( - f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " - f"issues={len(result['issues'])}, can_import={result['can_import']}, " - f"issues_list={result['issues']}" - ) - - return result - - except Exception as e: - logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") - result["issues"].append(f"Validation error: {str(e)}") - return result - - -def import_single_device( - device_id: int, - server_key: str = None, - validation: dict = None, - manual_mappings: dict = None, - sync_options: dict = None, - libre_device: dict = None, -) -> dict: - """ - Import a single LibreNMS device to NetBox. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server configuration key - validation: Pre-computed validation dict (optional) - manual_mappings: Manual object mappings (optional): - - site_id: NetBox Site ID - - device_type_id: NetBox DeviceType ID - - device_role_id: NetBox DeviceRole ID - - platform_id: NetBox Platform ID (optional) - - rack_id: NetBox Rack ID (optional) - sync_options: Sync options (optional): - - sync_interfaces: bool (default True) - - sync_cables: bool (default True) - - sync_ips: bool (default True) - - sync_fields: bool (default True) - libre_device: Pre-fetched LibreNMS device data (optional). - If provided, skips API call to fetch device info. - - Returns: - dict: Import result with structure: - { - 'success': bool, - 'device': Device object or None, - 'message': str, - 'error': str or None, - 'synced': { - 'interfaces': int, - 'cables': int, - 'ip_addresses': int - } - } - """ - try: - api = LibreNMSAPI(server_key=server_key) - - # Use pre-fetched device data if provided, otherwise fetch from API - if libre_device is None: - success, libre_device = api.get_device_info(device_id) - if not success or not libre_device: - return { - "success": False, - "device": None, - "message": "", - "error": f"Failed to retrieve device {device_id} from LibreNMS", - "synced": {}, - } - - # Validate device if validation not provided - if validation is None: - validation = validate_device_for_import(libre_device) - - # Check if device already exists - if validation.get("existing_device"): - return { - "success": False, - "device": validation["existing_device"], - "message": "", - "error": f"Device already exists: {validation['existing_device'].name}", - "synced": {}, - } - - # Use validation-derived matches, allow manual mappings to override specific fields - site = validation["site"].get("site") - device_type = validation["device_type"].get("device_type") - device_role = validation["device_role"].get("role") - platform = validation["platform"].get("platform") - rack = validation.get("rack", {}).get("rack") - - if manual_mappings: - site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site - device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type - device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role - - platform_id = manual_mappings.get("platform_id") - if platform_id: - from dcim.models import Platform - - platform = Platform.objects.filter(id=platform_id).first() or platform - - rack_id = manual_mappings.get("rack_id") - if rack_id: - rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack - - rack = rack or validation.get("rack", {}).get("rack") - - # Validate required fields - if not site: - return { - "success": False, - "device": None, - "message": "", - "error": "Site is required but not provided", - "synced": {}, - } - if not device_type: - return { - "success": False, - "device": None, - "message": "", - "error": "Device type is required but not provided", - "synced": {}, - } - if not device_role: - return { - "success": False, - "device": None, - "message": "", - "error": "Device role is required but not provided", - "synced": {}, - } - - # Create device in NetBox - with transaction.atomic(): - # Determine device name based on sync options - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - device_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=device_id, - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - device_data = { - "name": device_name, - "site": site, - "device_type": device_type, - "role": device_role, - "status": "active" if libre_device.get("status") == 1 else "offline", - "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": int(device_id)}, - } - - # Add optional fields - if platform: - device_data["platform"] = platform - - if rack: - device_data["rack"] = rack - - serial = libre_device.get("serial", "") - if serial and serial != "-": - device_data["serial"] = serial - - location_name = libre_device.get("location", "") - if location_name and location_name != "-": - from dcim.models import Location - - # Try to find matching location within the site - location = Location.objects.filter(site=site, name__iexact=location_name).first() - if location: - device_data["location"] = location - - # Create the device - device = Device(**device_data) - device.full_clean() - device.save() - - # Sync additional data based on options - sync_options = sync_options or {} - synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} - - try: - # Sync interfaces - if sync_options.get("sync_interfaces", True): - # This is simplified - would need proper request context - # For now, just log that it should be done - logger.info(f"Interface sync should be performed for device {device.name}") - - # Sync cables - if sync_options.get("sync_cables", True): - logger.info(f"Cable sync should be performed for device {device.name}") - - # Sync IP addresses - if sync_options.get("sync_ips", True): - logger.info(f"IP address sync should be performed for device {device.name}") - - except Exception as e: - logger.warning(f"Error during post-import sync: {str(e)}") - # Don't fail the import if sync fails - - return { - "success": True, - "device": device, - "message": f"Successfully imported device: {device.name}", - "error": None, - "synced": synced, - } - - except Exception as e: - logger.exception(f"Error importing device {device_id}") - return { - "success": False, - "device": None, - "message": "", - "error": str(e), - "synced": {}, - } - - -def bulk_import_devices_shared( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Shared function for importing multiple LibreNMS devices to NetBox. - - Used by both synchronous imports and background jobs. Handles per-device error - collection and optional progress logging when job context is provided. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - job: Optional JobRunner instance for progress logging and cancellation checks - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous usage - >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) - >>> # Background job usage - >>> result = bulk_import_devices_shared([1, 2, 3], job=self) - """ - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - required_perms = [ - "dcim.add_device", - "dcim.add_interface", - "dcim.add_virtualchassis", - ] - require_permissions(user, required_perms, "import devices") - - total = len(device_ids) - success_list = [] - failed_list = [] - skipped_list = [] - vc_created_count = 0 - processed_vc_domains = set() # Track VCs already created by domain - - # Initialize API client once for all devices to avoid repeated config parsing - api = LibreNMSAPI(server_key=server_key) - - for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation every 5 devices - if job and idx % 5 == 0: - # Refresh job from DB to get current status - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): - if job.logger: - job.logger.warning(f"Import job cancelled at device {idx} of {total}") - else: - logger.warning(f"Import cancelled at device {idx} of {total}") - break - # Log progress - if job.logger: - job.logger.info(f"Imported device {idx} of {total}") - - try: - # Use cached device data if available to avoid redundant API calls - if libre_devices_cache and device_id in libre_devices_cache: - libre_device = libre_devices_cache[device_id] - success = True - else: - success, libre_device = api.get_device_info(device_id) - - if not success or not libre_device: - error_msg = f"Failed to retrieve device {device_id} from LibreNMS" - failed_list.append({"device_id": device_id, "error": error_msg}) - if job and job.logger: - job.logger.error(error_msg) - else: - logger.error(error_msg) - continue - - validation = validate_device_for_import(libre_device, api=api) - - # Build manual mappings from validation + any provided overrides - device_mappings = {} - - # Get site and device_type from validation - if validation["site"].get("found") and validation["site"].get("site"): - device_mappings["site_id"] = validation["site"]["site"].id - if validation["device_type"].get("found") and validation["device_type"].get("device_type"): - device_mappings["device_type_id"] = validation["device_type"]["device_type"].id - if validation["platform"].get("found") and validation["platform"].get("platform"): - device_mappings["platform_id"] = validation["platform"]["platform"].id - - # Override with any manual mappings provided for this device - if manual_mappings_per_device and device_id in manual_mappings_per_device: - device_mappings.update(manual_mappings_per_device[device_id]) - - result = import_single_device( - device_id, - server_key=server_key, - sync_options=sync_options, - manual_mappings=device_mappings if device_mappings else None, - libre_device=libre_device, - ) - - if result["success"]: - success_list.append( - { - "device_id": device_id, - "device": result["device"], - "message": result["message"], - } - ) - - # Handle virtual chassis creation for stacks - vc_data = validation.get("virtual_chassis", {}) - if vc_data.get("is_stack", False): - vc_domain = f"librenms-{device_id}" - - # Only create VC if we haven't processed this stack yet - # Add to set BEFORE attempting creation to prevent race condition - if vc_domain not in processed_vc_domains: - processed_vc_domains.add(vc_domain) - try: - vc = create_virtual_chassis_with_members( - result["device"], - vc_data["members"], - libre_device, - ) - vc_created_count += 1 - log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" - if job and job.logger: - job.logger.info(log_msg) - else: - logger.info(log_msg) - except Exception as vc_error: - # Remove from set on failure so retry is possible - processed_vc_domains.discard(vc_domain) - warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" - if job and job.logger: - job.logger.warning(warn_msg) - else: - logger.warning(warn_msg) - # Don't fail the import, just log the warning - - elif result.get("device"): # Device exists - skipped_list.append({"device_id": device_id, "reason": result["error"]}) - else: # Failed to import - failed_list.append({"device_id": device_id, "error": result["error"]}) - if job and job.logger: - job.logger.error(f"Failed to import device {device_id}: {result['error']}") - - except Exception as e: - error_msg = f"Unexpected error importing device {device_id}: {str(e)}" - if job and job.logger: - job.logger.error(error_msg, exc_info=True) - else: - logger.exception(f"Unexpected error importing device {device_id}") - failed_list.append({"device_id": device_id, "error": str(e)}) - - return { - "total": total, - "success": success_list, - "failed": failed_list, - "skipped": skipped_list, - "virtual_chassis_created": vc_created_count, - } - - -def bulk_import_devices( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices to NetBox (synchronous). - - This is the public API for synchronous imports. For background job usage, - use bulk_import_devices_shared() with a job context. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - user: User performing the import (for permission checks) - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - """ - return bulk_import_devices_shared( - device_ids=device_ids, - server_key=server_key, - sync_options=sync_options, - manual_mappings_per_device=manual_mappings_per_device, - libre_devices_cache=libre_devices_cache, - job=None, # No job context for synchronous imports - user=user, - ) - - -def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: - """ - Retrieve a single device from LibreNMS by ID. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - Device dictionary or None if not found - """ - try: - # Use the dedicated API endpoint to get device by ID - success, device = api.get_device_info(device_id) - if success and device: - return device - - logger.warning(f"Device {device_id} not found in LibreNMS") - return None - except Exception as e: - logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") - return None - - -def fetch_device_with_cache( - device_id: int, - api: LibreNMSAPI, - server_key: str = None, - libre_devices_cache: dict = None, -) -> dict | None: - """ - Fetch LibreNMS device from cache or API with automatic caching. - - Checks three sources in order: - 1. Pre-fetched cache dict (if provided) - 2. Django cache (Redis/memory) - 3. LibreNMS API (caches result for future use) - - This function consolidates the device fetching pattern used throughout - the import workflow, eliminating code duplication. - - Args: - device_id: LibreNMS device ID to fetch - api: LibreNMSAPI instance for fallback API calls - server_key: Optional server key for multi-server setups (defaults to api.server_key) - libre_devices_cache: Optional pre-fetched device cache dict - - Returns: - Device dict from LibreNMS, or None if not found - - Example: - >>> # Simple usage - >>> libre_device = fetch_device_with_cache(123, api) - >>> if libre_device: - ... print(libre_device['hostname']) - >>> - >>> # With pre-fetched cache dict - >>> cache_dict = {123: {...}, 456: {...}} - >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) - """ - # Check pre-fetched cache dict first (fastest) - if libre_devices_cache and device_id in libre_devices_cache: - return libre_devices_cache[device_id] - - # Check Django cache - cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) - libre_device = cache.get(cache_key) - - if not libre_device: - # Fallback to API fetch - libre_device = get_librenms_device_by_id(api, device_id) - if libre_device: - # Cache for future use - cache.set(cache_key, libre_device, timeout=api.cache_timeout) - - return libre_device - - -def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): - """ - Create a NetBox VirtualMachine from LibreNMS device data. - - Args: - libre_device: Device data from LibreNMS - validation: Validation result from validate_device_for_import with import_as_vm=True - use_sysname: If True, prefer sysName; if False, use hostname - role: Optional DeviceRole to assign to the VM - - Returns: - Created VirtualMachine instance - - Raises: - Exception if VM cannot be created - """ - from virtualization.models import VirtualMachine - - if not validation["can_import"]: - raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") - - # Extract matched objects from validation - cluster = validation["cluster"]["cluster"] - platform = validation["platform"].get("platform") - - # Determine VM name - use pre-computed name if available (handles strip_domain) - vm_name = libre_device.get("_computed_name") - if not vm_name: - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=False, - device_id=libre_device.get("device_id"), - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - # Create the VM with librenms_id custom field - vm = VirtualMachine.objects.create( - name=vm_name, - cluster=cluster, - role=role, # Optional VM role - platform=platform, - comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - custom_field_data={"librenms_id": int(libre_device["device_id"])}, - ) - - logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") - return vm - - -def bulk_import_vms( - vm_imports: dict[int, dict[str, int]], - api: LibreNMSAPI, - sync_options: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices as VMs in NetBox. - - Handles validation, cluster/role assignment, name determination, - and VM creation. Supports both synchronous and background job execution. - - This function consolidates VM import logic that was previously duplicated - in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior - across synchronous and background import paths. - - Args: - vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} - api: LibreNMSAPI instance for device fetching - sync_options: Optional dict with use_sysname, strip_domain settings - libre_devices_cache: Optional pre-fetched device data cache - job: Optional JobRunner instance for background job logging/cancellation - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - Dict with keys: - - success: List of {"device_id": int, "device": VM, "message": str} - - failed: List of {"device_id": int, "error": str} - - skipped: List of {"device_id": int, "reason": str} - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous import from view - >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} - >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) - >>> print(f"Created {len(result['success'])} VMs") - >>> - >>> # Background job import - >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) - """ - from netbox_librenms_plugin.import_validation_helpers import ( - apply_cluster_to_validation, - apply_role_to_validation, - ) - - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") - - result = {"success": [], "failed": [], "skipped": []} - vm_ids = list(vm_imports.keys()) - - # Use job logger if available, otherwise standard logger - log = job.logger if job else logger - - for idx, vm_id in enumerate(vm_ids, start=1): - # Check for job cancellation every 5 VMs - if job and idx % 5 == 0: - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in ("failed", "errored"): - log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") - break - log.info(f"Imported VM {idx} of {len(vm_ids)}") - - try: - # Fetch device data (uses cache helper) - libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) - - if not libre_device: - result["failed"].append( - { - "device_id": vm_id, - "error": f"Device {vm_id} not found in LibreNMS", - } - ) - log.error(f"Device {vm_id} not found in LibreNMS") - continue - - # Validate as VM - validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) - - # Check if VM already exists - if validation.get("existing_device"): - result["skipped"].append( - { - "device_id": vm_id, - "reason": f"VM already exists: {validation['existing_device'].name}", - } - ) - log.info(f"VM already exists: {validation['existing_device'].name}") - continue - - # Apply manual cluster and role selections - vm_mappings = vm_imports[vm_id] - cluster_id = vm_mappings.get("cluster_id") - role_id = vm_mappings.get("device_role_id") - - if cluster_id: - cluster = Cluster.objects.filter(id=cluster_id).first() - if cluster: - apply_cluster_to_validation(validation, cluster) - - role = None - if role_id: - role = DeviceRole.objects.filter(id=role_id).first() - if role: - apply_role_to_validation(validation, role, is_vm=True) - - # Determine VM name - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=vm_id, - ) - - # Update validation with computed name - libre_device["_computed_name"] = vm_name - - # Create VM - vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) - - result["success"].append( - { - "device_id": vm_id, - "device": vm, - "message": f"VM {vm.name} created successfully", - } - ) - log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") - - except Exception as vm_error: - log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) - result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) - - return result - - -def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: - """ - Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. - Vendor-agnostic using standard hierarchical structure. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - dict with structure: - { - 'is_stack': bool, - 'member_count': int, - 'members': [ - { - 'serial': str, - 'position': int, - 'model': str, - 'name': str, - 'index': int, - 'description': str, - 'suggested_name': str # Generated using master device name - } - ] - } - Returns None if not a stack or detection fails. - - Detection Logic: - 1. Check root level (entPhysicalContainedIn=0) for parent container - 2. Find parent index (entPhysicalClass='stack' or 'chassis') - 3. Get children chassis at that parent's index - 4. If multiple chassis found β†’ Stack detected - """ - try: - # Get the master device info to use for naming - success, device_info = api.get_device_info(device_id) - master_name = None - if success and device_info: - master_name = device_info.get("sysName") or device_info.get("hostname") - - # Step 1: Get root level items - success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) - - if not success or not root_items: - logger.debug(f"No root inventory items found for device {device_id}") - return None - - # Step 2: Find parent container index - # Could be class="stack" or the main "chassis" - parent_index = None - for item in root_items: - item_class = item.get("entPhysicalClass") - if item_class in ["stack", "chassis"]: - parent_index = item.get("entPhysicalIndex") - logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") - break - - if not parent_index: - return None - - # Step 3: Get children chassis at next level - success, child_items = api.get_inventory_filtered( - device_id, - ent_physical_class="chassis", - ent_physical_contained_in=parent_index, - ) - - if not success: - return None - - # Filter for chassis only (in case API filter didn't work) - chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] - - # Step 4: Multiple chassis = stack - if len(chassis_items) <= 1: - return None - - # Step 5: Extract member info - members = [] - for idx, chassis in enumerate(chassis_items): - raw_position = chassis.get("entPhysicalParentRelPos", idx) - try: - position = int(raw_position) - except (TypeError, ValueError): - position = idx - member_data = { - "serial": chassis.get("entPhysicalSerialNum", ""), - "position": position, - "model": chassis.get("entPhysicalModelName", ""), - "name": chassis.get("entPhysicalName", ""), - "index": chassis.get("entPhysicalIndex"), - "description": chassis.get("entPhysicalDescr", ""), - } - - # Generate suggested name if we have master name - if master_name: - member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) - else: - member_data["suggested_name"] = f"Member-{position + 1}" - - members.append(member_data) - - # Sort by position - members.sort(key=lambda m: m["position"]) - - logger.info(f"Detected stack with {len(members)} members for device {device_id}") - - return {"is_stack": True, "member_count": len(members), "members": members} - - except Exception as e: - logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") - return None - - -def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: - """ - Generate name for VC member device using configured pattern from settings. - - Args: - master_name: Name of the master/primary device - position: VC position number - serial: Optional serial number of the member device - - Returns: - Generated member device name - - Examples: - pattern="-M{position}" β†’ "switch01-M2" - pattern=" ({position})" β†’ "switch01 (2)" - pattern="-SW{position}" β†’ "switch01-SW2" - pattern=" [{serial}]" β†’ "switch01 [ABC123]" - """ - # Import here to avoid circular dependency - from .models import LibreNMSSettings - - # Get pattern from settings with fallback to default - try: - settings = LibreNMSSettings.objects.first() - pattern = settings.vc_member_name_pattern if settings else "-M{position}" - except Exception as e: - logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") - pattern = "-M{position}" - - # Prepare format variables - format_vars = { - "master_name": master_name, - "position": position, - "serial": serial or "", - } - - # Apply pattern - pattern should be suffix/prefix, not full name - try: - formatted_suffix = pattern.format(**format_vars) - return f"{master_name}{formatted_suffix}" - except KeyError as e: - logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") - return f"{master_name}-M{position}" - - -def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: - """ - Regenerate suggested VC member names using the actual master device name. - - This ensures preview shows accurate names after use_sysname and strip_domain - are applied to the master device name. - - Args: - vc_data: Virtual chassis detection data dict - master_name: The actual name that will be used for master device in NetBox - - Returns: - Updated vc_data dict with corrected suggested_name for each member - """ - if not vc_data or not vc_data.get("is_stack"): - return vc_data - - for idx, member in enumerate(vc_data.get("members", [])): - raw_position = member.get("position", idx) - try: - base_position = int(raw_position) - except (TypeError, ValueError): - base_position = idx - position = base_position + 1 # Convert to 1-based position - member["position"] = base_position - member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) - - return vc_data - - -def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): - """ - Create Virtual Chassis and member devices from detection info. - - This function creates a NetBox VirtualChassis with the master device - and all detected member devices, wrapped in a transaction for safety. - - Args: - master_device: The imported device (becomes VC master) - members_info: List of member dicts from VC detection - libre_device: Original LibreNMS device data - - Returns: - VirtualChassis: The created virtual chassis instance - - Raises: - ValidationError: If member count validation fails - IntegrityError: If duplicate serials/names are detected - Exception: For other creation errors - - Example members_info: - [ - {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, - {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} - ] - """ - - # Store original master device state for rollback - original_master_name = master_device.name - original_vc = master_device.virtual_chassis - original_vc_position = master_device.vc_position - - try: - with transaction.atomic(): - # Rename master device to include position 1 pattern - master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) - - # Check if renamed master conflicts with existing device - if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): - logger.warning( - f"Cannot rename master to '{master_device_new_name}' - name already exists. " - f"Keeping original name '{original_master_name}'" - ) - master_base_name = original_master_name - else: - master_device.name = master_device_new_name - master_base_name = original_master_name - - # Create VC using original base name - vc_name = master_base_name - vc = VirtualChassis.objects.create( - name=vc_name, - master=master_device, - domain=f"librenms-{libre_device['device_id']}", - ) - - # Update master device - master_device.virtual_chassis = vc - master_device.vc_position = 1 # Master is position 1 - master_device.save() - - # Create member devices for remaining positions - position = 2 # Start at 2 (master is 1) - members_created = 0 - - for member in members_info: - # Skip if this is the master's serial - if member.get("serial") == master_device.serial: - continue - - serial = member.get("serial") - - member_rack = master_device.rack - member_location = master_device.location or ( - member_rack.location if member_rack and member_rack.location else None - ) - - # Check for duplicate serial - if serial and Device.objects.filter(serial=serial).exists(): - logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") - continue - - member_name = _generate_vc_member_name(master_base_name, position, serial=serial) - - # Check for duplicate name - if Device.objects.filter(name=member_name).exists(): - logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") - continue - - Device.objects.create( - name=member_name, - device_type=master_device.device_type, - role=master_device.role, - site=master_device.site, - location=member_location, - rack=member_rack, - platform=master_device.platform, - serial=serial, - virtual_chassis=vc, - vc_position=position, - comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" - f"Auto-created from stack inventory", - ) - members_created += 1 - position += 1 - - # Validate member count - expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) - if members_created < expected_members: - logger.warning( - f"Created {members_created} members but expected {expected_members}. " - "Some members may have been skipped due to duplicates." - ) - - logger.info( - f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " - f"(1 master + {members_created} additional)" - ) - - return vc - - except Exception as e: - # Rollback master device to original state - logger.error( - f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." - ) - master_device.name = original_master_name - master_device.virtual_chassis = original_vc - master_device.vc_position = original_vc_position - master_device.save() - raise - - -def process_device_filters( - api: LibreNMSAPI, - filters: dict, - vc_detection_enabled: bool, - clear_cache: bool, - show_disabled: bool, - exclude_existing: bool = False, - job=None, - request=None, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Process LibreNMS device filters and return validated devices. - - Shared function used by both synchronous view and background job processing. - Fetches devices, optionally pre-warms VC cache, validates each device, and - caches results for HTMX row updates. - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - vc_detection_enabled: Whether to detect virtual chassis - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - exclude_existing: Whether to exclude devices that already exist in NetBox - job: Optional JobRunner instance for logging job events - request: Optional Django request for client disconnect detection (synchronous only) - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from existing - cache; from_cache=False means data was just fetched from LibreNMS. - """ - # Fetch devices from LibreNMS - if job: - job.logger.info(f"Fetching devices with filters: {filters}") - else: - logger.info(f"Fetching devices with filters: {filters}") - - # Always get cache status internally, even if not returning it - # We need it to determine if metadata should be updated - libre_devices, from_cache = get_librenms_devices_for_import( - api, - filters=filters, - force_refresh=clear_cache, - return_cache_status=True, - ) - - # Filter out disabled devices if requested - if not show_disabled: - libre_devices = [d for d in libre_devices if d.get("status") == 1] - - if job: - job.logger.info(f"Found {len(libre_devices)} devices to process") - else: - logger.info(f"Found {len(libre_devices)} devices") - - # Pre-warm VC cache if needed - if vc_detection_enabled and libre_devices: - device_ids = [d["device_id"] for d in libre_devices] - if job: - job.logger.info( - f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." - ) - else: - logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") - - try: - prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) - if job: - job.logger.info("Virtual chassis data pre-fetch completed") - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during VC prefetch: {e}") - return [] - raise - - # Validate each device - validated_devices = [] - total = len(libre_devices) - api_for_validation = api if vc_detection_enabled else None - - if job: - job.logger.info(f"Starting validation of {total} devices") - # Initial check if job was already terminated before we even started - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - if rq_job.is_failed or rq_job.is_stopped: - job.logger.warning("Job was already stopped before validation started") - return [] - except Exception: - # Fall back to DB check if RQ check fails - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.warning("Job was stopped before validation started") - return [] - else: - logger.info(f"Validating {total} devices") - - for idx, device in enumerate(libre_devices, 1): - # Check for job termination or client disconnect periodically - if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) - if job: - # Check if job was terminated via stop API - # CRITICAL: Check the RQ job status in Redis, not just the DB model - # NetBox's stop endpoint marks the RQ job as failed in Redis - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - # Check if RQ job is in a stopped state - if rq_job.is_failed or rq_job.is_stopped: - job.logger.info( - f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." - ) - return [] - except Exception: - # If we can't check RQ status, fall back to DB status check - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return [] - elif request: - # Check for client disconnect - try: - if hasattr(request, "META") and request.META.get("wsgi.input"): - pass - except (BrokenPipeError, ConnectionError, IOError): - logger.info(f"Client disconnected during validation at device {idx}") - return [] - - # Drop any cached validation/meta keys before recomputing - device.pop("_validation", None) - - # Generate shared cache key for this validated device - device_id = device["device_id"] - cache_key = get_validated_device_cache_key( - server_key=api.server_key, - filters=filters, - device_id=device_id, - vc_enabled=vc_detection_enabled, - ) - - # Check if we already have cached validation for this device - # (only if not forcing refresh) - if not clear_cache: - cached_device = cache.get(cache_key) - if cached_device: - # Use cached validation - device["_validation"] = cached_device["_validation"] - - # Apply exclude_existing filter if enabled - if exclude_existing: - validation = device["_validation"] - if validation["existing_device"]: - continue - - validated_devices.append(device) - continue - - # Not in cache or forcing refresh - validate now - try: - validation = validate_device_for_import( - device, - api=api_for_validation, - include_vc_detection=vc_detection_enabled, - force_vc_refresh=clear_cache, - ) - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during device validation: {e}") - return [] - raise - - # Set VC detection metadata - if not vc_detection_enabled: - validation["virtual_chassis"] = empty_virtual_chassis_data() - - # Apply exclude_existing filter if enabled - if exclude_existing and validation["existing_device"]: - continue - - device["_validation"] = validation - validated_devices.append(device) - - # Cache with TWO keys for different purposes: - # 1. Complex key (with filter context) - for full validated device with all metadata - cache.set(cache_key, device, timeout=api.cache_timeout) - - # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates - # This avoids redundant API calls when user interacts with dropdowns - simple_cache_key = get_import_device_cache_key(device_id, api.server_key) - # Cache just the raw device data (not the full validation result) - # This is what get_validated_device_with_selections() expects - device_data_only = {k: v for k, v in device.items() if k != "_validation"} - cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) - - # Store cache metadata (timestamp) for all filter operations - # This enables countdown display regardless of background job vs synchronous execution - # Always store metadata when we have validated devices, even if from_cache - # This ensures metadata is available for countdown display - if validated_devices: - from datetime import datetime, timezone - - cache_metadata_key = get_cache_metadata_key( - server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled - ) - - # Check if metadata already exists to preserve original timestamp - # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it - existing_metadata = cache.get(cache_metadata_key) - should_update = clear_cache or not from_cache - - if existing_metadata and not should_update: - # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) - pass - else: - # No metadata exists, OR cache was cleared, OR fresh data - create/update it now - cache_metadata = { - "cached_at": datetime.now(timezone.utc).isoformat(), - "cache_timeout": api.cache_timeout, - "filters": filters, - "vc_enabled": vc_detection_enabled, - "device_count": len(validated_devices), - } - cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) - - # Maintain cache index for this server to enable listing active searches - cache_index_key = f"librenms_cache_index_{api.server_key}" - cache_index = cache.get(cache_index_key, []) - # Add this cache key if not already in index - if cache_metadata_key not in cache_index: - cache_index.append(cache_metadata_key) - # Store index with same timeout as the metadata - cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) - - if job: - if exclude_existing: - filtered_count = total - len(validated_devices) - job.logger.info( - f"Validation complete: {len(validated_devices)} devices passed filter, " - f"{filtered_count} filtered out (existing devices excluded)" - ) - else: - job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") - else: - logger.info(f"Processed {len(validated_devices)} validated devices") - - if return_cache_status: - return validated_devices, from_cache - return validated_devices diff --git a/netbox_librenms_plugin/import_utils/__init__.py b/netbox_librenms_plugin/import_utils/__init__.py new file mode 100644 index 0000000000..2a35d18563 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/__init__.py @@ -0,0 +1,54 @@ +""" +Utilities for importing devices from LibreNMS to NetBox. + +This package provides functions for: +- Validating LibreNMS devices for import +- Retrieving filtered LibreNMS devices +- Importing single and multiple devices +- Smart matching of NetBox objects +- Permission checking for import operations +- Virtual chassis detection and creation + +All imports below are intentional re-exports so that existing callers +can continue using ``from netbox_librenms_plugin.import_utils import X``. +The F401 suppressions prevent linters from flagging them as unused. +""" + +from .bulk_import import ( # noqa: F401 + _refresh_existing_device, + bulk_import_devices, + bulk_import_devices_shared, + process_device_filters, +) +from .cache import ( # noqa: F401 + get_active_cached_searches, + get_cache_metadata_key, + get_import_device_cache_key, + get_validated_device_cache_key, +) +from .device_operations import ( # noqa: F401 + _determine_device_name, + _try_chassis_device_type_match, + fetch_device_with_cache, + get_librenms_device_by_id, + import_single_device, + validate_device_for_import, +) +from .filters import ( # noqa: F401 + _apply_client_filters, + get_device_count_for_filters, + get_librenms_devices_for_import, +) +from .permissions import check_user_permissions, require_permissions # noqa: F401 +from .virtual_chassis import ( # noqa: F401 + _clone_virtual_chassis_data, + _generate_vc_member_name, + _vc_cache_key, + create_virtual_chassis_with_members, + detect_virtual_chassis_from_inventory, + empty_virtual_chassis_data, + get_virtual_chassis_data, + prefetch_vc_data_for_devices, + update_vc_member_suggested_names, +) +from .vm_operations import bulk_import_vms, create_vm_from_librenms # noqa: F401 diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py new file mode 100644 index 0000000000..0bf5fb82ca --- /dev/null +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -0,0 +1,607 @@ +"""Bulk import orchestration for devices and filter processing.""" + +import logging +from typing import List + +from core.choices import JobStatusChoices +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI +from .cache import get_cache_metadata_key, get_import_device_cache_key, get_validated_device_cache_key +from .device_operations import import_single_device, validate_device_for_import +from .filters import get_librenms_devices_for_import +from .permissions import require_permissions +from .virtual_chassis import ( + create_virtual_chassis_with_members, + empty_virtual_chassis_data, + prefetch_vc_data_for_devices, +) + +logger = logging.getLogger(__name__) + + +def bulk_import_devices_shared( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Shared function for importing multiple LibreNMS devices to NetBox. + + Used by both synchronous imports and background jobs. Handles per-device error + collection and optional progress logging when job context is provided. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + job: Optional JobRunner instance for progress logging and cancellation checks + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous usage + >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) + >>> # Background job usage + >>> result = bulk_import_devices_shared([1, 2, 3], job=self) + """ + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + required_perms = [ + "dcim.add_device", + "dcim.add_interface", + "dcim.add_virtualchassis", + ] + require_permissions(user, required_perms, "import devices") + + total = len(device_ids) + success_list = [] + failed_list = [] + skipped_list = [] + vc_created_count = 0 + processed_vc_domains = set() # Track VCs already created by domain + + # Initialize API client once for all devices to avoid repeated config parsing + api = LibreNMSAPI(server_key=server_key) + + for idx, device_id in enumerate(device_ids, start=1): + # Check for job cancellation every 5 devices + if job and idx % 5 == 0: + # Refresh job from DB to get current status + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): + if job.logger: + job.logger.warning(f"Import job cancelled at device {idx} of {total}") + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + # Log progress + if job.logger: + job.logger.info(f"Imported device {idx} of {total}") + + try: + # Use cached device data if available to avoid redundant API calls + if libre_devices_cache and device_id in libre_devices_cache: + libre_device = libre_devices_cache[device_id] + success = True + else: + success, libre_device = api.get_device_info(device_id) + + if not success or not libre_device: + error_msg = f"Failed to retrieve device {device_id} from LibreNMS" + failed_list.append({"device_id": device_id, "error": error_msg}) + if job and job.logger: + job.logger.error(error_msg) + else: + logger.error(error_msg) + continue + + validation = validate_device_for_import(libre_device, api=api) + + # Build manual mappings from validation + any provided overrides + device_mappings = {} + + # Get site and device_type from validation + if validation["site"].get("found") and validation["site"].get("site"): + device_mappings["site_id"] = validation["site"]["site"].id + if validation["device_type"].get("found") and validation["device_type"].get("device_type"): + device_mappings["device_type_id"] = validation["device_type"]["device_type"].id + if validation["platform"].get("found") and validation["platform"].get("platform"): + device_mappings["platform_id"] = validation["platform"]["platform"].id + + # Override with any manual mappings provided for this device + if manual_mappings_per_device and device_id in manual_mappings_per_device: + device_mappings.update(manual_mappings_per_device[device_id]) + + result = import_single_device( + device_id, + server_key=server_key, + sync_options=sync_options, + manual_mappings=device_mappings if device_mappings else None, + libre_device=libre_device, + ) + + if result["success"]: + success_list.append( + { + "device_id": device_id, + "device": result["device"], + "message": result["message"], + } + ) + + # Handle virtual chassis creation for stacks + vc_data = validation.get("virtual_chassis", {}) + if vc_data.get("is_stack", False): + vc_domain = f"librenms-{device_id}" + + # Only create VC if we haven't processed this stack yet + # Add to set BEFORE attempting creation to prevent race condition + if vc_domain not in processed_vc_domains: + processed_vc_domains.add(vc_domain) + try: + vc = create_virtual_chassis_with_members( + result["device"], + vc_data["members"], + libre_device, + ) + vc_created_count += 1 + log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" + if job and job.logger: + job.logger.info(log_msg) + else: + logger.info(log_msg) + except Exception as vc_error: + # Remove from set on failure so retry is possible + processed_vc_domains.discard(vc_domain) + warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" + if job and job.logger: + job.logger.warning(warn_msg) + else: + logger.warning(warn_msg) + # Don't fail the import, just log the warning + + elif result.get("device"): # Device exists + skipped_list.append({"device_id": device_id, "reason": result["error"]}) + else: # Failed to import + failed_list.append({"device_id": device_id, "error": result["error"]}) + if job and job.logger: + job.logger.error(f"Failed to import device {device_id}: {result['error']}") + + except Exception as e: + error_msg = f"Unexpected error importing device {device_id}: {str(e)}" + if job and job.logger: + job.logger.error(error_msg, exc_info=True) + else: + logger.exception(f"Unexpected error importing device {device_id}") + failed_list.append({"device_id": device_id, "error": str(e)}) + + return { + "total": total, + "success": success_list, + "failed": failed_list, + "skipped": skipped_list, + "virtual_chassis_created": vc_created_count, + } + + +def bulk_import_devices( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices to NetBox (synchronous). + + This is the public API for synchronous imports. For background job usage, + use bulk_import_devices_shared() with a job context. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + user: User performing the import (for permission checks) + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + """ + return bulk_import_devices_shared( + device_ids=device_ids, + server_key=server_key, + sync_options=sync_options, + manual_mappings_per_device=manual_mappings_per_device, + libre_devices_cache=libre_devices_cache, + job=None, # No job context for synchronous imports + user=user, + ) + + +def _refresh_existing_device(validation: dict, libre_device: dict = None) -> None: + """Refresh existing_device from DB to pick up changes made in NetBox since caching. + + When existing_device is None (wasn't found at cache time), re-check if the device + was imported since caching by looking up librenms_id or hostname. + """ + existing = validation.get("existing_device") + if existing and hasattr(existing, "pk"): + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + if validation.get("import_as_vm"): + refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() + else: + refreshed = Device.objects.filter(pk=existing.pk).first() + + if refreshed: + validation["existing_device"] = refreshed + if hasattr(refreshed, "role") and refreshed.role: + validation["device_role"] = {"found": True, "role": refreshed.role} + else: + # Device was deleted since caching β€” recompute readiness + validation["existing_device"] = None + validation["existing_match_type"] = None + validation["can_import"] = True + if validation.get("import_as_vm"): + validation["is_ready"] = bool( + validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") + ) + else: + validation["is_ready"] = bool( + validation.get("site", {}).get("found") + and validation.get("device_type", {}).get("found") + and validation.get("device_role", {}).get("found") + ) + except Exception as e: + existing_id = getattr(existing, "pk", "unknown") if existing else "none" + logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + return + + # existing_device was None at cache time β€” check if device was imported since + if not libre_device: + return + try: + from dcim.models import Device + + librenms_id = libre_device.get("device_id") + hostname = libre_device.get("hostname", "") + sys_name = libre_device.get("sysName", "") + + new_device = None + match_type = None + + # Check by librenms_id custom field first + if librenms_id: + try: + new_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + if new_device: + match_type = "librenms_id" + except (ValueError, TypeError): + pass + + # Fall back to hostname match + if not new_device and hostname: + new_device = Device.objects.filter(name__iexact=hostname).first() + if not new_device and sys_name: + new_device = Device.objects.filter(name__iexact=sys_name).first() + if new_device: + match_type = "hostname" + + if new_device: + validation["existing_device"] = new_device + validation["existing_match_type"] = match_type + validation["can_import"] = False + validation["is_ready"] = False + if hasattr(new_device, "role") and new_device.role: + validation["device_role"] = {"found": True, "role": new_device.role} + except Exception as e: + logger.error(f"Failed to check for newly imported device: {e}") + + +def process_device_filters( + api: LibreNMSAPI, + filters: dict, + vc_detection_enabled: bool, + clear_cache: bool, + show_disabled: bool, + exclude_existing: bool = False, + job=None, + request=None, + return_cache_status: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Process LibreNMS device filters and return validated devices. + + Shared function used by both synchronous view and background job processing. + Fetches devices, optionally pre-warms VC cache, validates each device, and + caches results for HTMX row updates. + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + vc_detection_enabled: Whether to detect virtual chassis + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + exclude_existing: Whether to exclude devices that already exist in NetBox + job: Optional JobRunner instance for logging job events + request: Optional Django request for client disconnect detection (synchronous only) + return_cache_status: When True, returns (devices, from_cache) tuple + + Returns: + List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from existing + cache; from_cache=False means data was just fetched from LibreNMS. + """ + # Fetch devices from LibreNMS + if job: + job.logger.info(f"Fetching devices with filters: {filters}") + else: + logger.info(f"Fetching devices with filters: {filters}") + + # Always get cache status internally, even if not returning it + # We need it to determine if metadata should be updated + libre_devices, from_cache = get_librenms_devices_for_import( + api, + filters=filters, + force_refresh=clear_cache, + return_cache_status=True, + ) + + # Filter out disabled devices if requested + if not show_disabled: + libre_devices = [d for d in libre_devices if d.get("status") == 1] + + if job: + job.logger.info(f"Found {len(libre_devices)} devices to process") + else: + logger.info(f"Found {len(libre_devices)} devices") + + # Pre-warm VC cache if needed + if vc_detection_enabled and libre_devices: + device_ids = [d["device_id"] for d in libre_devices] + if job: + job.logger.info( + f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." + ) + else: + logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") + + try: + prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) + if job: + job.logger.info("Virtual chassis data pre-fetch completed") + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during VC prefetch: {e}") + return [] + raise + + # Validate each device + validated_devices = [] + total = len(libre_devices) + api_for_validation = api if vc_detection_enabled else None + + if job: + job.logger.info(f"Starting validation of {total} devices") + # Initial check if job was already terminated before we even started + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + if rq_job.is_failed or rq_job.is_stopped: + job.logger.warning("Job was already stopped before validation started") + return [] + except Exception: + # Fall back to DB check if RQ check fails + job.job.refresh_from_db() + if job.job.status == JobStatusChoices.STATUS_FAILED: + job.logger.warning("Job was stopped before validation started") + return [] + else: + logger.info(f"Validating {total} devices") + + for idx, device in enumerate(libre_devices, 1): + # Check for job termination or client disconnect periodically + if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) + if job: + # Check if job was terminated via stop API + # CRITICAL: Check the RQ job status in Redis, not just the DB model + # NetBox's stop endpoint marks the RQ job as failed in Redis + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + # Check if RQ job is in a stopped state + if rq_job.is_failed or rq_job.is_stopped: + job.logger.info( + f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." + ) + return [] + except Exception: + # If we can't check RQ status, fall back to DB status check + job.job.refresh_from_db() + if job.job.status == JobStatusChoices.STATUS_FAILED: + job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") + return [] + elif request: + # Check for client disconnect + try: + if hasattr(request, "META") and request.META.get("wsgi.input"): + pass + except (BrokenPipeError, ConnectionError, IOError): + logger.info(f"Client disconnected during validation at device {idx}") + return [] + + # Drop any cached validation/meta keys before recomputing + device.pop("_validation", None) + + # Generate shared cache key for this validated device + device_id = device["device_id"] + cache_key = get_validated_device_cache_key( + server_key=api.server_key, + filters=filters, + device_id=device_id, + vc_enabled=vc_detection_enabled, + ) + + # Check if we already have cached validation for this device + # (only if not forcing refresh) + if not clear_cache: + cached_device = cache.get(cache_key) + if cached_device: + # Use cached validation + device["_validation"] = cached_device["_validation"] + + # Refresh existing_device from DB to avoid stale data + # (user may have imported the device or changed it in NetBox) + _refresh_existing_device(device["_validation"], libre_device=device) + + # Apply exclude_existing filter if enabled + if exclude_existing: + validation = device["_validation"] + if validation["existing_device"]: + continue + + validated_devices.append(device) + continue + + # Not in cache or forcing refresh - validate now + try: + validation = validate_device_for_import( + device, + api=api_for_validation, + include_vc_detection=vc_detection_enabled, + force_vc_refresh=clear_cache, + ) + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during device validation: {e}") + return [] + raise + + # Set VC detection metadata + if not vc_detection_enabled: + validation["virtual_chassis"] = empty_virtual_chassis_data() + + # Apply exclude_existing filter if enabled + if exclude_existing and validation["existing_device"]: + continue + + device["_validation"] = validation + validated_devices.append(device) + + # Cache with TWO keys for different purposes: + # 1. Complex key (with filter context) - for full validated device with all metadata + cache.set(cache_key, device, timeout=api.cache_timeout) + + # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates + # This avoids redundant API calls when user interacts with dropdowns + simple_cache_key = get_import_device_cache_key(device_id, api.server_key) + # Cache just the raw device data (not the full validation result) + # This is what get_validated_device_with_selections() expects + device_data_only = {k: v for k, v in device.items() if k != "_validation"} + cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) + + # Store cache metadata (timestamp) for all filter operations + # This enables countdown display regardless of background job vs synchronous execution + # Always store metadata when we have validated devices, even if from_cache + # This ensures metadata is available for countdown display + if validated_devices: + from datetime import datetime, timezone + + cache_metadata_key = get_cache_metadata_key( + server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled + ) + + # Check if metadata already exists to preserve original timestamp + # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it + existing_metadata = cache.get(cache_metadata_key) + should_update = clear_cache or not from_cache + + if existing_metadata and not should_update: + # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) + pass + else: + # No metadata exists, OR cache was cleared, OR fresh data - create/update it now + cache_metadata = { + "cached_at": datetime.now(timezone.utc).isoformat(), + "cache_timeout": api.cache_timeout, + "filters": filters, + "vc_enabled": vc_detection_enabled, + "device_count": len(validated_devices), + } + cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) + + # Maintain cache index for this server to enable listing active searches + cache_index_key = f"librenms_cache_index_{api.server_key}" + cache_index = cache.get(cache_index_key, []) + # Add this cache key if not already in index + if cache_metadata_key not in cache_index: + cache_index.append(cache_metadata_key) + # Store index with same timeout as the metadata + cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) + + if job: + if exclude_existing: + filtered_count = total - len(validated_devices) + job.logger.info( + f"Validation complete: {len(validated_devices)} devices passed filter, " + f"{filtered_count} filtered out (existing devices excluded)" + ) + else: + job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") + else: + logger.info(f"Processed {len(validated_devices)} validated devices") + + if return_cache_status: + return validated_devices, from_cache + return validated_devices diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py new file mode 100644 index 0000000000..716e9dc6a1 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -0,0 +1,158 @@ +"""Cache key generation and management for device import operations.""" + +import logging + +from django.core.cache import cache + +logger = logging.getLogger(__name__) + + +def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: + """ + Generate a consistent cache metadata key from filter parameters. + + Args: + server_key: LibreNMS server identifier + filters: Filter dictionary + vc_enabled: Whether VC detection is enabled + + Returns: + str: Consistent cache key for metadata + """ + # Sort filter items to ensure consistent key generation + filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) + return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" + + +def get_active_cached_searches(server_key: str) -> list[dict]: + """ + Retrieve all active cached searches for a server and enrich with display-friendly values. + + Enriches raw filter IDs with human-readable names by looking up location names + from cached choices and converting type codes to display names. + + Args: + server_key: LibreNMS server identifier + + Returns: + List of dicts containing cache metadata with enriched display_filters + """ + from datetime import datetime, timezone + + cache_index_key = f"librenms_cache_index_{server_key}" + cache_index = cache.get(cache_index_key, []) + + active_searches = [] + valid_cache_keys = [] + + # Get location and type choices for enriching display + location_choices = {} + type_choices = { + "": "All Types", + "network": "Network", + "server": "Server", + "storage": "Storage", + "wireless": "Wireless", + "firewall": "Firewall", + "power": "Power", + "appliance": "Appliance", + "printer": "Printer", + "loadbalancer": "Load Balancer", + "other": "Other", + } + + # Get cached location choices for enrichment + location_cache_key = "librenms_locations_choices" + cached_locations = cache.get(location_cache_key) + if cached_locations: + location_choices = dict(cached_locations) + + for cache_key in cache_index: + metadata = cache.get(cache_key) + if metadata: + # Cache still exists, calculate time remaining + cached_at = datetime.fromisoformat(metadata.get("cached_at")) + cache_timeout = metadata.get("cache_timeout", 300) + now = datetime.now(timezone.utc) + age_seconds = (now - cached_at).total_seconds() + remaining_seconds = max(0, cache_timeout - age_seconds) + + if remaining_seconds > 0: + # Add remaining time and cache key + metadata["remaining_seconds"] = int(remaining_seconds) + metadata["cache_key"] = cache_key + + # Enrich filters with human-readable display values + if "filters" in metadata: + display_filters = metadata["filters"].copy() + # Convert location ID to location name + if "location" in display_filters and display_filters["location"] in location_choices: + display_filters["location"] = location_choices[display_filters["location"]] + # Convert type code to display name + if "type" in display_filters and display_filters["type"] in type_choices: + display_filters["type"] = type_choices[display_filters["type"]] + metadata["display_filters"] = display_filters + else: + # Fallback if filters key missing + metadata["display_filters"] = {} + + active_searches.append(metadata) + valid_cache_keys.append(cache_key) + + # Clean up index if any keys have expired + if len(valid_cache_keys) < len(cache_index): + cache.set(cache_index_key, valid_cache_keys, timeout=3600) + + # Sort by most recent first + active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) + + return active_searches + + +def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: + """ + Generate a consistent cache key for validated device data. + + This ensures both synchronous and background job processing use the same + cache keys, avoiding duplicate validation work and cache entries. + + Args: + server_key: LibreNMS server key + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + device_id: LibreNMS device ID + vc_enabled: Whether virtual chassis detection was enabled + + Returns: + str: Cache key for the validated device + + Example: + >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) + >>> key + 'validated_device_default_-1234567890_123_vc' + """ + # Sort filters for consistent hashing + filter_hash = hash(str(sorted(filters.items()))) + vc_part = "vc" if vc_enabled else "novc" + return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" + + +def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: + """ + Generate cache key for raw LibreNMS device data. + + This key is used to cache raw device data (without validation metadata) + to avoid redundant API calls when users interact with dropdowns during + the import workflow. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server identifier for multi-server setups + + Returns: + str: Cache key for the device data + + Example: + >>> get_import_device_cache_key(123, "production") + 'import_device_data_production_123' + """ + return f"import_device_data_{server_key}_{device_id}" diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py new file mode 100644 index 0000000000..31ed2b2be7 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -0,0 +1,906 @@ +"""Device validation, import, and fetch operations.""" + +import logging + +from dcim.models import Device, DeviceRole, DeviceType, Rack, Site +from django.core.cache import cache +from virtualization.models import Cluster +from django.db import transaction +from django.utils import timezone + +from ..librenms_api import LibreNMSAPI +from ..utils import ( + find_matching_platform, + find_matching_site, + match_librenms_hardware_to_device_type, +) +from .cache import get_import_device_cache_key +from .virtual_chassis import empty_virtual_chassis_data, get_virtual_chassis_data + +logger = logging.getLogger(__name__) + + +def _determine_device_name( + libre_device: dict, + use_sysname: bool = True, + strip_domain: bool = False, + device_id: int | str = None, +) -> str: + """ + Determine the device/VM name from LibreNMS data. + + Centralized logic for building device names with consistent handling of: + - sysName vs hostname preference + - Domain stripping (avoiding IP addresses) + - Fallback to device_id when name is missing + + Args: + libre_device: Device data from LibreNMS + use_sysname: If True, prefer sysName; if False, use hostname + strip_domain: If True, strip domain suffix (e.g., '.example.com') + device_id: LibreNMS device ID for fallback name generation + + Returns: + str: The determined device name + + Example: + >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, + ... use_sysname=True, strip_domain=True) + 'router' + """ + # Determine base name based on use_sysname preference + if use_sysname: + name = libre_device.get("sysName") or libre_device.get("hostname") + else: + name = libre_device.get("hostname") or libre_device.get("sysName") + + # Fallback to device_id if no name found + if not name: + if device_id is not None: + name = f"device-{device_id}" + else: + name = libre_device.get("device_id", "unknown") + name = f"device-{name}" + + # Strip domain if requested (but not for IP addresses) + if strip_domain and name and "." in name: + try: + from ipaddress import ip_address + + ip_address(name) + # It's a valid IP address, don't strip + except ValueError: + # Not an IP, safe to strip domain + name = name.split(".")[0] + + return name + + +def _try_chassis_device_type_match(api, device_id): + """ + Attempt device type matching using chassis inventory fields. + + When the LibreNMS hardware string doesn't match any NetBox device type, + the chassis entity often contains a more standardized identifier + (e.g., entPhysicalName 'CHAS-BP-MX480-S' or entPhysicalModelName '710-017414') + that matches a DeviceType part_number or model. + + Tries entPhysicalName first (typically the chassis part number), + then entPhysicalModelName as fallback. + + Returns: + dict with matched/device_type/match_type keys, or None on failure. + """ + skip_values = {"", "-", "Unspecified", "BUILTIN", "None"} + + try: + success, inventory = api.get_inventory_filtered(device_id, ent_physical_class="chassis") + if not success or not inventory: + return None + + for item in inventory: + # Try entPhysicalName first (often the chassis part number like CHAS-BP-MX480-S) + for field in ("entPhysicalName", "entPhysicalModelName"): + value = item.get(field) or "" + if value and value not in skip_values: + chassis_match = match_librenms_hardware_to_device_type(value) + if chassis_match["matched"]: + chassis_match["match_type"] = "chassis" + chassis_match["chassis_model"] = value + return chassis_match + except Exception: + logger.debug(f"Chassis inventory fallback failed for device {device_id}", exc_info=True) + + return None + + +def validate_device_for_import( + libre_device: dict, + import_as_vm: bool = False, + api: "LibreNMSAPI" = None, + *, + include_vc_detection: bool = True, + force_vc_refresh: bool = False, +) -> dict: + """ + Validate if a LibreNMS device can be imported to NetBox. + + Performs comprehensive validation: + - Checks if device already exists in NetBox + - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) + OR (Cluster for VMs) + - Provides smart matching for missing objects + - Detects virtual chassis/stack configuration (if API provided) + - Returns detailed validation status + + Args: + libre_device: Device data from LibreNMS + import_as_vm: If True, validate for VM import instead of device import + api: Optional LibreNMSAPI instance for virtual chassis detection + include_vc_detection: Skip VC detection when False to speed up bulk operations + force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS + + Returns: + dict: Validation result with structure: + { + 'is_ready': bool, # Can import without user intervention + 'can_import': bool, # Can import (possibly after configuration) + 'import_as_vm': bool, # Whether importing as VM + 'existing_device': Device or VirtualMachine or None, + 'issues': List[str], # Blocking issues + 'warnings': List[str], # Non-blocking warnings + 'site': { # Only for devices + 'found': bool, + 'site': Site or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[Site] # Alternative suggestions + }, + 'device_type': { # Only for devices + 'found': bool, + 'device_type': DeviceType or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[dict] # Device types for user selection + }, + 'device_role': { # Only for devices + 'found': bool, # Always False - requires manual selection + 'role': DeviceRole or None, + 'available_roles': List[DeviceRole] # All roles for user selection + }, + 'cluster': { # Only for VMs + 'found': bool, # Always False - requires manual selection + 'cluster': Cluster or None, + 'available_clusters': List[Cluster] # All clusters for user selection + }, + 'platform': { + 'found': bool, + 'platform': Platform or None, + 'match_type': str # 'exact' or None + } + } + + Example: + >>> validation = validate_device_for_import(libre_device) + >>> if validation['is_ready']: + ... import_single_device(libre_device['device_id']) + """ + result = { + "is_ready": False, + "can_import": False, + "import_as_vm": import_as_vm, + "existing_device": None, + "existing_match_type": None, # Track how existing device was matched + "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" + "serial_confirmed": False, # True when librenms_id match and serial matches + "serial_duplicate": False, # True when incoming serial is already on a different device + "name_matches": False, # True when existing device name matches LibreNMS sysName + "name_sync_available": False, # True when existing device name differs from sysName + "suggested_name": None, # sysName to suggest when name_sync_available is True + "device_type_mismatch": False, # True when existing device's type differs from LibreNMS + "issues": [], + "warnings": [], + "virtual_chassis": empty_virtual_chassis_data(), + "site": { + "found": False, + "site": None, + "match_type": None, + "suggestions": [], + }, + "device_type": { + "found": False, + "device_type": None, + "match_type": None, + "suggestions": [], + }, + "device_role": { + "found": False, + "role": None, + "available_roles": [], + }, + "cluster": { + "found": False, + "cluster": None, + "available_clusters": [], + }, + "platform": {"found": False, "platform": None, "match_type": None}, + "rack": { + "found": False, + "rack": None, + "available_racks": [], + }, + } + + try: + # 1. Check if device/VM already exists in NetBox + # Always check both Devices AND VMs to properly detect existing objects + librenms_id = libre_device.get("device_id") + hostname = libre_device.get("hostname", "") + logger.debug( + f"Checking for existing device/VM: " + f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " + f"hostname={hostname}" + ) + + from virtualization.models import VirtualMachine + + # Check for existing VM first (by librenms_id custom field) + # Always query with int to match custom field type + try: + existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_vm = None + + if existing_vm: + logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_vm + result["existing_match_type"] = "librenms_id" + result["import_as_vm"] = True # Force VM mode since VM exists + result["can_import"] = False + + # Check if name matches sysName + # Note: name_sync_available/suggested_name are intentionally not set for VMs + # because UpdateDeviceNameView only supports Device objects; VM name-sync + # would require a separate implementation. + sys_name = libre_device.get("sysName") or "" + if sys_name and existing_vm.name == sys_name: + result["name_matches"] = True + + # Check for existing Device (by librenms_id custom field) + # Always query with int to match custom field type + if not result["existing_device"]: + try: + existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_device = None + + if existing_device: + logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_device + result["existing_match_type"] = "librenms_id" + result["can_import"] = False + + # Check if name matches sysName + sys_name = libre_device.get("sysName") or "" + if sys_name and existing_device.name == sys_name: + result["name_matches"] = True + elif sys_name and existing_device.name != sys_name: + result["name_sync_available"] = True + result["suggested_name"] = sys_name + + # Check for serial drift on the linked device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + if existing_device.serial and existing_device.serial == incoming_serial: + result["serial_confirmed"] = True + elif existing_device.serial and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before updating." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Serial number differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + + # Only check hostname/serial/IP if not already matched by librenms_id + if not result["existing_device"]: + # Check by hostname/name - Check both VMs and Devices for conflicts + existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() + existing_device = Device.objects.filter(name__iexact=hostname).first() + + # If BOTH exist with same hostname, it's ambiguous - don't match either + if existing_vm and existing_device: + logger.warning( + f"Hostname conflict: Both VM '{existing_vm.name}' and Device " + f"'{existing_device.name}' exist with hostname '{hostname}'" + ) + result["warnings"].append( + f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " + f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." + ) + # Don't set existing_device, don't block import - let user proceed as new + # This allows them to import and then resolve the conflict manually + elif existing_vm: + logger.info(f"Found existing VM by hostname: {existing_vm.name}") + result["existing_device"] = existing_vm + result["existing_match_type"] = "hostname" + result["import_as_vm"] = True # Force VM mode since VM exists + result["warnings"].append( + f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + elif existing_device: + logger.info(f"Found existing device by hostname: {existing_device.name}") + result["existing_device"] = existing_device + result["existing_match_type"] = "hostname" + + # Check for serial conflict on hostname-matched device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-" and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before importing." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Hostname matches but serial differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + else: + result["warnings"].append( + f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" + ) + + result["can_import"] = False + + # Check by serial number (strong physical match - hardware identity) + if not result["existing_device"]: + serial = libre_device.get("serial") or "" + if serial and serial != "-" and not import_as_vm: + existing_by_serial = Device.objects.filter(serial=serial).first() + if existing_by_serial: + logger.info(f"Found existing device by serial: {existing_by_serial.name} (serial={serial})") + result["existing_device"] = existing_by_serial + result["existing_match_type"] = "serial" + result["can_import"] = False + + if existing_by_serial.name and existing_by_serial.name.lower() == hostname.lower(): + result["warnings"].append( + f"Device with same serial and hostname exists as '{existing_by_serial.name}' " + f"(not linked to LibreNMS)" + ) + result["serial_action"] = "link" + else: + result["warnings"].append( + f"Device with same serial ({serial}) exists as '{existing_by_serial.name}' " + f"but hostname differs (LibreNMS: '{hostname}'). Device may have been reinstalled." + ) + result["serial_action"] = "hostname_differs" + + # Check by primary IP (weaker match, IP could be reassigned) - only for devices + if not result["existing_device"]: + primary_ip = libre_device.get("ip") + if primary_ip and not import_as_vm: + from ipam.models import IPAddress + + existing_ip = IPAddress.objects.filter(address__net_host=primary_ip).first() + if existing_ip and existing_ip.assigned_object: + device = ( + existing_ip.assigned_object.device + if hasattr(existing_ip.assigned_object, "device") + else None + ) + if device: + result["existing_device"] = device + result["existing_match_type"] = "primary_ip" + result["warnings"].append( + f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + + # Validate based on import type (Device or VM) + if import_as_vm: + # 2. For VMs: Validate Cluster (required) - Must be manually selected + result["cluster"]["found"] = False + result["issues"].append("Cluster must be manually selected before importing as VM") + # Provide list of available clusters for user selection (cached) + cache_key = "librenms_import_all_clusters" + all_clusters = cache.get(cache_key) + if all_clusters is None: + all_clusters = list(Cluster.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_clusters, cache_timeout) + result["cluster"]["available_clusters"] = all_clusters + + # Skip device-specific validations for VMs + result["site"]["found"] = True # Not required for VMs + result["device_type"]["found"] = True # Not required for VMs + result["device_role"]["found"] = True # Not required for VMs + + else: + # 2. For Devices: Validate Site (required) + location = libre_device.get("location", "") + site_match = find_matching_site(location) + result["site"] = site_match + + if not site_match["found"]: + result["issues"].append(f"No matching site found for location: '{location}'") + # Get alternative suggestions + if location: + all_sites = Site.objects.all()[:10] # Limit for performance + result["site"]["suggestions"] = list(all_sites) + + # 3. Validate DeviceType (required) + hardware = libre_device.get("hardware", "") + dt_match = match_librenms_hardware_to_device_type(hardware) + + # Chassis inventory fallback: when hardware doesn't match, + # try the chassis entPhysicalModelName as an additional lookup source + if not dt_match["matched"] and api: + device_id = libre_device.get("device_id") + if device_id: + chassis_match = _try_chassis_device_type_match(api, device_id) + if chassis_match and chassis_match["matched"]: + dt_match = chassis_match + + result["device_type"] = dt_match + + if not dt_match["matched"]: + result["issues"].append(f"No matching device type found for hardware: '{hardware}'") + # Get some device types for user to choose from + all_device_types = DeviceType.objects.all()[:10] + result["device_type"]["suggestions"] = [ + { + "device_type": dt, + "similarity": 0.0, # No fuzzy matching, just showing options + "match_field": None, + } + for dt in all_device_types + ] + else: + # Rename 'matched' to 'found' for consistency + result["device_type"]["found"] = dt_match["matched"] + result["device_type"]["device_type"] = dt_match["device_type"] + result["device_type"]["match_type"] = dt_match["match_type"] + + # 4. DeviceRole (required) - Must be manually selected by user + logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") + result["device_role"]["found"] = False + result["issues"].append("Device role must be manually selected before import") + logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") + # Provide list of available roles for user selection (cached) + cache_key = "librenms_import_all_roles" + all_roles = cache.get(cache_key) + if all_roles is None: + all_roles = list(DeviceRole.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_roles, cache_timeout) + result["device_role"]["available_roles"] = all_roles + + # 4b. Rack (optional) - Provide available racks for the matched site + if site_match["found"] and site_match["site"]: + site = site_match["site"] + # Use cache to optimize rack lookups per site + cache_key = f"librenms_import_racks_site_{site.pk}" + available_racks = cache.get(cache_key) + + if available_racks is None: + from dcim.models import Rack + from django.db.models import Q + + # Query racks for this site - include both: + # 1. Racks assigned to locations within the site + # 2. Racks directly assigned to the site (without location) + available_racks = list( + Rack.objects.filter(Q(location__site=site) | Q(site=site)) + .select_related("location", "site") + .order_by("location__name", "name") + ) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, available_racks, cache_timeout) + + result["rack"]["available_racks"] = available_racks + # Rack is optional, don't add to issues + result["rack"]["found"] = True # Mark as "found" even if None (optional field) + + # Skip VM-specific validations for devices + result["cluster"]["found"] = True # Not required for devices + + # 5. Match Platform (optional - same for both devices and VMs) + os = libre_device.get("os", "") + platform_match = find_matching_platform(os) + result["platform"] = platform_match + + if not platform_match["found"] and os: + result["warnings"].append(f"No matching platform found for OS: '{os}'") + + # 6. Additional validations + if not hostname: + result["issues"].append("Device has no hostname") + + # 7. Virtual chassis detection (only for devices, not VMs) + if include_vc_detection and not import_as_vm and api is not None: + device_id = libre_device.get("device_id") + if device_id: + try: + logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") + vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) + logger.debug( + f"VC detection result: is_stack={vc_detection.get('is_stack')}, " + f"member_count={vc_detection.get('member_count')}, " + f"members={len(vc_detection.get('members', []))}" + ) + if vc_detection: + result["virtual_chassis"] = vc_detection + if vc_detection["is_stack"]: + logger.debug( + f"Virtual chassis CONFIRMED for device {hostname}: " + f"{vc_detection['member_count']} members" + ) + except Exception as e: + logger.exception(f"Exception during VC detection for device {hostname}: {e}") + result["virtual_chassis"]["detection_error"] = str(e) + else: + logger.debug(f"No device_id found for {hostname}") + + # 8. Determine if device/VM is ready to import + if result["existing_device"]: + # Already matched - can_import was already set to False + result["is_ready"] = False + # Populate role from existing device so the modal shows it + existing = result["existing_device"] + if hasattr(existing, "role") and existing.role: + result["device_role"]["found"] = True + result["device_role"]["role"] = existing.role + + # Check for device type mismatch between existing device and LibreNMS + if hasattr(existing, "device_type") and existing.device_type: + librenms_dt = result["device_type"].get("device_type") + if librenms_dt and existing.device_type.pk != librenms_dt.pk: + result["device_type_mismatch"] = True + result["warnings"].append( + f"Device type mismatch: NetBox has '{existing.device_type}' " + f"but LibreNMS reports '{librenms_dt}'. " + f"This may indicate the wrong device was matched." + ) + else: + result["can_import"] = len(result["issues"]) == 0 + + if import_as_vm: + # For VMs: only cluster is required + result["is_ready"] = result["can_import"] and result["cluster"]["found"] + else: + # For Devices: site, device_type, and device_role are required + result["is_ready"] = ( + result["can_import"] + and result["site"]["found"] + and result["device_type"]["found"] + and result["device_role"]["found"] + ) + + logger.debug( + f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " + f"issues={len(result['issues'])}, can_import={result['can_import']}, " + f"issues_list={result['issues']}" + ) + + return result + + except Exception as e: + logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") + result["issues"].append(f"Validation error: {str(e)}") + return result + + +def import_single_device( + device_id: int, + server_key: str = None, + validation: dict = None, + manual_mappings: dict = None, + sync_options: dict = None, + libre_device: dict = None, +) -> dict: + """ + Import a single LibreNMS device to NetBox. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server configuration key + validation: Pre-computed validation dict (optional) + manual_mappings: Manual object mappings (optional): + - site_id: NetBox Site ID + - device_type_id: NetBox DeviceType ID + - device_role_id: NetBox DeviceRole ID + - platform_id: NetBox Platform ID (optional) + - rack_id: NetBox Rack ID (optional) + sync_options: Sync options (optional): + - sync_interfaces: bool (default True) + - sync_cables: bool (default True) + - sync_ips: bool (default True) + - sync_fields: bool (default True) + libre_device: Pre-fetched LibreNMS device data (optional). + If provided, skips API call to fetch device info. + + Returns: + dict: Import result with structure: + { + 'success': bool, + 'device': Device object or None, + 'message': str, + 'error': str or None, + 'synced': { + 'interfaces': int, + 'cables': int, + 'ip_addresses': int + } + } + """ + try: + api = LibreNMSAPI(server_key=server_key) + + # Use pre-fetched device data if provided, otherwise fetch from API + if libre_device is None: + success, libre_device = api.get_device_info(device_id) + if not success or not libre_device: + return { + "success": False, + "device": None, + "message": "", + "error": f"Failed to retrieve device {device_id} from LibreNMS", + "synced": {}, + } + + # Validate device if validation not provided + if validation is None: + validation = validate_device_for_import(libre_device) + + # Check if device already exists + if validation.get("existing_device"): + return { + "success": False, + "device": validation["existing_device"], + "message": "", + "error": f"Device already exists: {validation['existing_device'].name}", + "synced": {}, + } + + # Use validation-derived matches, allow manual mappings to override specific fields + site = validation["site"].get("site") + device_type = validation["device_type"].get("device_type") + device_role = validation["device_role"].get("role") + platform = validation["platform"].get("platform") + rack = validation.get("rack", {}).get("rack") + + if manual_mappings: + site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site + device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type + device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role + + platform_id = manual_mappings.get("platform_id") + if platform_id: + from dcim.models import Platform + + platform = Platform.objects.filter(id=platform_id).first() or platform + + rack_id = manual_mappings.get("rack_id") + if rack_id: + rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack + + rack = rack or validation.get("rack", {}).get("rack") + + # Validate required fields + if not site: + return { + "success": False, + "device": None, + "message": "", + "error": "Site is required but not provided", + "synced": {}, + } + if not device_type: + return { + "success": False, + "device": None, + "message": "", + "error": "Device type is required but not provided", + "synced": {}, + } + if not device_role: + return { + "success": False, + "device": None, + "message": "", + "error": "Device role is required but not provided", + "synced": {}, + } + + # Create device in NetBox + with transaction.atomic(): + # Determine device name based on sync options + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + device_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=device_id, + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + device_data = { + "name": device_name, + "site": site, + "device_type": device_type, + "role": device_role, + "status": "active" if libre_device.get("status") == 1 else "offline", + "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + "custom_field_data": {"librenms_id": int(device_id)}, + } + + # Add optional fields + if platform: + device_data["platform"] = platform + + if rack: + device_data["rack"] = rack + + serial = libre_device.get("serial", "") + if serial and serial != "-": + device_data["serial"] = serial + + location_name = libre_device.get("location", "") + if location_name and location_name != "-": + from dcim.models import Location + + # Try to find matching location within the site + location = Location.objects.filter(site=site, name__iexact=location_name).first() + if location: + device_data["location"] = location + + # Create the device + device = Device(**device_data) + device.full_clean() + device.save() + + # Sync additional data based on options + sync_options = sync_options or {} + synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} + + try: + # Sync interfaces + if sync_options.get("sync_interfaces", True): + # This is simplified - would need proper request context + # For now, just log that it should be done + logger.info(f"Interface sync should be performed for device {device.name}") + + # Sync cables + if sync_options.get("sync_cables", True): + logger.info(f"Cable sync should be performed for device {device.name}") + + # Sync IP addresses + if sync_options.get("sync_ips", True): + logger.info(f"IP address sync should be performed for device {device.name}") + + except Exception as e: + logger.warning(f"Error during post-import sync: {str(e)}") + # Don't fail the import if sync fails + + return { + "success": True, + "device": device, + "message": f"Successfully imported device: {device.name}", + "error": None, + "synced": synced, + } + + except Exception as e: + logger.exception(f"Error importing device {device_id}") + return { + "success": False, + "device": None, + "message": "", + "error": str(e), + "synced": {}, + } + + +def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: + """ + Retrieve a single device from LibreNMS by ID. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + Device dictionary or None if not found + """ + try: + # Use the dedicated API endpoint to get device by ID + success, device = api.get_device_info(device_id) + if success and device: + return device + + logger.warning(f"Device {device_id} not found in LibreNMS") + return None + except Exception as e: + logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") + return None + + +def fetch_device_with_cache( + device_id: int, + api: LibreNMSAPI, + server_key: str = None, + libre_devices_cache: dict = None, +) -> dict | None: + """ + Fetch LibreNMS device from cache or API with automatic caching. + + Checks three sources in order: + 1. Pre-fetched cache dict (if provided) + 2. Django cache (Redis/memory) + 3. LibreNMS API (caches result for future use) + + This function consolidates the device fetching pattern used throughout + the import workflow, eliminating code duplication. + + Args: + device_id: LibreNMS device ID to fetch + api: LibreNMSAPI instance for fallback API calls + server_key: Optional server key for multi-server setups (defaults to api.server_key) + libre_devices_cache: Optional pre-fetched device cache dict + + Returns: + Device dict from LibreNMS, or None if not found + + Example: + >>> # Simple usage + >>> libre_device = fetch_device_with_cache(123, api) + >>> if libre_device: + ... print(libre_device['hostname']) + >>> + >>> # With pre-fetched cache dict + >>> cache_dict = {123: {...}, 456: {...}} + >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) + """ + # Check pre-fetched cache dict first (fastest) + if libre_devices_cache and device_id in libre_devices_cache: + return libre_devices_cache[device_id] + + # Check Django cache + cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) + libre_device = cache.get(cache_key) + + if not libre_device: + # Fallback to API fetch + libre_device = get_librenms_device_by_id(api, device_id) + if libre_device: + # Cache for future use + cache.set(cache_key, libre_device, timeout=api.cache_timeout) + + return libre_device diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py new file mode 100644 index 0000000000..27f4449266 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -0,0 +1,253 @@ +"""Device filtering and retrieval from LibreNMS.""" + +import logging +from typing import List + +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def get_device_count_for_filters( + api: LibreNMSAPI, + filters: dict, + clear_cache: bool = False, + show_disabled: bool = True, +) -> int: + """ + Get count of LibreNMS devices matching filters. + + This is a lightweight function to determine device count for background job + decision making. Uses the same caching as get_librenms_devices_for_import(). + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname keys + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + + Returns: + int: Count of devices matching filters + """ + devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) + + # Filter out disabled devices if requested + if not show_disabled: + devices = [d for d in devices if d.get("status") == 1] + + return len(devices) + + +def get_librenms_devices_for_import( + api: LibreNMSAPI = None, + filters: dict = None, + server_key: str = None, + *, + force_refresh: bool = False, + return_cache_status: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Retrieve LibreNMS devices based on filters. + + Args: + api: LibreNMSAPI instance (if not provided, creates one with server_key) + filters: Dict containing filter parameters: + - location: LibreNMS location/site filter + - type: Device type filter + - os: Operating system filter + - hostname: Hostname filter (partial match) + - sysname: System name filter (partial match) + - status: Device status filter (1=up, 0=down) + - disabled: Include disabled devices (0=active only, 1=all) + server_key: Key for specific server configuration (used if api not provided) + force_refresh: When True, bypass the cache and fetch fresh data + return_cache_status: When True, returns (devices, from_cache) tuple + + Returns: + List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from + existing cache; from_cache=False means data was just fetched from LibreNMS. + """ + try: + # Use provided API instance or create a new one + if api is None: + api = LibreNMSAPI(server_key=server_key) + + # Build LibreNMS API filters using the type/query format + # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters + # NOTE: API only supports ONE type/query pair, so we'll use the most + # specific filter for the API and apply others client-side + api_filters = {} + client_filters = {} # Filters to apply after fetching from API + + if filters: + # Check for status filter first - it has special handling + if filters.get("status") is not None: + # Status filter uses special types that don't need query param + if filters["status"] == 1: + api_filters["type"] = "up" + elif filters["status"] == 0: + api_filters["type"] = "down" + + # Save ALL other filters for client-side filtering when status is used + if filters.get("location"): + client_filters["location"] = filters["location"] + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + else: + # Priority order for type/query filters: location > type > os > hostname > sysname + # Note: When sysname is combined with other filters, it's applied client-side for partial matching + # When sysname is alone, it uses API exact match (type=sysName) + # Note: hardware is always applied client-side for partial matching + # Use first available for API, save others for client-side filtering + if filters.get("location"): + api_filters["type"] = "location_id" + api_filters["query"] = filters["location"] + # Save remaining filters for client-side + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("type"): + api_filters["type"] = "type" + api_filters["query"] = filters["type"] + # Save remaining filters for client-side + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("os"): + api_filters["type"] = "os" + api_filters["query"] = filters["os"] + # Save remaining filters for client-side + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hostname"): + api_filters["type"] = "hostname" + api_filters["query"] = filters["hostname"] + # Save sysname and hardware for client-side + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("sysname"): + # sysname-only filter: Use API exact match (type=sysName&query=) + # This is safe - returns empty if no exact match found + api_filters["type"] = "sysName" + api_filters["query"] = filters["sysname"] + # Save hardware for client-side + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hardware"): + # hardware-only filter: apply client-side for partial matching + client_filters["hardware"] = filters["hardware"] + + # Note: disabled filter isn't directly supported by LibreNMS API + # We'll filter client-side if needed + + # Use caching to avoid repeated API calls + # Include both API and client filters in cache key + cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" + from_cache = False + + if force_refresh: + cache.delete(cache_key) + else: + cached_result = cache.get(cache_key) + if cached_result is not None: + # No need to deepcopy - cached data isn't mutated + devices = cached_result + from_cache = True + if return_cache_status: + return devices, from_cache + return devices + + success, devices = api.list_devices(api_filters if api_filters else None) + + if not success: + logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") + if return_cache_status: + return [], False + return [] + + # Apply client-side filters if any + if client_filters: + devices = _apply_client_filters(devices, client_filters) + + # Cache using configured timeout (default 300s) + # No need to deepcopy - Django's cache backend handles serialization + cache.set(cache_key, devices, timeout=api.cache_timeout) + + if return_cache_status: + return devices, from_cache + return devices + + except Exception: + logger.exception("Error retrieving LibreNMS devices for import") + if return_cache_status: + return [], False + return [] + + +def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: + """ + Apply client-side filters to device list. + + Args: + devices: List of device dicts from LibreNMS + filters: Dict of filters to apply (location, type, os, hostname, sysname) + + Returns: + Filtered list of devices + """ + filtered = devices + + if filters.get("location"): + location_id = str(filters["location"]) + filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] + + if filters.get("type"): + device_type = filters["type"].lower() + filtered = [d for d in filtered if d.get("type", "").lower() == device_type] + + if filters.get("os"): + os_filter = filters["os"].lower() + filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] + + if filters.get("hostname"): + hostname_filter = filters["hostname"].lower() + filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] + + if filters.get("sysname"): + sysname_filter = filters["sysname"].lower() + filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] + + if filters.get("hardware"): + hardware_filter = filters["hardware"].lower() + filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] + + return filtered diff --git a/netbox_librenms_plugin/import_utils/permissions.py b/netbox_librenms_plugin/import_utils/permissions.py new file mode 100644 index 0000000000..9e9b4521e0 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/permissions.py @@ -0,0 +1,48 @@ +"""Permission check helpers for device import operations.""" + +import logging + +from django.core.exceptions import PermissionDenied + +logger = logging.getLogger(__name__) + + +def check_user_permissions(user, permissions): + """ + Check if user has all required permissions. + + Args: + user: The user object to check permissions for + permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) + + Returns: + tuple: (has_all_permissions: bool, missing_permissions: list[str]) + + Raises: + PermissionDenied: If user is None (no user context available) + """ + if user is None: + raise PermissionDenied("No user context available for permission check") + + missing = [perm for perm in permissions if not user.has_perm(perm)] + return (len(missing) == 0, missing) + + +def require_permissions(user, permissions, action_description="perform this action"): + """ + Require user has all permissions, raising PermissionDenied if not. + + Args: + user: The user object to check permissions for + permissions: List of permission strings + action_description: Human-readable description for error message + + Raises: + PermissionDenied: If user lacks any required permission + """ + has_perms, missing = check_user_permissions(user, permissions) + if not has_perms: + missing_str = ", ".join(missing) + raise PermissionDenied( + f"You do not have permission to {action_description}. Missing permissions: {missing_str}" + ) diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py new file mode 100644 index 0000000000..79db61e3fe --- /dev/null +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -0,0 +1,440 @@ +"""Virtual chassis detection, creation, and management.""" + +import logging +from typing import List + +from dcim.models import Device, VirtualChassis +from django.core.cache import cache +from django.db import transaction + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def empty_virtual_chassis_data() -> dict: + """Public helper for callers that need a blank VC payload.""" + + return { + "is_stack": False, + "member_count": 0, + "members": [], + "detection_error": None, + } + + +def _clone_virtual_chassis_data(data: dict | None) -> dict: + """Return a defensive copy of cached VC data to avoid shared references.""" + + if not data: + return empty_virtual_chassis_data() + + members = [] + for idx, member in enumerate(data.get("members", [])): + member_copy = member.copy() + raw_position = member_copy.get("position", idx) + try: + member_copy["position"] = int(raw_position) + except (TypeError, ValueError): + member_copy["position"] = idx + members.append(member_copy) + + member_count = data.get("member_count") or len(members) + + return { + "is_stack": bool(data.get("is_stack")), + "member_count": member_count, + "members": members, + "detection_error": data.get("detection_error"), + } + + +_VC_CACHE_VERSION = "v1" + + +def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: + server_key = getattr(api, "server_key", "default") + return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" + + +def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: + """Fetch (and cache) virtual chassis data for a LibreNMS device.""" + + if not api or device_id is None: + return empty_virtual_chassis_data() + + cache_key = _vc_cache_key(api, device_id) + if not force_refresh: + cached = cache.get(cache_key) + if cached is not None: + return _clone_virtual_chassis_data(cached) + + detection_data = detect_virtual_chassis_from_inventory(api, device_id) + if detection_data and "detection_error" not in detection_data: + detection_data["detection_error"] = None + + cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() + + cache_timeout = getattr(api, "cache_timeout", 300) or 300 + cache.set(cache_key, cache_value, timeout=cache_timeout) + return _clone_virtual_chassis_data(cache_value) + + +def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: + """ + Pre-warm the virtual chassis cache for multiple devices. + + This eliminates the 0.5-1s delay when rendering the import table + by proactively fetching VC data before validation. + + Args: + api: LibreNMSAPI instance + device_ids: List of LibreNMS device IDs to prefetch VC data for + force_refresh: When True, bypass cache and fetch fresh data + + Example: + >>> # Before rendering import table + >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) + >>> # Now all validate_device_for_import() calls hit cache instantly + """ + if not api or not device_ids: + return + + logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") + + for idx, device_id in enumerate(device_ids): + # This populates the cache if empty, or skips if already cached + try: + get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) + except (BrokenPipeError, ConnectionError, IOError, OSError) as e: + logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") + # Stop processing if connection is broken + return + except Exception as e: + # Log but continue for other errors + logger.warning(f"Error prefetching VC data for device {device_id}: {e}") + + logger.debug(f"VC cache warming complete for {len(device_ids)} devices") + + +def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: + """ + Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. + Vendor-agnostic using standard hierarchical structure. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + dict with structure: + { + 'is_stack': bool, + 'member_count': int, + 'members': [ + { + 'serial': str, + 'position': int, + 'model': str, + 'name': str, + 'index': int, + 'description': str, + 'suggested_name': str # Generated using master device name + } + ] + } + Returns None if not a stack or detection fails. + + Detection Logic: + 1. Check root level (entPhysicalContainedIn=0) for parent container + 2. Find parent index (entPhysicalClass='stack' or 'chassis') + 3. Get children chassis at that parent's index + 4. If multiple chassis found -> Stack detected + """ + try: + # Get the master device info to use for naming + success, device_info = api.get_device_info(device_id) + master_name = None + if success and device_info: + master_name = device_info.get("sysName") or device_info.get("hostname") + + # Step 1: Get root level items + success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) + + if not success or not root_items: + logger.debug(f"No root inventory items found for device {device_id}") + return None + + # Step 2: Find parent container index + # Could be class="stack" or the main "chassis" + parent_index = None + for item in root_items: + item_class = item.get("entPhysicalClass") + if item_class in ["stack", "chassis"]: + parent_index = item.get("entPhysicalIndex") + logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") + break + + if not parent_index: + return None + + # Step 3: Get children chassis at next level + success, child_items = api.get_inventory_filtered( + device_id, + ent_physical_class="chassis", + ent_physical_contained_in=parent_index, + ) + + if not success: + return None + + # Filter for chassis only (in case API filter didn't work) + chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] + + # Step 4: Multiple chassis = stack + if len(chassis_items) <= 1: + return None + + # Step 5: Extract member info + members = [] + for idx, chassis in enumerate(chassis_items): + raw_position = chassis.get("entPhysicalParentRelPos", idx) + try: + position = int(raw_position) + except (TypeError, ValueError): + position = idx + member_data = { + "serial": chassis.get("entPhysicalSerialNum", ""), + "position": position, + "model": chassis.get("entPhysicalModelName", ""), + "name": chassis.get("entPhysicalName", ""), + "index": chassis.get("entPhysicalIndex"), + "description": chassis.get("entPhysicalDescr", ""), + } + + # Generate suggested name if we have master name + if master_name: + member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) + else: + member_data["suggested_name"] = f"Member-{position + 1}" + + members.append(member_data) + + # Sort by position + members.sort(key=lambda m: m["position"]) + + logger.info(f"Detected stack with {len(members)} members for device {device_id}") + + return {"is_stack": True, "member_count": len(members), "members": members} + + except Exception as e: + logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") + return None + + +def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: + """ + Generate name for VC member device using configured pattern from settings. + + Args: + master_name: Name of the master/primary device + position: VC position number + serial: Optional serial number of the member device + + Returns: + Generated member device name + + Examples: + pattern="-M{position}" -> "switch01-M2" + pattern=" ({position})" -> "switch01 (2)" + pattern="-SW{position}" -> "switch01-SW2" + pattern=" [{serial}]" -> "switch01 [ABC123]" + """ + # Import here to avoid circular dependency + from ..models import LibreNMSSettings + + # Get pattern from settings with fallback to default + try: + settings = LibreNMSSettings.objects.first() + pattern = settings.vc_member_name_pattern if settings else "-M{position}" + except Exception as e: + logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") + pattern = "-M{position}" + + # Prepare format variables + format_vars = { + "master_name": master_name, + "position": position, + "serial": serial or "", + } + + # Apply pattern - pattern should be suffix/prefix, not full name + try: + formatted_suffix = pattern.format(**format_vars) + return f"{master_name}{formatted_suffix}" + except KeyError as e: + logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") + return f"{master_name}-M{position}" + + +def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: + """ + Regenerate suggested VC member names using the actual master device name. + + This ensures preview shows accurate names after use_sysname and strip_domain + are applied to the master device name. + + Args: + vc_data: Virtual chassis detection data dict + master_name: The actual name that will be used for master device in NetBox + + Returns: + Updated vc_data dict with corrected suggested_name for each member + """ + if not vc_data or not vc_data.get("is_stack"): + return vc_data + + for idx, member in enumerate(vc_data.get("members", [])): + raw_position = member.get("position", idx) + try: + base_position = int(raw_position) + except (TypeError, ValueError): + base_position = idx + position = base_position + 1 # Convert to 1-based position + member["position"] = base_position + member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) + + return vc_data + + +def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): + """ + Create Virtual Chassis and member devices from detection info. + + This function creates a NetBox VirtualChassis with the master device + and all detected member devices, wrapped in a transaction for safety. + + Args: + master_device: The imported device (becomes VC master) + members_info: List of member dicts from VC detection + libre_device: Original LibreNMS device data + + Returns: + VirtualChassis: The created virtual chassis instance + + Raises: + ValidationError: If member count validation fails + IntegrityError: If duplicate serials/names are detected + Exception: For other creation errors + + Example members_info: + [ + {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, + {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} + ] + """ + + # Store original master device state for rollback + original_master_name = master_device.name + original_vc = master_device.virtual_chassis + original_vc_position = master_device.vc_position + + try: + with transaction.atomic(): + # Rename master device to include position 1 pattern + master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) + + # Check if renamed master conflicts with existing device + if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): + logger.warning( + f"Cannot rename master to '{master_device_new_name}' - name already exists. " + f"Keeping original name '{original_master_name}'" + ) + master_base_name = original_master_name + else: + master_device.name = master_device_new_name + master_base_name = original_master_name + + # Create VC using original base name + vc_name = master_base_name + vc = VirtualChassis.objects.create( + name=vc_name, + master=master_device, + domain=f"librenms-{libre_device['device_id']}", + ) + + # Update master device + master_device.virtual_chassis = vc + master_device.vc_position = 1 # Master is position 1 + master_device.save() + + # Create member devices for remaining positions + position = 2 # Start at 2 (master is 1) + members_created = 0 + + for member in members_info: + # Skip if this is the master's serial + if member.get("serial") == master_device.serial: + continue + + serial = member.get("serial") + + member_rack = master_device.rack + member_location = master_device.location or ( + member_rack.location if member_rack and member_rack.location else None + ) + + # Check for duplicate serial + if serial and Device.objects.filter(serial=serial).exists(): + logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") + continue + + member_name = _generate_vc_member_name(master_base_name, position, serial=serial) + + # Check for duplicate name + if Device.objects.filter(name=member_name).exists(): + logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") + continue + + Device.objects.create( + name=member_name, + device_type=master_device.device_type, + role=master_device.role, + site=master_device.site, + location=member_location, + rack=member_rack, + platform=master_device.platform, + serial=serial, + virtual_chassis=vc, + vc_position=position, + comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" + f"Auto-created from stack inventory", + ) + members_created += 1 + position += 1 + + # Validate member count + expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) + if members_created < expected_members: + logger.warning( + f"Created {members_created} members but expected {expected_members}. " + "Some members may have been skipped due to duplicates." + ) + + logger.info( + f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " + f"(1 master + {members_created} additional)" + ) + + return vc + + except Exception as e: + # Rollback master device to original state + logger.error( + f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." + ) + master_device.name = original_master_name + master_device.virtual_chassis = original_vc + master_device.vc_position = original_vc_position + master_device.save() + raise diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py new file mode 100644 index 0000000000..eadd97d200 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -0,0 +1,216 @@ +"""Virtual machine creation and import operations.""" + +import logging + +from dcim.models import DeviceRole +from django.utils import timezone +from virtualization.models import Cluster + +from ..librenms_api import LibreNMSAPI +from .device_operations import _determine_device_name, fetch_device_with_cache, validate_device_for_import +from .permissions import require_permissions + +logger = logging.getLogger(__name__) + + +def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): + """ + Create a NetBox VirtualMachine from LibreNMS device data. + + Args: + libre_device: Device data from LibreNMS + validation: Validation result from validate_device_for_import with import_as_vm=True + use_sysname: If True, prefer sysName; if False, use hostname + role: Optional DeviceRole to assign to the VM + + Returns: + Created VirtualMachine instance + + Raises: + Exception if VM cannot be created + """ + from virtualization.models import VirtualMachine + + if not validation["can_import"]: + raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") + + # Extract matched objects from validation + cluster = validation["cluster"]["cluster"] + platform = validation["platform"].get("platform") + + # Determine VM name - use pre-computed name if available (handles strip_domain) + vm_name = libre_device.get("_computed_name") + if not vm_name: + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=False, + device_id=libre_device.get("device_id"), + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + # Create the VM with librenms_id custom field + vm = VirtualMachine.objects.create( + name=vm_name, + cluster=cluster, + role=role, # Optional VM role + platform=platform, + comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + custom_field_data={"librenms_id": int(libre_device["device_id"])}, + ) + + logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") + return vm + + +def bulk_import_vms( + vm_imports: dict[int, dict[str, int]], + api: LibreNMSAPI, + sync_options: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices as VMs in NetBox. + + Handles validation, cluster/role assignment, name determination, + and VM creation. Supports both synchronous and background job execution. + + This function consolidates VM import logic that was previously duplicated + in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior + across synchronous and background import paths. + + Args: + vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} + api: LibreNMSAPI instance for device fetching + sync_options: Optional dict with use_sysname, strip_domain settings + libre_devices_cache: Optional pre-fetched device data cache + job: Optional JobRunner instance for background job logging/cancellation + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + Dict with keys: + - success: List of {"device_id": int, "device": VM, "message": str} + - failed: List of {"device_id": int, "error": str} + - skipped: List of {"device_id": int, "reason": str} + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous import from view + >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} + >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) + >>> print(f"Created {len(result['success'])} VMs") + >>> + >>> # Background job import + >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) + """ + from netbox_librenms_plugin.import_validation_helpers import ( + apply_cluster_to_validation, + apply_role_to_validation, + ) + + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") + + result = {"success": [], "failed": [], "skipped": []} + vm_ids = list(vm_imports.keys()) + + # Use job logger if available, otherwise standard logger + log = job.logger if job else logger + + for idx, vm_id in enumerate(vm_ids, start=1): + # Check for job cancellation every 5 VMs + if job and idx % 5 == 0: + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in ("failed", "errored"): + log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") + break + log.info(f"Imported VM {idx} of {len(vm_ids)}") + + try: + # Fetch device data (uses cache helper) + libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) + + if not libre_device: + result["failed"].append( + { + "device_id": vm_id, + "error": f"Device {vm_id} not found in LibreNMS", + } + ) + log.error(f"Device {vm_id} not found in LibreNMS") + continue + + # Validate as VM + validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) + + # Check if VM already exists + if validation.get("existing_device"): + result["skipped"].append( + { + "device_id": vm_id, + "reason": f"VM already exists: {validation['existing_device'].name}", + } + ) + log.info(f"VM already exists: {validation['existing_device'].name}") + continue + + # Apply manual cluster and role selections + vm_mappings = vm_imports[vm_id] + cluster_id = vm_mappings.get("cluster_id") + role_id = vm_mappings.get("device_role_id") + + if cluster_id: + cluster = Cluster.objects.filter(id=cluster_id).first() + if cluster: + apply_cluster_to_validation(validation, cluster) + + role = None + if role_id: + role = DeviceRole.objects.filter(id=role_id).first() + if role: + apply_role_to_validation(validation, role, is_vm=True) + + # Determine VM name + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=vm_id, + ) + + # Update validation with computed name + libre_device["_computed_name"] = vm_name + + # Create VM + vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) + + result["success"].append( + { + "device_id": vm_id, + "device": vm, + "message": f"VM {vm.name} created successfully", + } + ) + log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") + + except Exception as vm_error: + log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) + result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) + + return result diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index 5de9db6c25..d2e1013505 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -686,6 +686,51 @@ def get_device_inventory(self, device_id): except requests.exceptions.RequestException as e: return False, str(e) + def get_device_transceivers(self, device_id): + """ + Fetch all transceiver data for a device from LibreNMS. + + Route: /api/v0/devices/{device_id}/transceivers + + This is a separate data source from entity inventory. Some vendors + (e.g., Nokia/SROS) don't expose SFPs via ENTITY-MIB but do report + them through vendor-specific MIBs which LibreNMS surfaces here. + + Args: + device_id: LibreNMS device ID + + Returns: + tuple: (success: bool, data: list) + + Example transceiver item: + { + "port_id": 519, + "entity_physical_index": 1610899520, + "type": "CFP2/QSFP28", + "model": "3HE10550AARA01", + "serial": "X42AU0D", + "channels": 4, + "connector": "LC", + "wavelength": 1301, + ... + } + """ + try: + response = requests.get( + f"{self.librenms_url}/api/v0/devices/{device_id}/transceivers", + headers=self.headers, + timeout=DEFAULT_API_TIMEOUT, + verify=self.verify_ssl, + ) + response.raise_for_status() + + if response.status_code == 200: + data = response.json() + return True, data.get("transceivers", []) + return False, [] + except requests.exceptions.RequestException as e: + return False, str(e) + def get_poller_groups(self): """ Fetch all poller groups from LibreNMS. diff --git a/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py b/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py new file mode 100644 index 0000000000..dcd8fc4fd5 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py @@ -0,0 +1,45 @@ +# Generated by Django 5.2.10 on 2026-02-17 11:48 + +import django.db.models.deletion +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0225_gfk_indexes"), + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0008_librenmssettings_import_defaults"), + ] + + operations = [ + migrations.CreateModel( + name="DeviceTypeMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_hardware", models.CharField(max_length=255, unique=True)), + ("description", models.TextField(blank=True)), + ( + "netbox_device_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="librenms_mappings", + to="dcim.devicetype", + ), + ), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_hardware"], + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py b/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py new file mode 100644 index 0000000000..796bbceafd --- /dev/null +++ b/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py @@ -0,0 +1,49 @@ +# Generated by Django 5.2.10 on 2026-02-17 12:23 + +import django.db.models.deletion +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0225_gfk_indexes"), + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0009_add_devicetypemapping"), + ] + + operations = [ + migrations.AlterModelOptions( + name="interfacetypemapping", + options={"ordering": ["librenms_type", "librenms_speed"]}, + ), + migrations.CreateModel( + name="ModuleTypeMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_model", models.CharField(max_length=255, unique=True)), + ("description", models.TextField(blank=True)), + ( + "netbox_module_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="librenms_mappings", + to="dcim.moduletype", + ), + ), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_model"], + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0011_modulebaymapping.py b/netbox_librenms_plugin/migrations/0011_modulebaymapping.py new file mode 100644 index 0000000000..5b3c2c3be0 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0011_modulebaymapping.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.10 on 2026-02-17 12:29 + +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0010_add_moduletypemapping"), + ] + + operations = [ + migrations.CreateModel( + name="ModuleBayMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_name", models.CharField(max_length=255)), + ("librenms_class", models.CharField(blank=True, max_length=50)), + ("netbox_bay_name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_name"], + "unique_together": {("librenms_name", "librenms_class")}, + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py b/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py new file mode 100644 index 0000000000..52ff053e20 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("netbox_librenms_plugin", "0011_modulebaymapping"), + ] + + operations = [ + migrations.AddField( + model_name="modulebaymapping", + name="is_regex", + field=models.BooleanField( + default=False, + help_text="Treat LibreNMS Name as a regex pattern with backreferences in NetBox Bay Name", + ), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0013_normalizationrule.py b/netbox_librenms_plugin/migrations/0013_normalizationrule.py new file mode 100644 index 0000000000..71d1f80509 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0013_normalizationrule.py @@ -0,0 +1,93 @@ +"""Restore NormalizationRule model. + +The table was created by earlier migrations (0013 + 0014 in a previous branch) +and already exists in the database. This migration uses SeparateDatabaseAndState +so Django's ORM knows about the model without trying to CREATE the table again. +If the table doesn't exist (fresh install), the database_operations handle creation. +""" + +import django.db.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0001_initial"), + ("extras", "0001_initial"), + ("netbox_librenms_plugin", "0012_add_is_regex_to_modulebaymapping"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name="NormalizationRule", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ( + "scope", + models.CharField( + choices=[ + ("module_type", "Module Type"), + ("device_type", "Device Type"), + ("module_bay", "Module Bay"), + ], + max_length=50, + ), + ), + ("match_pattern", models.CharField(max_length=500)), + ("replacement", models.CharField(max_length=500)), + ("priority", models.PositiveIntegerField(default=100)), + ("description", models.TextField(blank=True)), + ( + "manufacturer", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="normalization_rules", + to="dcim.manufacturer", + ), + ), + ( + "tags", + taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag"), + ), + ], + options={ + "ordering": ["scope", "priority", "pk"], + }, + ), + ], + database_operations=[ + migrations.RunSQL( + sql=""" + CREATE TABLE IF NOT EXISTS "netbox_librenms_plugin_normalizationrule" ( + "id" bigserial NOT NULL PRIMARY KEY, + "created" timestamp with time zone NULL, + "last_updated" timestamp with time zone NULL, + "custom_field_data" jsonb NOT NULL DEFAULT '{}'::jsonb, + "scope" varchar(50) NOT NULL, + "match_pattern" varchar(500) NOT NULL, + "replacement" varchar(500) NOT NULL, + "priority" integer NOT NULL DEFAULT 100 CHECK ("priority" >= 0), + "description" text NOT NULL DEFAULT '', + "manufacturer_id" bigint NULL REFERENCES "dcim_manufacturer" ("id") + DEFERRABLE INITIALLY DEFERRED + ); + CREATE INDEX IF NOT EXISTS "netbox_librenms_plugin_norm_mfg_idx" + ON "netbox_librenms_plugin_normalizationrule" ("manufacturer_id"); + """, + reverse_sql="DROP TABLE IF EXISTS netbox_librenms_plugin_normalizationrule;", + ), + ], + ), + ] diff --git a/netbox_librenms_plugin/models.py b/netbox_librenms_plugin/models.py index cd79f47550..3978d9c5f4 100644 --- a/netbox_librenms_plugin/models.py +++ b/netbox_librenms_plugin/models.py @@ -1,4 +1,8 @@ +import re + from dcim.choices import InterfaceTypeChoices +from dcim.models import DeviceType, Manufacturer, ModuleType +from django.core.exceptions import ValidationError from django.db import models from django.urls import reverse from netbox.models import NetBoxModel @@ -71,6 +75,205 @@ class Meta: """Meta options for InterfaceTypeMapping.""" unique_together = ["librenms_type", "librenms_speed"] + ordering = ["librenms_type", "librenms_speed"] def __str__(self): return f"{self.librenms_type} + {self.librenms_speed} -> {self.netbox_type}" + + +class DeviceTypeMapping(NetBoxModel): + """Map LibreNMS hardware strings to NetBox DeviceType objects.""" + + librenms_hardware = models.CharField( + max_length=255, + unique=True, + help_text="Hardware string as reported by LibreNMS (e.g., 'Juniper MX480 Internet Backbone Router')", + ) + netbox_device_type = models.ForeignKey( + DeviceType, + on_delete=models.CASCADE, + related_name="librenms_mappings", + help_text="The NetBox DeviceType this hardware string maps to", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:devicetypemapping_detail", args=[self.pk]) + + class Meta: + """Meta options for DeviceTypeMapping.""" + + ordering = ["librenms_hardware"] + + def __str__(self): + return f"{self.librenms_hardware} -> {self.netbox_device_type}" + + +class ModuleTypeMapping(NetBoxModel): + """Map LibreNMS inventory model names to NetBox ModuleType objects.""" + + librenms_model = models.CharField( + max_length=255, + unique=True, + help_text="Model name from LibreNMS inventory (entPhysicalModelName)", + ) + netbox_module_type = models.ForeignKey( + ModuleType, + on_delete=models.CASCADE, + related_name="librenms_mappings", + help_text="The NetBox ModuleType this model name maps to", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:moduletypemapping_detail", args=[self.pk]) + + class Meta: + """Meta options for ModuleTypeMapping.""" + + ordering = ["librenms_model"] + + def __str__(self): + return f"{self.librenms_model} -> {self.netbox_module_type}" + + +class ModuleBayMapping(NetBoxModel): + """Map LibreNMS inventory names to NetBox module bay names. + + Used when LibreNMS inventory names don't match NetBox bay names exactly. + For example: LibreNMS "Power Supply 1" β†’ NetBox "PS1". + When is_regex is True, librenms_name is treated as a regex pattern and + netbox_bay_name can use backreferences (\\1, \\2, etc.). + Mappings are global (not scoped to device type or manufacturer). + """ + + librenms_name = models.CharField( + max_length=255, + help_text="Name from LibreNMS inventory (entPhysicalName). " + "When 'Use Regex' is enabled, this is a Python regex pattern.", + ) + librenms_class = models.CharField( + max_length=50, + blank=True, + help_text="Optional entPhysicalClass filter (e.g. 'powerSupply', 'fan', 'module')", + ) + netbox_bay_name = models.CharField( + max_length=255, + help_text="NetBox module bay name to match. With regex, supports backreferences (\\1, \\2, etc.).", + ) + is_regex = models.BooleanField( + default=False, + help_text="Treat LibreNMS Name as a regex pattern with backreferences in NetBox Bay Name", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def clean(self): + """Validate that regex patterns compile when is_regex is True.""" + super().clean() + if self.is_regex: + try: + re.compile(self.librenms_name) + except re.error as e: + raise ValidationError({"librenms_name": f"Invalid regex: {e}"}) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:modulebaymapping_detail", args=[self.pk]) + + class Meta: + """Meta options for ModuleBayMapping.""" + + unique_together = ["librenms_name", "librenms_class"] + ordering = ["librenms_name"] + + def __str__(self): + cls = f" [{self.librenms_class}]" if self.librenms_class else "" + return f"{self.librenms_name}{cls} -> {self.netbox_bay_name}" + + +class NormalizationRule(NetBoxModel): + """Regex-based string normalization applied before matching lookups. + + Generic building block: a single rule engine handles normalization + for module types, device types, module bays, and future scopes. + Rules are applied in priority order; each transforms the string + for the next rule in the chain. + + Example – strip Nokia revision suffixes: + scope: module_type + match_pattern: ^(3HE\\w{5}[A-Z]{2})[A-Z]{2}\\d{2}$ + replacement: \\1 + Result: 3HE16474AARA01 β†’ 3HE16474AA + """ + + SCOPE_MODULE_TYPE = "module_type" + SCOPE_DEVICE_TYPE = "device_type" + SCOPE_MODULE_BAY = "module_bay" + + SCOPE_CHOICES = [ + (SCOPE_MODULE_TYPE, "Module Type"), + (SCOPE_DEVICE_TYPE, "Device Type"), + (SCOPE_MODULE_BAY, "Module Bay"), + ] + + scope = models.CharField( + max_length=50, + choices=SCOPE_CHOICES, + help_text="Which matching lookup this rule applies to", + ) + manufacturer = models.ForeignKey( + Manufacturer, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="normalization_rules", + help_text="Optional: only apply this rule to items from this manufacturer. " + "Leave blank for vendor-agnostic rules.", + ) + match_pattern = models.CharField( + max_length=500, + help_text="Regex pattern to match against input string (Python re syntax)", + ) + replacement = models.CharField( + max_length=500, + help_text="Replacement string (supports regex back-references \\1, \\2, …)", + ) + priority = models.PositiveIntegerField( + default=100, + help_text="Lower values run first. Rules chain: each transforms the output of the previous.", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this rule", + ) + + def clean(self): + """Validate that match_pattern compiles as a regex.""" + super().clean() + try: + re.compile(self.match_pattern) + except re.error as e: + raise ValidationError({"match_pattern": f"Invalid regex: {e}"}) + + def get_absolute_url(self): + """Return the URL for this rule's detail page.""" + return reverse("plugins:netbox_librenms_plugin:normalizationrule_detail", args=[self.pk]) + + class Meta: + """Meta options for NormalizationRule.""" + + ordering = ["scope", "priority", "pk"] + + def __str__(self): + return f"[{self.get_scope_display()}] {self.match_pattern} β†’ {self.replacement}" diff --git a/netbox_librenms_plugin/navigation.py b/netbox_librenms_plugin/navigation.py index a08e62740f..052c06363c 100644 --- a/netbox_librenms_plugin/navigation.py +++ b/netbox_librenms_plugin/navigation.py @@ -31,6 +31,74 @@ ), ), ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:devicetypemapping_list", + link_text="Device Type Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:devicetypemapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:devicetypemapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:moduletypemapping_list", + link_text="Module Type Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:moduletypemapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:moduletypemapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:modulebaymapping_list", + link_text="Module Bay Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:modulebaymapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:modulebaymapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:normalizationrule_list", + link_text="Normalization Rules", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:normalizationrule_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:normalizationrule_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), ), ), ( diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js index c89473c02a..728406c703 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js @@ -310,6 +310,8 @@ * * @param {HTMLElement} modalElement - The modal element to hide * @param {Object} fallbackBackdropRef - Reference object containing fallback backdrop (deprecated) + * WONTFIX: fallbackBackdropRef is unused β€” _hideManual uses querySelector which is + * correct for this plugin since only one modal is ever open at a time (Tabler, no Bootstrap). */ function hideModal(modalElement, fallbackBackdropRef) { if (!modalElement) { @@ -317,6 +319,14 @@ } const manager = new ModalManager(modalElement); + + // Try to recover an existing Bootstrap instance before falling back to manual + if (typeof bootstrap !== 'undefined' && bootstrap.Modal) { + manager.instance = bootstrap.Modal.getInstance(modalElement); + } else if (typeof window.bootstrap !== 'undefined' && window.bootstrap.Modal) { + manager.instance = window.bootstrap.Modal.getInstance(modalElement); + } + manager.hide(); } @@ -338,6 +348,7 @@ function pollJobStatus(jobId, jobPk, pollUrl, baseUrl, originalFilters, deviceCount) { const messageEl = document.getElementById('filter-progress-message'); const cancelBtn = document.getElementById('cancel-filter-btn'); + const filterModal = document.getElementById('filter-processing-modal'); // Get CSRF token from cookie or form (needed for cancel and status sync) let csrfToken = getCookie('csrftoken'); @@ -402,11 +413,8 @@ messageEl.textContent = 'Job already completed, loading results...'; } cancelBtn.textContent = 'Completed'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -448,11 +456,8 @@ messageEl.textContent = 'Job cancelled successfully.'; } cancelBtn.textContent = 'Cancelled'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -469,11 +474,8 @@ messageEl.textContent = 'Job stopped (status sync failed).'; } cancelBtn.textContent = 'Stopped'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -486,11 +488,8 @@ messageEl.textContent = 'Job completed, loading results...'; } cancelBtn.textContent = 'Completed'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -505,11 +504,8 @@ } cancelBtn.textContent = 'Close'; cancelBtn.disabled = false; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => window.location.href = baseUrl, 1000); @@ -590,11 +586,8 @@ if (statusValue === 'completed' || statusValue === 'finished') { pollingStopped = true; // Stop future polls - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } // Small delay to let modal close before redirect @@ -604,21 +597,15 @@ return; // Stop polling } else if (statusValue === 'stopped') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => window.location.href = baseUrl, 100); } else if (statusValue === 'failed') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } const errorMsg = data.data?.error; @@ -628,11 +615,8 @@ setTimeout(() => window.location.href = baseUrl, 100); } else if (statusValue === 'errored') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } const errorMsg = data.data?.error || 'Job encountered an error. Please try again.'; @@ -1031,11 +1015,8 @@ if (failedCount && failedCount.dataset.failedCount === '0') { setTimeout(() => { const resultsModal = document.getElementById('import-results-modal'); - if (resultsModal && typeof bootstrap !== 'undefined' && bootstrap.Modal) { - const modalInstance = bootstrap.Modal.getInstance(resultsModal); - if (modalInstance) { - modalInstance.hide(); - } + if (resultsModal) { + hideModal(resultsModal); } window.location.reload(); }, MODAL_AUTO_CLOSE_MS); diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js index cd470af1b1..0b42112b24 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js @@ -153,11 +153,15 @@ function initializeCountdowns() { if (window.vlanCountdownInterval) { clearInterval(window.vlanCountdownInterval); } + if (window.moduleCountdownInterval) { + clearInterval(window.moduleCountdownInterval); + } window.interfaceCountdownInterval = initializeCountdown("countdown-timer"); window.cableCountdownInterval = initializeCountdown("cable-countdown-timer"); window.ipCountdownInterval = initializeCountdown("ip-countdown-timer"); window.vlanCountdownInterval = initializeCountdown("vlan-countdown-timer"); + window.moduleCountdownInterval = initializeCountdown("module-countdown-timer"); } // ============================================ diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 8bf978ccd4..c7f60d07b8 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -444,7 +444,7 @@ def render_actions(self, value, record): buttons = [] if existing: - # Link to existing device/VM in NetBox + # Link to existing device/VM in NetBox + details button for conflict resolution if isinstance(existing, VirtualMachine): url_name = "virtualization:virtualmachine" title = "View VM in NetBox" @@ -457,6 +457,44 @@ def render_actions(self, value, record): f'' ) + + # Add details/conflict button for conflict resolution actions + details_url = self._build_validation_details_url(device_id, validation) + match_type = validation.get("existing_match_type", "") + serial_action = validation.get("serial_action") + has_mismatch = validation.get("device_type_mismatch", False) + has_actions = match_type in ("hostname", "serial") and serial_action is not None + has_name_sync = validation.get("name_sync_available", False) + has_sync_needed = match_type == "librenms_id" and serial_action in ("update_serial", "conflict") + + if has_mismatch: + btn_class = "btn-outline-danger" + btn_icon = "mdi-alert-circle" + btn_label = " Conflict" + elif has_actions: + btn_class = "btn-outline-warning" + btn_icon = "mdi-alert" + btn_label = " Conflict" + elif has_name_sync or has_sync_needed: + btn_class = "btn-outline-warning" + btn_icon = "mdi-information-outline" + btn_label = " Details" + else: + btn_class = "btn-outline-info" + btn_icon = "mdi-information-outline" + btn_label = " Details" + + btn_title = "Resolve conflict" if (has_actions or has_mismatch) else "View details" + buttons.append( + f'' + ) elif is_ready: # Ready to import - show Import and Details buttons details_url = self._build_validation_details_url(device_id, validation) diff --git a/netbox_librenms_plugin/tables/mappings.py b/netbox_librenms_plugin/tables/mappings.py index 73949fd2c8..4b4b31a41d 100644 --- a/netbox_librenms_plugin/tables/mappings.py +++ b/netbox_librenms_plugin/tables/mappings.py @@ -1,7 +1,13 @@ import django_tables2 as tables from netbox.tables import NetBoxTable, columns -from netbox_librenms_plugin.models import InterfaceTypeMapping +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) class InterfaceTypeMappingTable(NetBoxTable): @@ -36,3 +42,132 @@ class Meta: "actions", ) attrs = {"class": "table table-hover table-headings table-striped"} + + +class DeviceTypeMappingTable(NetBoxTable): + """Table for displaying DeviceTypeMapping data.""" + + librenms_hardware = tables.Column(verbose_name="LibreNMS Hardware", linkify=True) + netbox_device_type = tables.Column(verbose_name="NetBox Device Type", linkify=True) + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for DeviceTypeMappingTable.""" + + model = DeviceTypeMapping + fields = ( + "id", + "librenms_hardware", + "netbox_device_type", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_hardware", + "netbox_device_type", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class ModuleTypeMappingTable(NetBoxTable): + """Table for displaying ModuleTypeMapping data.""" + + librenms_model = tables.Column(verbose_name="LibreNMS Model", linkify=True) + netbox_module_type = tables.Column(verbose_name="NetBox Module Type", linkify=True) + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for ModuleTypeMappingTable.""" + + model = ModuleTypeMapping + fields = ( + "id", + "librenms_model", + "netbox_module_type", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_model", + "netbox_module_type", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class ModuleBayMappingTable(NetBoxTable): + """Table for displaying ModuleBayMapping data.""" + + librenms_name = tables.Column(verbose_name="LibreNMS Name", linkify=True) + librenms_class = tables.Column(verbose_name="LibreNMS Class") + netbox_bay_name = tables.Column(verbose_name="NetBox Bay Name") + is_regex = columns.BooleanColumn(verbose_name="Regex") + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for ModuleBayMappingTable.""" + + model = ModuleBayMapping + fields = ( + "id", + "librenms_name", + "librenms_class", + "netbox_bay_name", + "is_regex", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_name", + "librenms_class", + "netbox_bay_name", + "is_regex", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class NormalizationRuleTable(NetBoxTable): + """Table for displaying NormalizationRule data.""" + + scope = tables.Column(verbose_name="Scope", linkify=True) + manufacturer = tables.Column(verbose_name="Manufacturer", linkify=True) + match_pattern = tables.Column(verbose_name="Match Pattern") + replacement = tables.Column(verbose_name="Replacement") + priority = tables.Column(verbose_name="Priority") + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for NormalizationRuleTable.""" + + model = NormalizationRule + fields = ( + "id", + "scope", + "manufacturer", + "match_pattern", + "replacement", + "priority", + "description", + "actions", + ) + default_columns = ( + "id", + "scope", + "match_pattern", + "replacement", + "priority", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} diff --git a/netbox_librenms_plugin/tables/modules.py b/netbox_librenms_plugin/tables/modules.py new file mode 100644 index 0000000000..458364fe10 --- /dev/null +++ b/netbox_librenms_plugin/tables/modules.py @@ -0,0 +1,180 @@ +import django_tables2 as tables +from django.urls import reverse +from django.utils.html import format_html +from utilities.paginator import EnhancedPaginator + +from netbox_librenms_plugin.utils import get_table_paginate_count + + +class LibreNMSModuleTable(tables.Table): + """Table for displaying LibreNMS inventory items mapped to NetBox modules.""" + + name = tables.Column(verbose_name="Name", attrs={"td": {"data-col": "name"}}) + model = tables.Column(verbose_name="Model", attrs={"td": {"data-col": "model"}}) + serial = tables.Column(verbose_name="Serial", attrs={"td": {"data-col": "serial"}}) + description = tables.Column(verbose_name="Description", attrs={"td": {"data-col": "description"}}) + item_class = tables.Column(verbose_name="Class", attrs={"td": {"data-col": "item_class"}}) + module_bay = tables.Column(verbose_name="Module Bay", attrs={"td": {"data-col": "module_bay"}}) + module_type = tables.Column(verbose_name="Module Type", attrs={"td": {"data-col": "module_type"}}) + status = tables.Column(verbose_name="Status", attrs={"td": {"data-col": "status"}}) + actions = tables.Column( + verbose_name="Actions", orderable=False, empty_values=(), attrs={"td": {"data-col": "actions"}} + ) + + class Meta: + attrs = {"class": "table table-hover object-list", "id": "librenms-module-table"} + row_attrs = {"class": lambda record: record.get("row_class", "")} + + def __init__(self, *args, device=None, **kwargs): + """Initialize table with optional device context.""" + self.device = device + self.csrf_token = "" + super().__init__(*args, **kwargs) + self.tab = "modules" + self.htmx_url = None + self.prefix = "modules_" + + def configure(self, request): + """Configure pagination settings and CSRF token.""" + from django.middleware.csrf import get_token + + self.csrf_token = get_token(request) + paginate = {"paginator_class": EnhancedPaginator, "per_page": get_table_paginate_count(request, self.prefix)} + tables.RequestConfig(request, paginate).configure(self) + + def render_name(self, value, record): + """Render inventory item name with tree indentation for sub-components.""" + depth = record.get("depth", 0) + if depth == 0: + return value or "-" + # Build visual tree prefix based on nesting depth + padding_px = depth * 20 + prefix = "└─ " + return format_html('{}{}', padding_px, prefix, value or "-") + + def render_model(self, value, record): + """Render model with link to module type if matched.""" + if not value or value == "-": + return "-" + if url := record.get("module_type_url"): + return format_html('{}', url, value) + return value + + def render_serial(self, value, record): + """Render serial number.""" + return value or "-" + + def render_description(self, value, record): + """Render description, truncated for display.""" + if not value: + return "-" + if len(value) > 60: + return format_html('{}…', value, value[:57]) + return value + + def render_item_class(self, value, record): + """Render the entPhysicalClass with an icon.""" + icons = { + "module": "mdi-expansion-card", + "ioModule": "mdi-expansion-card", + "cpmModule": "mdi-expansion-card", + "mdaModule": "mdi-expansion-card", + "fabricModule": "mdi-expansion-card", + "xioModule": "mdi-expansion-card", + "powerSupply": "mdi-power-plug", + "fan": "mdi-fan", + "port": "mdi-ethernet", + "other": "mdi-card-outline", + } + icon = icons.get(value, "mdi-card-outline") + return format_html(' {}', icon, value) + + def render_module_bay(self, value, record): + """Render module bay with link if found in NetBox.""" + if not value or value == "-": + return format_html('No matching bay') + if url := record.get("module_bay_url"): + return format_html('{}', url, value) + return value + + def render_module_type(self, value, record): + """Render module type match status.""" + if not value or value == "-": + return format_html('No matching type') + if url := record.get("module_type_url"): + return format_html('{}', url, value) + return value + + def render_status(self, value, record): + """Render sync status with badge.""" + badge_classes = { + "Installed": "bg-success", + "Matched": "bg-info", + "No Bay": "bg-warning", + "No Type": "bg-warning", + "Unmatched": "bg-secondary", + "Serial Mismatch": "bg-danger", + "Requires Upgrade": "bg-warning", + "Name Conflict": "bg-warning", + } + badge_class = badge_classes.get(value, "bg-secondary") + if warning := record.get("module_path_warning"): + return format_html('{}', badge_class, warning, value) + if warning := record.get("name_conflict_warning"): + return format_html( + '{}' + ' ', + badge_class, + warning, + value, + warning, + ) + return format_html('{}', badge_class, value) + + def render_actions(self, value, record): + """Render install button for matched modules and install branch for parents.""" + if not self.device: + return "" + + buttons = [] + + # Single install button + if record.get("can_install"): + url = reverse("plugins:netbox_librenms_plugin:install_module", kwargs={"pk": self.device.pk}) + buttons.append( + format_html( + '
' + '' + '' + '' + '' + '
", + url, + self.csrf_token, + record.get("module_bay_id", ""), + record.get("module_type_id", ""), + record.get("serial", ""), + ) + ) + + # Install branch button for parents with installable children + if record.get("has_installable_children") and record.get("ent_physical_index"): + url = reverse("plugins:netbox_librenms_plugin:install_branch", kwargs={"pk": self.device.pk}) + buttons.append( + format_html( + '
' + '' + '' + '
", + url, + self.csrf_token, + record.get("ent_physical_index", ""), + ) + ) + + return format_html("{}", format_html("".join(str(b) for b in buttons))) if buttons else "" diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_interface_sync_content.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_interface_sync_content.html index f3d55b8710..3d7e1406cd 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_interface_sync_content.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_interface_sync_content.html @@ -265,7 +265,6 @@ {% include 'netbox_librenms_plugin/_ipaddress_sync.html' %} + {% if module_sync %} +
+ {% include 'netbox_librenms_plugin/inc/_module_sync.html' %} +
+ {% endif %} + {% with model_name=object|meta:"model_name" %} {% if model_name == "device" %}
+
+
+ + + + + + + + + + + + + + + + + +
LibreNMS NameLibreNMS ClassNetBox Bay NameDescription
{{ object.librenms_name }}{{ object.librenms_class|default:"β€”" }}{{ object.netbox_bay_name }}{{ object.description|default:"β€”" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html new file mode 100644 index 0000000000..fb87f901ec --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Module Bay Mapping

+

Map LibreNMS inventory container names to NetBox module bay names. + When synchronizing modules from LibreNMS, these mappings determine which + NetBox module bay a LibreNMS component should be installed into.

+

Example: Map "Linecard(slot 1)" to "Slot 1", or "Power Supply 1" to "PS1"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html new file mode 100644 index 0000000000..019b0e51ed --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html @@ -0,0 +1,28 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + +
LibreNMS ModelNetBox Module TypeDescription
{{ object.librenms_model }}{{ object.netbox_module_type }}{{ object.description|default:"β€”" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html new file mode 100644 index 0000000000..4cfc22d592 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Module Type Mapping

+

Map LibreNMS inventory model names (entPhysicalModelName) to NetBox module types. + When synchronizing modules from LibreNMS, these mappings are checked first before + falling back to exact model / part number matching.

+

Example: Map "710-017414" to module type "WS-X4908-10GE"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html new file mode 100644 index 0000000000..a1be7537a3 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html @@ -0,0 +1,34 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + + + + + + + +
ScopeManufacturerMatch PatternReplacementPriorityDescription
{{ object.get_scope_display }}{% if object.manufacturer %}{{ object.manufacturer }}{% else %}β€”{% endif %}{{ object.match_pattern }}{{ object.replacement }}{{ object.priority }}{{ object.description|default:"β€”" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html new file mode 100644 index 0000000000..d543141680 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html @@ -0,0 +1,16 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Normalization Rules

+

Regex-based string normalization applied before matching lookups. + When a LibreNMS string doesn't match any NetBox object or mapping entry, + normalization rules transform it (e.g. strip revision suffixes) and retry.

+

Rules are chained in priority order per scope. One rule engine serves + module types, device types, and module bays.

+

Example β€” strip Nokia revision suffixes:
+ ^(3HE\w{5}[A-Z]{2})[A-Z]{2}\d{2}$ β†’ \1
+ Turns 3HE16474AARA01 into 3HE16474AA which matches the part number.

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 96ed7ba62f..8e6d67c18f 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -149,8 +149,8 @@ def test_determine_device_name_fallback_to_device_id(self): class TestDeviceRetrieval: """Test device retrieval and filtering functions.""" - @patch("netbox_librenms_plugin.import_utils.cache") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.filters.cache") + @patch("netbox_librenms_plugin.import_utils.filters.LibreNMSAPI") def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cache): """Retrieve devices from LibreNMS API.""" mock_cache.get.return_value = None # Cache miss @@ -171,7 +171,7 @@ def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cach assert len(devices) == 2 assert devices[0]["hostname"] == "switch-01" - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_librenms_devices_for_import_uses_cache(self, mock_cache): """Cached results returned on repeat call.""" cached_devices = [ @@ -189,7 +189,7 @@ def test_get_librenms_devices_for_import_uses_cache(self, mock_cache): assert devices[0]["hostname"] == "cached-device" mock_api.list_devices.assert_not_called() - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_librenms_devices_for_import_cache_miss(self, mock_cache): """API called when cache empty.""" mock_cache.get.return_value = None @@ -209,7 +209,7 @@ def test_get_librenms_devices_for_import_cache_miss(self, mock_cache): mock_api.list_devices.assert_called_once() assert len(devices) == 1 - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_for_filters_success(self, mock_cache): """Returns correct count from API.""" mock_cache.get.return_value = [ @@ -225,7 +225,7 @@ def test_get_device_count_for_filters_success(self, mock_cache): assert count == 3 - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_excludes_disabled(self, mock_cache): """Count respects show_disabled filter parameter.""" mock_cache.get.return_value = [ @@ -279,7 +279,7 @@ def test_empty_virtual_chassis_data(self): assert data["members"] == [] assert data["detection_error"] is None - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.virtual_chassis.cache") def test_get_virtual_chassis_data_returns_empty_without_api(self, mock_cache): """Get VC data returns empty structure without API.""" from netbox_librenms_plugin.import_utils import get_virtual_chassis_data @@ -299,14 +299,14 @@ class TestDeviceValidation: """Test device validation for import.""" @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_site_match_found( self, mock_site_model, @@ -358,14 +358,14 @@ def test_validate_device_site_match_found( assert result["site"]["site"] == mock_site @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_site_not_found( self, mock_site_model, @@ -416,15 +416,15 @@ def test_validate_device_site_not_found( assert any("site" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_platform_match_found( self, mock_device_type, @@ -478,14 +478,14 @@ def test_validate_device_platform_match_found( assert result["platform"]["platform"] == mock_platform @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_platform_not_found( self, mock_site_model, @@ -535,14 +535,14 @@ def test_validate_device_platform_not_found( assert result["platform"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_type_match_found( self, mock_site_model, @@ -594,15 +594,15 @@ def test_validate_device_type_match_found( assert result["device_type"]["device_type"] == mock_dt @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_type_not_found( self, mock_device_type, @@ -655,14 +655,14 @@ def test_validate_device_type_not_found( assert any("device type" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_role_required( self, mock_site_model, @@ -716,14 +716,14 @@ def test_validate_device_role_required( assert any("role" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_handles_empty_location( self, mock_site_model, @@ -775,14 +775,14 @@ def test_validate_device_handles_empty_location( assert result["site"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_handles_empty_os( self, mock_site_model, @@ -833,15 +833,15 @@ def test_validate_device_handles_empty_os( assert result["platform"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_handles_empty_hardware( self, mock_device_type, @@ -894,14 +894,14 @@ def test_validate_device_handles_empty_hardware( assert result["device_type"]["matched"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_duplicate_detection( self, mock_site_model, @@ -933,14 +933,14 @@ def test_validate_device_duplicate_detection( assert result["can_import"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_returns_complete_state( self, mock_site_model, @@ -999,17 +999,17 @@ def test_validate_device_returns_complete_state( assert "cluster" in result assert "platform" in result - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.device_operations.cache") @patch("virtualization.models.Cluster") @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_import_as_vm( self, mock_site_model, @@ -1064,14 +1064,14 @@ def test_validate_device_import_as_vm( assert result["cluster"]["available_clusters"] == mock_clusters @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_existing_vm_blocks_import( self, mock_site_model, @@ -1102,3 +1102,1076 @@ def test_validate_device_existing_vm_blocks_import( assert result["existing_device"] == existing_vm assert result["can_import"] is False assert result["import_as_vm"] is True + + +class TestSerialNumberMatching: + """Test serial number matching in device validation.""" + + SERIAL_PATCHES = [ + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", + "virtualization.models.VirtualMachine", + ] + + def _start_patches(self): + """Start all common patches and return mocks in standard order.""" + self._patchers = [patch(p) for p in self.SERIAL_PATCHES] + mocks = [p.start() for p in self._patchers] + ( + self.mock_site_model, + self.mock_rack, + self.mock_cluster, + self.mock_role, + self.mock_match_type, + self.mock_find_platform, + self.mock_find_site, + self.mock_device, + self.mock_vm, + ) = mocks + + def _stop_patches(self): + """Stop all patches.""" + for p in self._patchers: + p.stop() + + def setup_method(self): + """Set up common patches for serial number tests.""" + self._start_patches() + + def teardown_method(self): + """Tear down patches.""" + self._stop_patches() + + def test_serial_match_blocks_import(self): + """Device with matching serial blocks import.""" + existing = MagicMock() + existing.name = "existing-device" + existing.serial = "ABC123" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "new-hostname", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["can_import"] is False + assert result["existing_match_type"] == "serial" + assert result["existing_device"] == existing + + def test_serial_match_same_hostname_offers_link(self): + """Serial + hostname match offers link action.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["serial_action"] == "link" + assert result["existing_match_type"] == "serial" + assert "not linked to LibreNMS" in result["warnings"][0] + + def test_serial_match_diff_hostname_offers_hostname_differs(self): + """Serial matches but hostname differs offers hostname_differs action.""" + existing = MagicMock() + existing.name = "old-hostname" + existing.serial = "ABC123" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "new-hostname", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["serial_action"] == "hostname_differs" + assert result["existing_match_type"] == "serial" + assert "reinstalled" in result["warnings"][0] + + def test_hostname_match_diff_serial_offers_update(self): + """Hostname matches but serial differs offers update_serial action.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "OLD_SERIAL" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "name__iexact" in kwargs: + result.first.return_value = existing + elif "serial" in kwargs: + result.first.return_value = None + result.exclude.return_value.first.return_value = None + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "NEW_SERIAL"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["serial_action"] == "update_serial" + assert result["existing_match_type"] == "hostname" + assert "Hardware may have been replaced" in result["warnings"][0] + + def _setup_no_match_mocks(self): + """Configure mocks for tests where no device match is expected.""" + self.mock_vm.objects.filter.return_value.first.return_value = None + self.mock_device.objects.filter.return_value.first.return_value = None + self.mock_find_site.return_value = {"found": False, "site": None, "match_type": None, "confidence": 0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + def test_serial_dash_ignored(self): + """Serial '-' is not treated as a match.""" + self._setup_no_match_mocks() + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "-"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] is None + assert result["serial_action"] is None + + def test_serial_empty_ignored(self): + """Empty serial skips serial matching.""" + self._setup_no_match_mocks() + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": ""} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] is None + assert result["serial_action"] is None + + def test_serial_none_ignored(self): + """None serial skips serial matching.""" + self._setup_no_match_mocks() + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": None} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] is None + assert result["serial_action"] is None + + def test_hostname_match_serial_conflict_warns(self): + """Hostname matches, incoming serial already on another device warns about conflict.""" + hostname_device = MagicMock() + hostname_device.name = "switch-01" + hostname_device.serial = "OLD_SERIAL" + + serial_conflict_device = MagicMock() + serial_conflict_device.name = "other-device" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "name__iexact" in kwargs: + result.first.return_value = hostname_device + elif "serial" in kwargs: + result.exclude.return_value.first.return_value = serial_conflict_device + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "CONFLICTING_SERIAL"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["serial_action"] == "conflict" + assert result["existing_match_type"] == "hostname" + assert "Serial conflict" in result["warnings"][0] + + def test_librenms_id_match_shows_serial_confirmed(self): + """librenms_id match with matching serial shows confirmation.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "custom_field_data__librenms_id" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "sysName": "switch-01", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] == "librenms_id" + assert result["can_import"] is False + assert result["serial_confirmed"] is True + assert result["name_matches"] is True + + def test_librenms_id_match_detects_serial_drift(self): + """librenms_id match with different serial warns about drift.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "OLD_SERIAL" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "custom_field_data__librenms_id" in kwargs: + result.first.return_value = existing + elif "serial" in kwargs: + result.first.return_value = None + result.exclude.return_value.first.return_value = None + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "NEW_SERIAL"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] == "librenms_id" + assert result["serial_action"] == "update_serial" + assert any("Hardware may have been replaced" in w for w in result["warnings"]) + + def test_librenms_id_match_still_validates_site(self): + """librenms_id match continues to populate site/type validation.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "custom_field_data__librenms_id" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + mock_site = MagicMock(id=1, name="DC1") + self.mock_find_site.return_value = {"found": True, "site": mock_site, "match_type": "exact", "confidence": 1.0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + mock_dt = MagicMock() + self.mock_match_type.return_value = {"matched": True, "device_type": mock_dt, "match_type": "exact"} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "location": "DC1", "hardware": "WS-C4900M"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] == "librenms_id" + assert result["can_import"] is False + assert result["is_ready"] is False + # Site and device_type should still be populated + assert result["site"]["found"] is True + assert result["site"]["site"] == mock_site + assert result["device_type"]["found"] is True + + def test_existing_device_role_populated(self): + """Existing device's role should be shown in validation details.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + mock_existing_role = MagicMock() + mock_existing_role.name = "Access Switch" + existing.role = mock_existing_role + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = {"found": False, "site": None, "match_type": None, "confidence": 0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": True, "device_type": MagicMock(), "match_type": "exact"} + self.mock_role.objects.all.return_value = [mock_existing_role] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: + mock_cache.get.return_value = None + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + "serial": "ABC123", + "location": "", + "hardware": "", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_device"] == existing + assert result["device_role"]["found"] is True + assert result["device_role"]["role"] == mock_existing_role + + def test_device_type_mismatch_flagged(self): + """Device type mismatch between existing device and LibreNMS should be flagged.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + existing_device_type = MagicMock() + existing_device_type.pk = 1 + existing_device_type.__str__ = lambda self: "Old Type" + existing.device_type = existing_device_type + existing.role = MagicMock() + + librenms_device_type = MagicMock() + librenms_device_type.pk = 2 + librenms_device_type.__str__ = lambda self: "New Type" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = {"found": False, "site": None, "match_type": None, "confidence": 0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = { + "matched": True, + "device_type": librenms_device_type, + "match_type": "exact", + } + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: + mock_cache.get.return_value = None + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + "serial": "ABC123", + "location": "", + "hardware": "New Type", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["device_type_mismatch"] is True + assert any("Device type mismatch" in w for w in result["warnings"]) + + def test_no_device_type_mismatch_when_types_match(self): + """No mismatch flag when existing device type matches LibreNMS.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + same_device_type = MagicMock() + same_device_type.pk = 1 + existing.device_type = same_device_type + existing.role = MagicMock() + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = {"found": False, "site": None, "match_type": None, "confidence": 0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": True, "device_type": same_device_type, "match_type": "exact"} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: + mock_cache.get.return_value = None + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + "serial": "ABC123", + "location": "", + "hardware": "Same Type", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["device_type_mismatch"] is False + + +class TestDeviceConflictActionView: + """Test DeviceConflictActionView conflict resolution actions.""" + + def _create_view(self): + """Create a DeviceConflictActionView instance with mocked dependencies.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = object.__new__(DeviceConflictActionView) + view._librenms_api = MagicMock() + view._librenms_api.server_key = "default" + view.request = MagicMock() + view.request.user.has_perm.return_value = True + return view + + def _create_request(self, action, existing_device_id, use_sysname=False, strip_domain=False): + """Create a mock request with POST data.""" + request = MagicMock() + post_data = {"action": action, "existing_device_id": str(existing_device_id)} + if use_sysname: + post_data["use-sysname-toggle"] = "on" + if strip_domain: + post_data["strip-domain-toggle"] = "on" + request.POST = post_data + return request + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache): + """Link action should set librenms_id and update name from sysName.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "84.116.251.35" + + libre_device = { + "device_id": 10, + "hostname": "84.116.251.35", + "sysName": "switch-01.example.com", + "serial": "ABC123", + } + validation = {"can_import": False} + selections = {} + + request = self._create_request("link", 42, use_sysname=True) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.name == "switch-01.example.com" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key, mock_cache): + """Update action should set hostname, serial, and librenms_id.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "old-name" + existing_device.serial = "OLD-SERIAL" + + libre_device = { + "device_id": 10, + "hostname": "84.116.251.35", + "sysName": "new-name.example.com", + "serial": "NEW-SERIAL", + } + validation = {"can_import": False} + selections = {} + + request = self._create_request("update", 42, use_sysname=True) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.serial == "NEW-SERIAL" + assert existing_device.name == "new-name.example.com" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cache): + """Update serial action should update serial and librenms_id but not hostname.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "switch-01" + existing_device.serial = "OLD-SERIAL" + + libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "NEW-SERIAL"} + validation = {"can_import": False} + selections = {} + + request = self._create_request("update_serial", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.serial == "NEW-SERIAL" + # Name should NOT be changed by update_serial + assert existing_device.name == "switch-01" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_update_skips_dash_serial(self, mock_cache_key, mock_cache): + """Update should not set serial to '-' (LibreNMS placeholder).""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "switch-01" + existing_device.serial = "EXISTING" + + libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "-"} + validation = {"can_import": False} + selections = {} + + request = self._create_request("update_serial", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + # Serial should NOT be updated to '-' + assert existing_device.serial == "EXISTING" + + def test_missing_action_returns_400(self): + """Missing action or existing_device_id should return 400.""" + view = self._create_view() + request = MagicMock() + request.POST = {} + + response = view.post(request, device_id=10) + assert response.status_code == 400 + + def test_unknown_action_returns_400(self): + """Unknown action should return 400.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + request = self._create_request("invalid_action", 42) + + existing_device = MagicMock() + libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC"} + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_validate.return_value = (libre_device, {}, {}) + + response = view.post(request, device_id=10) + + assert response.status_code == 400 + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_sync_name_action_updates_name(self, mock_cache_key, mock_cache): + """Sync name action should update device name using sysName.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {"librenms_id": 10} + existing_device.name = "84.116.251.35" + + libre_device = { + "device_id": 10, + "hostname": "84.116.251.35", + "sysName": "switch-01.example.com", + "serial": "ABC123", + } + validation = {"can_import": False} + selections = {} + + request = self._create_request("sync_name", 42, use_sysname=True) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.name == "switch-01.example.com" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_device_type_mismatch_blocked_without_force(self, mock_cache_key, mock_cache): + """Action should be blocked when device_type_mismatch is True and force is not set.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + + libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC123"} + validation = {"can_import": False, "device_type_mismatch": True} + selections = {} + + request = self._create_request("link", 42, use_sysname=True) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_validate.return_value = (libre_device, validation, selections) + + response = view.post(request, device_id=10) + + assert response.status_code == 400 + existing_device.save.assert_not_called() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cache): + """Action should proceed when device_type_mismatch is True and force is set.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "old-name" + + libre_device = { + "device_id": 10, + "hostname": "switch-01", + "sysName": "switch-01.example.com", + "serial": "ABC123", + } + validation = {"can_import": False, "device_type_mismatch": True} + selections = {} + + request = self._create_request("link", 42, use_sysname=True) + request.POST["force"] = "on" + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.custom_field_data["librenms_id"] == 10 + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cache): + """Force with device_type_mismatch should update existing device's device_type.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "old-name" + + librenms_device_type = MagicMock() + librenms_device_type.pk = 99 + libre_device = { + "device_id": 10, + "hostname": "switch-01", + "sysName": "switch-01.example.com", + "serial": "ABC123", + } + validation = { + "can_import": False, + "device_type_mismatch": True, + "device_type": {"device_type": librenms_device_type}, + } + selections = {} + + request = self._create_request("link", 42, use_sysname=True) + request.POST["force"] = "on" + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.device_type == librenms_device_type + assert existing_device.custom_field_data["librenms_id"] == 10 + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_update_type_action_changes_device_type(self, mock_cache_key, mock_cache): + """update_type action should change device type on existing device.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {"librenms_id": 10} + existing_device.name = "switch-01" + old_device_type = MagicMock() + existing_device.device_type = old_device_type + + new_device_type = MagicMock() + new_device_type.pk = 99 + libre_device = { + "device_id": 10, + "hostname": "switch-01", + "serial": "ABC123", + } + validation = { + "can_import": False, + "device_type_mismatch": True, + "device_type": {"device_type": new_device_type}, + } + selections = {} + + request = self._create_request("update_type", 42) + request.POST["force"] = "on" + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.device_type == new_device_type + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_sync_serial_action(self, mock_cache_key, mock_cache): + """sync_serial action should update serial from LibreNMS.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.serial = "OLD123" + libre_device = {"device_id": 10, "serial": "NEW456", "sysName": "test"} + validation = {"existing_device": existing_device, "device_type_mismatch": False} + selections = {} + + request = self._create_request("sync_serial", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.serial == "NEW456" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_sync_platform_action(self, mock_cache_key, mock_cache): + """sync_platform action should update platform from LibreNMS OS.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.platform = None + libre_device = {"device_id": 10, "os": "ios", "sysName": "test"} + validation = {"existing_device": existing_device, "device_type_mismatch": False} + selections = {} + + mock_platform = MagicMock() + request = self._create_request("sync_platform", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("dcim.models.Platform") as mock_platform_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_platform_cls.objects.get.return_value = mock_platform + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.platform == mock_platform + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_sync_device_type_action(self, mock_cache_key, mock_cache): + """sync_device_type action should update device type from LibreNMS hardware match.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + new_device_type = MagicMock() + libre_device = {"device_id": 10, "hardware": "Catalyst C4900M", "sysName": "test"} + validation = {"existing_device": existing_device, "device_type_mismatch": False} + selections = {} + + request = self._create_request("sync_device_type", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.utils.match_librenms_hardware_to_device_type") as mock_hw_match, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_hw_match.return_value = {"matched": True, "device_type": new_device_type} + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.device_type == new_device_type + existing_device.save.assert_called_once() + + +class TestBuildSyncInfo: + """Test DeviceValidationDetailsView._build_sync_info method.""" + + def test_all_synced(self): + """When serial, platform, device type all match, all_synced is True.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + platform = MagicMock() + platform.pk = 1 + existing.platform = platform + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "ios", "hardware": "Catalyst C4900M"} + + with ( + patch("dcim.models.Platform") as mock_platform_cls, + patch("netbox_librenms_plugin.utils.match_librenms_hardware_to_device_type") as mock_hw_match, + ): + mock_platform_cls.objects.get.return_value = platform + mock_hw_match.return_value = {"matched": True, "device_type": device_type} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + assert result["all_synced"] is True + assert result["serial_synced"] is True + assert result["platform_synced"] is True + assert result["device_type_synced"] is True + + def test_serial_out_of_sync(self): + """When serial differs, serial_synced is False.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "OLD123" + existing.platform = None + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "NEW456", "os": "-", "hardware": "-"} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + assert result["serial_synced"] is False + assert result["all_synced"] is False + + def test_platform_out_of_sync(self): + """When platform differs, platform_synced is False.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + old_platform = MagicMock() + old_platform.pk = 1 + existing.platform = old_platform + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + new_platform = MagicMock() + new_platform.pk = 2 + + libre_device = {"serial": "ABC123", "os": "junos", "hardware": "-"} + + with patch("dcim.models.Platform") as mock_platform_cls: + mock_platform_cls.objects.get.return_value = new_platform + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + assert result["platform_synced"] is False + assert result["all_synced"] is False diff --git a/netbox_librenms_plugin/tests/test_init.py b/netbox_librenms_plugin/tests/test_init.py new file mode 100644 index 0000000000..6225484dd6 --- /dev/null +++ b/netbox_librenms_plugin/tests/test_init.py @@ -0,0 +1,171 @@ +"""Tests for netbox_librenms_plugin.__init__ module. + +Covers the _ensure_librenms_id_custom_field post_migrate signal handler. +""" + +from unittest.mock import MagicMock, patch + + +# ============================================================================= +# TestEnsureLibreNMSIdCustomField - 6 tests +# ============================================================================= + + +class TestEnsureLibreNMSIdCustomField: + """Test _ensure_librenms_id_custom_field signal handler.""" + + def setup_method(self): + """Reset the _executed flag before each test for consistent isolation.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = False + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_creates_custom_field_when_missing( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """Custom field is created with correct defaults when it does not exist.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [] + MockCustomField.objects.get_or_create.return_value = (mock_cf, True) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + + MockCustomField.objects.get_or_create.assert_called_once_with( + name="librenms_id", + defaults={ + "type": "integer", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Should have added content types for all 4 models + assert mock_cf.object_types.add.call_count == 4 + + # Should log when created + mock_get_logger.assert_called_with("netbox_librenms_plugin") + mock_get_logger.return_value.info.assert_called_once() + + def test_skips_when_already_executed(self): + """Handler is a no-op on second invocation (per-migrate dedup).""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = True + + with patch("extras.models.CustomField") as MockCustomField: + _ensure_librenms_id_custom_field(sender=None) + MockCustomField.objects.get_or_create.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_existing_field_not_recreated( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When custom field already exists, it is not recreated but types are checked.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + _ensure_librenms_id_custom_field(sender=None) + + # All pks already present, no types should be added + mock_cf.object_types.add.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_adds_missing_content_types( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When some content types are missing, only those are added.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + ct_existing = MagicMock() + ct_existing.pk = 1 + ct_new = MagicMock() + ct_new.pk = 99 + MockContentType.objects.get_for_model.side_effect = [ct_existing, ct_existing, ct_new, ct_new] + + _ensure_librenms_id_custom_field(sender=None) + + assert mock_cf.object_types.add.call_count == 2 + mock_cf.object_types.add.assert_any_call(ct_new) + + @patch("extras.models.CustomField") + def test_exception_does_not_propagate(self, MockCustomField): + """Exceptions during custom field creation are caught and logged.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + MockCustomField.objects.get_or_create.side_effect = Exception("DB not ready") + + with patch("logging.getLogger") as mock_get_logger: + # Should not raise + _ensure_librenms_id_custom_field(sender=None) + + # Verify the exception was logged + logger_instance = mock_get_logger.return_value + logger_instance.exception.assert_called_once() + call_args = logger_instance.exception.call_args + assert "librenms_id" in call_args[0][0] + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_no_log_when_field_already_exists( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """No log message when the custom field already existed.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + # When the field already exists (created=False), the info log should + # not be emitted. We verify via the logger instance rather than + # asserting getLogger was never called, which is fragile. + logger_instance = mock_get_logger.return_value + logger_instance.info.assert_not_called() diff --git a/netbox_librenms_plugin/tests/test_librenms_api.py b/netbox_librenms_plugin/tests/test_librenms_api.py index 5ce115f340..2db833dd1c 100644 --- a/netbox_librenms_plugin/tests/test_librenms_api.py +++ b/netbox_librenms_plugin/tests/test_librenms_api.py @@ -643,7 +643,7 @@ def test_add_device_snmpv1_success(self, mock_post, mock_librenms_config): @patch("netbox_librenms_plugin.librenms_api.requests.post") def test_add_device_duplicate_error(self, mock_post, mock_librenms_config): """Verify duplicate device handling.""" - mock_post.return_value.status_code = 500 + mock_post.return_value.status_code = 200 mock_post.return_value.json.return_value = { "status": "error", "message": "Device already exists", diff --git a/netbox_librenms_plugin/tests/test_permissions.py b/netbox_librenms_plugin/tests/test_permissions.py index d366965ead..50bc2b6d04 100644 --- a/netbox_librenms_plugin/tests/test_permissions.py +++ b/netbox_librenms_plugin/tests/test_permissions.py @@ -633,8 +633,8 @@ class TestView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin): class TestBulkImportPermissions: """Tests for permission checks in bulk import functions.""" - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_checks_permissions(self, mock_api_class, mock_require): """bulk_import_devices_shared calls require_permissions.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -658,8 +658,8 @@ def test_bulk_import_devices_checks_permissions(self, mock_api_class, mock_requi assert "dcim.add_device" in call_args[0][1] assert "dcim.add_interface" in call_args[0][1] - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_extracts_user_from_job(self, mock_api_class, mock_require): """bulk_import_devices_shared extracts user from job if not provided.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -682,7 +682,7 @@ def test_bulk_import_devices_extracts_user_from_job(self, mock_api_class, mock_r call_args = mock_require.call_args assert job_user == call_args[0][0] - @patch("netbox_librenms_plugin.import_utils.require_permissions") + @patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions") def test_bulk_import_vms_checks_permissions(self, mock_require): """bulk_import_vms calls require_permissions.""" from netbox_librenms_plugin.import_utils import bulk_import_vms @@ -703,7 +703,7 @@ def test_bulk_import_vms_checks_permissions(self, mock_require): assert user == call_args[0][0] assert "virtualization.add_virtualmachine" in call_args[0][1] - @patch("netbox_librenms_plugin.import_utils.require_permissions") + @patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions") def test_bulk_import_vms_extracts_user_from_job(self, mock_require): """bulk_import_vms extracts user from job if not provided.""" from netbox_librenms_plugin.import_utils import bulk_import_vms @@ -729,7 +729,7 @@ def test_bulk_import_vms_extracts_user_from_job(self, mock_require): class TestBulkImportPermissionDenied: """Tests for permission denied behavior in bulk import.""" - @patch("netbox_librenms_plugin.import_utils.check_user_permissions") + @patch("netbox_librenms_plugin.import_utils.permissions.check_user_permissions") def test_bulk_import_devices_raises_on_missing_permissions(self, mock_check): """bulk_import_devices_shared raises PermissionDenied when permissions missing.""" import pytest @@ -748,7 +748,7 @@ def test_bulk_import_devices_raises_on_missing_permissions(self, mock_check): server_key="default", ) - @patch("netbox_librenms_plugin.import_utils.check_user_permissions") + @patch("netbox_librenms_plugin.import_utils.permissions.check_user_permissions") def test_bulk_import_vms_raises_on_missing_permissions(self, mock_check): """bulk_import_vms raises PermissionDenied when permissions missing.""" import pytest @@ -873,8 +873,8 @@ def test_htmx_rejects_external_referrer(self): class TestBulkImportVCPermission: """Tests that bulk import checks virtualchassis permission.""" - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_checks_vc_permission(self, mock_api_class, mock_require): """bulk_import_devices_shared includes dcim.add_virtualchassis in required perms.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared diff --git a/netbox_librenms_plugin/tests/test_utils.py b/netbox_librenms_plugin/tests/test_utils.py index 96065ab760..d98ca28b9c 100644 --- a/netbox_librenms_plugin/tests/test_utils.py +++ b/netbox_librenms_plugin/tests/test_utils.py @@ -15,9 +15,12 @@ class TestDeviceTypeMatching: """Test device type matching logic.""" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_exact_match_by_part_number(self, mock_device_type): + def test_match_device_type_exact_match_by_part_number(self, mock_device_type, mock_mapping): """Exact part_number string should match.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_dt = MagicMock(id=1, model="C9300-48P") mock_device_type.objects.get.return_value = mock_dt @@ -29,9 +32,12 @@ def test_match_device_type_exact_match_by_part_number(self, mock_device_type): assert result["device_type"] == mock_dt assert result["match_type"] == "exact" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_exact_match_by_model(self, mock_device_type): + def test_match_device_type_exact_match_by_model(self, mock_device_type, mock_mapping): """Exact model string should match when part_number fails.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_dt = MagicMock(id=1, model="WS-C3750X-48P") # Part number lookup fails, model lookup succeeds mock_device_type.DoesNotExist = Exception @@ -48,9 +54,12 @@ def test_match_device_type_exact_match_by_model(self, mock_device_type): assert result["device_type"] == mock_dt assert result["match_type"] == "exact" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_not_found(self, mock_device_type): + def test_match_device_type_not_found(self, mock_device_type, mock_mapping): """Returns None when no match found.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_device_type.DoesNotExist = Exception mock_device_type.objects.get.side_effect = mock_device_type.DoesNotExist @@ -62,6 +71,22 @@ def test_match_device_type_not_found(self, mock_device_type): assert result["device_type"] is None assert result["match_type"] is None + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") + def test_match_device_type_mapping_match(self, mock_mapping): + """DeviceTypeMapping entry should be used before part_number/model fallback.""" + mock_dt = MagicMock(id=1, model="MX480") + mock_mapping_obj = MagicMock(netbox_device_type=mock_dt) + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.return_value = mock_mapping_obj + + from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type + + result = match_librenms_hardware_to_device_type("Juniper MX480 Internet Backbone Router") + + assert result["matched"] is True + assert result["device_type"] == mock_dt + assert result["match_type"] == "mapping" + def test_match_device_type_empty_hardware(self): """Empty string returns None.""" from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type diff --git a/netbox_librenms_plugin/urls.py b/netbox_librenms_plugin/urls.py index af12187c6f..2270905288 100644 --- a/netbox_librenms_plugin/urls.py +++ b/netbox_librenms_plugin/urls.py @@ -1,6 +1,6 @@ from django.urls import include, path -from .models import InterfaceTypeMapping +from .models import DeviceTypeMapping, InterfaceTypeMapping, ModuleBayMapping, ModuleTypeMapping, NormalizationRule from .views import ( AddDeviceToLibreNMSView, AssignVCSerialView, @@ -10,15 +10,27 @@ DeleteNetBoxInterfacesView, DeviceCableTableView, DeviceClusterUpdateView, + DeviceConflictActionView, DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceRackUpdateView, DeviceRoleUpdateView, DeviceStatusListView, + DeviceTypeMappingBulkDeleteView, + DeviceTypeMappingBulkImportView, + DeviceTypeMappingChangeLogView, + DeviceTypeMappingCreateView, + DeviceTypeMappingDeleteView, + DeviceTypeMappingEditView, + DeviceTypeMappingListView, + DeviceTypeMappingView, DeviceValidationDetailsView, DeviceVCDetailsView, DeviceVLANTableView, + InstallBranchView, + InstallModuleView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -29,6 +41,30 @@ InterfaceTypeMappingView, LibreNMSImportView, LibreNMSSettingsView, + ModuleBayMappingBulkDeleteView, + ModuleBayMappingBulkImportView, + ModuleBayMappingChangeLogView, + ModuleBayMappingCreateView, + ModuleBayMappingDeleteView, + ModuleBayMappingEditView, + ModuleBayMappingListView, + ModuleBayMappingView, + ModuleTypeMappingBulkDeleteView, + ModuleTypeMappingBulkImportView, + ModuleTypeMappingChangeLogView, + ModuleTypeMappingCreateView, + ModuleTypeMappingDeleteView, + ModuleTypeMappingEditView, + ModuleTypeMappingListView, + ModuleTypeMappingView, + NormalizationRuleBulkDeleteView, + NormalizationRuleBulkImportView, + NormalizationRuleChangeLogView, + NormalizationRuleCreateView, + NormalizationRuleDeleteView, + NormalizationRuleEditView, + NormalizationRuleListView, + NormalizationRuleView, SaveUserPrefView, SingleCableVerifyView, SingleInterfaceVerifyView, @@ -43,6 +79,7 @@ SyncVLANsView, TestLibreNMSConnectionView, UpdateDeviceLocationView, + UpdateDeviceNameView, UpdateDevicePlatformView, UpdateDeviceSerialView, UpdateDeviceTypeView, @@ -69,6 +106,21 @@ DeviceCableTableView.as_view(), name="device_cable_sync", ), + path( + "devices//module-sync/", + DeviceModuleTableView.as_view(), + name="device_module_sync", + ), + path( + "devices//install-module/", + InstallModuleView.as_view(), + name="install_module", + ), + path( + "devices//install-branch/", + InstallBranchView.as_view(), + name="install_branch", + ), path( "devices//ipaddress-sync/", DeviceIPAddressTableView.as_view(), @@ -188,7 +240,12 @@ UpdateDeviceLocationView.as_view(), name="update_device_location", ), - # Update device field URLs (serial, device type, platform) + # Update device field URLs (name, serial, device type, platform) + path( + "devices//update-name/", + UpdateDeviceNameView.as_view(), + name="update_device_name", + ), path( "devices//update-serial/", UpdateDeviceSerialView.as_view(), @@ -259,6 +316,11 @@ DeviceRackUpdateView.as_view(), name="device_rack_update", ), + path( + "device-import/conflict-action//", + DeviceConflictActionView.as_view(), + name="device_conflict_action", + ), path( "save-user-pref/", SaveUserPrefView.as_view(), @@ -323,5 +385,173 @@ InterfaceTypeMappingBulkDeleteView.as_view(), name="interfacetypemapping_bulk_delete", ), + # Device type mapping URLs + path( + "device-type-mappings/", + DeviceTypeMappingListView.as_view(), + name="devicetypemapping_list", + ), + path( + "device-type-mappings//", + DeviceTypeMappingView.as_view(), + name="devicetypemapping_detail", + ), + path( + "device-type-mappings/add/", + DeviceTypeMappingCreateView.as_view(), + name="devicetypemapping_add", + ), + path( + "device-type-mappings/import/", + DeviceTypeMappingBulkImportView.as_view(), + name="devicetypemapping_bulk_import", + ), + path( + "device-type-mappings//delete/", + DeviceTypeMappingDeleteView.as_view(), + name="devicetypemapping_delete", + ), + path( + "device-type-mappings//edit/", + DeviceTypeMappingEditView.as_view(), + name="devicetypemapping_edit", + ), + path( + "device-type-mappings//changelog/", + DeviceTypeMappingChangeLogView.as_view(), + name="devicetypemapping_changelog", + kwargs={"model": DeviceTypeMapping}, + ), + path( + "device-type-mappings/delete/", + DeviceTypeMappingBulkDeleteView.as_view(), + name="devicetypemapping_bulk_delete", + ), + # Module type mapping URLs + path( + "module-type-mappings/", + ModuleTypeMappingListView.as_view(), + name="moduletypemapping_list", + ), + path( + "module-type-mappings//", + ModuleTypeMappingView.as_view(), + name="moduletypemapping_detail", + ), + path( + "module-type-mappings/add/", + ModuleTypeMappingCreateView.as_view(), + name="moduletypemapping_add", + ), + path( + "module-type-mappings/import/", + ModuleTypeMappingBulkImportView.as_view(), + name="moduletypemapping_bulk_import", + ), + path( + "module-type-mappings//delete/", + ModuleTypeMappingDeleteView.as_view(), + name="moduletypemapping_delete", + ), + path( + "module-type-mappings//edit/", + ModuleTypeMappingEditView.as_view(), + name="moduletypemapping_edit", + ), + path( + "module-type-mappings//changelog/", + ModuleTypeMappingChangeLogView.as_view(), + name="moduletypemapping_changelog", + kwargs={"model": ModuleTypeMapping}, + ), + path( + "module-type-mappings/delete/", + ModuleTypeMappingBulkDeleteView.as_view(), + name="moduletypemapping_bulk_delete", + ), + # Module Bay Mapping URLs + path( + "module-bay-mappings/", + ModuleBayMappingListView.as_view(), + name="modulebaymapping_list", + ), + path( + "module-bay-mappings//", + ModuleBayMappingView.as_view(), + name="modulebaymapping_detail", + ), + path( + "module-bay-mappings/add/", + ModuleBayMappingCreateView.as_view(), + name="modulebaymapping_add", + ), + path( + "module-bay-mappings/import/", + ModuleBayMappingBulkImportView.as_view(), + name="modulebaymapping_bulk_import", + ), + path( + "module-bay-mappings//delete/", + ModuleBayMappingDeleteView.as_view(), + name="modulebaymapping_delete", + ), + path( + "module-bay-mappings//edit/", + ModuleBayMappingEditView.as_view(), + name="modulebaymapping_edit", + ), + path( + "module-bay-mappings//changelog/", + ModuleBayMappingChangeLogView.as_view(), + name="modulebaymapping_changelog", + kwargs={"model": ModuleBayMapping}, + ), + path( + "module-bay-mappings/delete/", + ModuleBayMappingBulkDeleteView.as_view(), + name="modulebaymapping_bulk_delete", + ), + # Normalization Rule URLs + path( + "normalization-rules/", + NormalizationRuleListView.as_view(), + name="normalizationrule_list", + ), + path( + "normalization-rules//", + NormalizationRuleView.as_view(), + name="normalizationrule_detail", + ), + path( + "normalization-rules/add/", + NormalizationRuleCreateView.as_view(), + name="normalizationrule_add", + ), + path( + "normalization-rules/import/", + NormalizationRuleBulkImportView.as_view(), + name="normalizationrule_bulk_import", + ), + path( + "normalization-rules//delete/", + NormalizationRuleDeleteView.as_view(), + name="normalizationrule_delete", + ), + path( + "normalization-rules//edit/", + NormalizationRuleEditView.as_view(), + name="normalizationrule_edit", + ), + path( + "normalization-rules//changelog/", + NormalizationRuleChangeLogView.as_view(), + name="normalizationrule_changelog", + kwargs={"model": NormalizationRule}, + ), + path( + "normalization-rules/delete/", + NormalizationRuleBulkDeleteView.as_view(), + name="normalizationrule_bulk_delete", + ), path("api/", include("netbox_librenms_plugin.api.urls")), ] diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index 4a5bf113a4..d1ac116ce0 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -1,3 +1,4 @@ +import logging import re from typing import Optional @@ -8,6 +9,8 @@ from netbox.plugins import get_plugin_config from utilities.paginator import get_paginate_count as netbox_get_paginate_count +logger = logging.getLogger(__name__) + def convert_speed_to_kbps(speed_bps: int) -> int: """ @@ -193,7 +196,8 @@ def match_librenms_hardware_to_device_type(hardware_name: str) -> dict: """ Match LibreNMS hardware string to a NetBox DeviceType. - Only performs exact matching on part_number and model fields (case-insensitive). + Checks DeviceTypeMapping table first, then falls back to exact matching + on part_number and model fields (case-insensitive). Args: hardware_name (str): Hardware string from LibreNMS API (e.g., 'C9200L-48P-4X') @@ -202,13 +206,29 @@ def match_librenms_hardware_to_device_type(hardware_name: str) -> dict: dict: Dictionary containing: - matched (bool): Whether a match was found - device_type (DeviceType|None): The matched DeviceType object - - match_type (str|None): Always 'exact' if found, None otherwise + - match_type (str|None): 'mapping' if via DeviceTypeMapping, 'exact' if via + part_number/model, None otherwise """ from dcim.models import DeviceType + from netbox_librenms_plugin.models import DeviceTypeMapping + if not hardware_name or hardware_name == "-": return {"matched": False, "device_type": None, "match_type": None} + # Check DeviceTypeMapping table first + try: + mapping = DeviceTypeMapping.objects.get(librenms_hardware__iexact=hardware_name) + return { + "matched": True, + "device_type": mapping.netbox_device_type, + "match_type": "mapping", + } + except DeviceTypeMapping.DoesNotExist: + pass + except DeviceTypeMapping.MultipleObjectsReturned: + pass + # Try part number exact match try: device_type = DeviceType.objects.get(part_number__iexact=hardware_name) @@ -447,3 +467,109 @@ def check_vlan_group_matches( netbox_gid = netbox_tagged_group_ids.get(vid) return netbox_gid == selected_group_id return True + + +# Minimum NetBox version that supports {module_path} token in module templates + + +def supports_module_path(): + """Check if the running NetBox supports the {module_path} template token. + + Detects by checking for MODULE_PATH_TOKEN in dcim.constants rather than + comparing version strings β€” works with patched/pre-release builds too. + """ + try: + from dcim.constants import MODULE_PATH_TOKEN # noqa: F401 + + return True + except ImportError: + return False + + +def module_type_uses_module_path(module_type): + """Check if a ModuleType has any interface templates using {module_path}.""" + return any("{module_path}" in t.name for t in module_type.interfacetemplates.all()) + + +def has_nested_name_conflict(module_type, module_bay): + """Check if installing this module type in a nested bay would cause a name conflict. + + Returns True when ALL of the following are true: + - The module type has interface templates using only ``{module}`` (not ``{module_path}``) + - The bay is nested (its parent is owned by an installed module) + - There is at least one sibling bay under the same parent + + In this situation NetBox's ``resolve_name()`` replaces ``{module}`` with the + root ancestor's bay position, producing the same interface name for every + sibling at this nesting level. + """ + from dcim.constants import MODULE_TOKEN + + if not module_bay or not module_bay.module_id: + return False # Top-level bay β€” no conflict + + templates = list(module_type.interfacetemplates.all()) + if not templates: + return False # No interface templates + + uses_module_token = any(MODULE_TOKEN in t.name for t in templates) + if not uses_module_token: + return False # Template doesn't use {module} + + # Count how many unique interface names this template would produce across siblings + # If all siblings resolve to the same name, there's a conflict + from dcim.models import ModuleBay as ModuleBayModel + + sibling_count = ModuleBayModel.objects.filter( + device=module_bay.device, + module_id=module_bay.module_id, + ).count() + + return sibling_count > 1 + + +def apply_normalization_rules(value: str, scope: str, manufacturer=None) -> str: + """Apply NormalizationRule chain to transform a string before matching. + + Rules for the given scope are applied in priority order. Each rule's + regex substitution transforms the output of the previous rule, forming + a pipeline. If no rules match, the original value is returned unchanged. + + When *manufacturer* is given, manufacturer-scoped rules run first, + followed by unscoped (manufacturer=NULL) rules. When *manufacturer* + is ``None``, all rules for the scope run in priority order. + + Args: + value: The raw string to normalize (e.g. '3HE16474AARA01'). + scope: One of NormalizationRule.SCOPE_* constants. + manufacturer: Optional Manufacturer instance to scope rules. + + Returns: + The normalized string after all matching rules have been applied. + """ + from netbox_librenms_plugin.models import NormalizationRule + + if not value: + return value + + if manufacturer: + # Manufacturer-specific rules first, then unscoped rules + for mfg_filter in [{"manufacturer": manufacturer}, {"manufacturer__isnull": True}]: + rules = NormalizationRule.objects.filter(scope=scope, **mfg_filter).order_by("priority", "pk") + for rule in rules: + try: + value = re.sub(rule.match_pattern, rule.replacement, value) + except re.error: + logger.error( + "Invalid regex in NormalizationRule pk=%s pattern=%r β€” skipping", rule.pk, rule.match_pattern + ) + else: + rules = NormalizationRule.objects.filter(scope=scope).order_by("priority", "pk") + for rule in rules: + try: + value = re.sub(rule.match_pattern, rule.replacement, value) + except re.error: + logger.error( + "Invalid regex in NormalizationRule pk=%s pattern=%r β€” skipping", rule.pk, rule.match_pattern + ) + return value diff --git a/netbox_librenms_plugin/views/__init__.py b/netbox_librenms_plugin/views/__init__.py index d2b3bbdd43..e3b4348ba7 100644 --- a/netbox_librenms_plugin/views/__init__.py +++ b/netbox_librenms_plugin/views/__init__.py @@ -2,15 +2,18 @@ Module for initializing views for the NetBox LibreNMS plugin. """ -from .base.cables_view import BaseCableTableView, SingleCableVerifyView -from .base.interfaces_view import BaseInterfaceTableView -from .base.ip_addresses_view import BaseIPAddressTableView, SingleIPAddressVerifyView -from .base.librenms_sync_view import BaseLibreNMSSyncView -from .base.vlan_table_view import BaseVLANTableView -from .imports import ( +# These are intentional re-exports for consumers of this package. # noqa: F401 +from .base.cables_view import BaseCableTableView, SingleCableVerifyView # noqa: F401 +from .base.interfaces_view import BaseInterfaceTableView # noqa: F401 +from .base.ip_addresses_view import BaseIPAddressTableView, SingleIPAddressVerifyView # noqa: F401 +from .base.librenms_sync_view import BaseLibreNMSSyncView # noqa: F401 +from .base.modules_view import InstallBranchView, InstallModuleView # noqa: F401 +from .base.vlan_table_view import BaseVLANTableView # noqa: F401 +from .imports import ( # noqa: F401 BulkImportConfirmView, BulkImportDevicesView, DeviceClusterUpdateView, + DeviceConflictActionView, DeviceRackUpdateView, DeviceRoleUpdateView, DeviceValidationDetailsView, @@ -18,7 +21,15 @@ LibreNMSImportView, SaveUserPrefView, ) -from .mapping_views import ( +from .mapping_views import ( # noqa: F401 + DeviceTypeMappingBulkDeleteView, + DeviceTypeMappingBulkImportView, + DeviceTypeMappingChangeLogView, + DeviceTypeMappingCreateView, + DeviceTypeMappingDeleteView, + DeviceTypeMappingEditView, + DeviceTypeMappingListView, + DeviceTypeMappingView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -27,12 +38,37 @@ InterfaceTypeMappingEditView, InterfaceTypeMappingListView, InterfaceTypeMappingView, + ModuleBayMappingBulkDeleteView, + ModuleBayMappingBulkImportView, + ModuleBayMappingChangeLogView, + ModuleBayMappingCreateView, + ModuleBayMappingDeleteView, + ModuleBayMappingEditView, + ModuleBayMappingListView, + ModuleBayMappingView, + ModuleTypeMappingBulkDeleteView, + ModuleTypeMappingBulkImportView, + ModuleTypeMappingChangeLogView, + ModuleTypeMappingCreateView, + ModuleTypeMappingDeleteView, + ModuleTypeMappingEditView, + ModuleTypeMappingListView, + ModuleTypeMappingView, + NormalizationRuleBulkDeleteView, + NormalizationRuleBulkImportView, + NormalizationRuleChangeLogView, + NormalizationRuleCreateView, + NormalizationRuleDeleteView, + NormalizationRuleEditView, + NormalizationRuleListView, + NormalizationRuleView, ) -from .object_sync import ( +from .object_sync import ( # noqa: F401 DeviceCableTableView, DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceVLANTableView, SaveVlanGroupOverridesView, SingleInterfaceVerifyView, @@ -42,18 +78,19 @@ VMIPAddressTableView, VMLibreNMSSyncView, ) -from .settings_views import LibreNMSSettingsView, TestLibreNMSConnectionView -from .status_check import DeviceStatusListView, VMStatusListView -from .sync.cables import SyncCablesView -from .sync.device_fields import ( +from .settings_views import LibreNMSSettingsView, TestLibreNMSConnectionView # noqa: F401 +from .status_check import DeviceStatusListView, VMStatusListView # noqa: F401 +from .sync.cables import SyncCablesView # noqa: F401 +from .sync.device_fields import ( # noqa: F401 AssignVCSerialView, CreateAndAssignPlatformView, + UpdateDeviceNameView, UpdateDevicePlatformView, UpdateDeviceSerialView, UpdateDeviceTypeView, ) -from .sync.devices import AddDeviceToLibreNMSView, UpdateDeviceLocationView -from .sync.interfaces import DeleteNetBoxInterfacesView, SyncInterfacesView -from .sync.ip_addresses import SyncIPAddressesView -from .sync.locations import SyncSiteLocationView -from .sync.vlans import SyncVLANsView +from .sync.devices import AddDeviceToLibreNMSView, UpdateDeviceLocationView # noqa: F401 +from .sync.interfaces import DeleteNetBoxInterfacesView, SyncInterfacesView # noqa: F401 +from .sync.ip_addresses import SyncIPAddressesView # noqa: F401 +from .sync.locations import SyncSiteLocationView # noqa: F401 +from .sync.vlans import SyncVLANsView # noqa: F401 diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index c390cdd539..47ab62ab42 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -24,7 +24,6 @@ class BaseCableTableView(LibreNMSPermissionMixin, LibreNMSAPIMixin, CacheMixin, model = None # To be defined in subclasses partial_template_name = "netbox_librenms_plugin/_cable_sync_content.html" - interface_name_field = get_interface_name_field() def get_object(self, pk): """Retrieve the object (Device or VirtualMachine).""" @@ -53,11 +52,17 @@ def get_links_data(self, obj): if not success or "error" in data: return None + interface_name_field = get_interface_name_field(getattr(self, "request", None)) ports_data = self.get_ports_data(obj) local_ports_map = {} for port in ports_data.get("ports", []): - port_id = str(port["port_id"]) - port_name = port[self.interface_name_field] + raw_port_id = port.get("port_id") + if raw_port_id is None: + continue + port_id = str(raw_port_id) + port_name = port.get(interface_name_field) + if port_name is None: + continue local_ports_map[port_id] = port_name links = data.get("links", []) diff --git a/netbox_librenms_plugin/views/base/librenms_sync_view.py b/netbox_librenms_plugin/views/base/librenms_sync_view.py index 1346c3cb13..4a85cb1642 100644 --- a/netbox_librenms_plugin/views/base/librenms_sync_view.py +++ b/netbox_librenms_plugin/views/base/librenms_sync_view.py @@ -86,6 +86,7 @@ def get_context_data(self, request, obj): cable_context = self.get_cable_context(request, obj) ip_context = self.get_ip_context(request, obj) vlan_context = self.get_vlan_context(request, obj) + module_context = self.get_module_context(request, obj) interface_name_field = get_interface_name_field(request) @@ -103,6 +104,7 @@ def get_context_data(self, request, obj): "cable_sync": cable_context, "ip_sync": ip_context, "vlan_sync": vlan_context, + "module_sync": module_context, "v1v2form": AddToLIbreSNMPV1V2(prefix="v1v2"), "v3form": AddToLIbreSNMPV3(prefix="v3"), "librenms_device_id": self.librenms_id, @@ -230,6 +232,8 @@ def get_librenms_device_info(self, obj): if netbox_identities & librenms_identities: mismatched_device = False else: + # Device is still found (we have librenms_id), just mismatched + found_in_librenms = True mismatched_device = True librenms_device_details["netbox_dns_name"] = netbox_dns_name or "-" @@ -268,6 +272,13 @@ def get_vlan_context(self, request, obj): """ return None + def get_module_context(self, request, obj): + """ + Get the context data for module sync. + Subclasses should override this method if applicable. + """ + return None + @staticmethod def _strip_vc_pattern(name): """Strip the VC member naming suffix from a device name. diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py new file mode 100644 index 0000000000..0a7317a51c --- /dev/null +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -0,0 +1,1064 @@ +from django.contrib import messages +from django.core.cache import cache +from django.db import transaction +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.utils import timezone +from django.views import View + +from netbox_librenms_plugin.views.mixins import ( + CacheMixin, + LibreNMSAPIMixin, + LibreNMSPermissionMixin, + NetBoxObjectPermissionMixin, +) + + +# entPhysicalClass values relevant for module sync +# Includes vendor-specific classes (Nokia TIMETRA-CHASSIS-MIB uses ioModule, cpmModule, etc.) +INVENTORY_CLASSES = { + "module", + "powerSupply", + "fan", + "port", + "container", + "ioModule", + "cpmModule", + "mdaModule", + "fabricModule", + "xioModule", +} + +# Model name values that indicate a generic/empty container (not real hardware) +_GENERIC_CONTAINER_MODELS = {"", "BUILTIN", "Default", "N/A"} + + +class BaseModuleTableView(LibreNMSPermissionMixin, LibreNMSAPIMixin, CacheMixin, View): + """ + Base view for synchronizing module/inventory data from LibreNMS. + Fetches inventory, matches against NetBox module bays and module types, + and renders a comparison table. + """ + + model = None + partial_template_name = "netbox_librenms_plugin/_module_sync_content.html" + + def get_object(self, pk): + """Retrieve the object (Device).""" + return get_object_or_404(self.model, pk=pk) + + def get_table(self, data, obj): + """Returns the table class. Subclasses should override.""" + raise NotImplementedError("Subclasses must implement get_table()") + + def post(self, request, pk): + """Fetch inventory from LibreNMS, cache it, and render the module sync table.""" + obj = self.get_object(pk) + + self.librenms_id = self.librenms_api.get_librenms_id(obj) + if not self.librenms_id: + messages.error(request, "Device not found in LibreNMS.") + return render( + request, + self.partial_template_name, + {"module_sync": {"object": obj, "table": None, "cache_expiry": None}}, + ) + + success, inventory_data = self.librenms_api.get_device_inventory(self.librenms_id) + + if not success: + messages.error(request, f"Failed to fetch inventory from LibreNMS: {inventory_data}") + return render( + request, + self.partial_template_name, + {"module_sync": {"object": obj, "table": None, "cache_expiry": None}}, + ) + + # Fetch transceiver data and merge with inventory + inventory_data = self._merge_transceiver_data(inventory_data) + + # Cache the merged inventory data + cache.set( + self.get_cache_key(obj, "inventory"), + inventory_data, + timeout=self.librenms_api.cache_timeout, + ) + + context = self._build_context(request, obj, inventory_data) + messages.success(request, "Inventory data refreshed successfully.") + return render(request, self.partial_template_name, {"module_sync": context}) + + def get_context_data(self, request, obj): + """Get context from cache (used by the main sync view on initial page load).""" + cached_data = cache.get(self.get_cache_key(obj, "inventory")) + if not cached_data: + return {"table": None, "object": obj, "cache_expiry": None} + return self._build_context(request, obj, cached_data) + + def _build_context(self, request, obj, inventory_data): + """Build context with matched inventory items and table.""" + # Build a lookup of all inventory items by index for parent resolution + index_map = {item["entPhysicalIndex"]: item for item in inventory_data} + + # Store manufacturer for normalization rules in _build_row + self._device_manufacturer = getattr(getattr(obj, "device_type", None), "manufacturer", None) + + # Get NetBox module bays and modules for this device + device_bays, module_scoped_bays = self._get_module_bays(obj) + module_types = self._get_module_types() + + # Collect top-level items and their sub-components + # Include synthetic transceiver items (from vendors without ENTITY-MIB SFP data) + # Exclude items that have any ancestor with an INVENTORY_CLASSES class + # (they appear as sub-components under that ancestor) + top_items = [] + for item in inventory_data: + if item.get("_from_transceiver_api"): + top_items.append(item) + continue + phys_class = item.get("entPhysicalClass") + if phys_class not in INVENTORY_CLASSES: + continue + # Skip items with generic model names (not real hardware). + # Containers with empty model are physical slot representations. + model = (item.get("entPhysicalModelName") or "").strip() + if phys_class == "container" and model in _GENERIC_CONTAINER_MODELS: + continue + if model and model in _GENERIC_CONTAINER_MODELS: + continue + # Walk up ancestor chain; skip if any ancestor is an inventory-class item. + # Containers with empty model are physical slot/bay representations, not + # real modules β€” skip them so children can be top-level items. + is_descendant = False + current_idx = item.get("entPhysicalContainedIn", 0) + for _ in range(10): + if not current_idx or current_idx not in index_map: + break + ancestor = index_map[current_idx] + anc_class = ancestor.get("entPhysicalClass") + if anc_class in INVENTORY_CLASSES: + anc_model = (ancestor.get("entPhysicalModelName") or "").strip() + # Empty-model containers are just physical slot representations + if anc_class == "container" and not anc_model: + current_idx = ancestor.get("entPhysicalContainedIn", 0) + continue + is_descendant = True + break + current_idx = ancestor.get("entPhysicalContainedIn", 0) + if is_descendant: + continue + top_items.append(item) + + table_data = [] + from netbox_librenms_plugin.utils import apply_normalization_rules + + # Build combined bay lookup so top-level items (including synthetic + # transceiver entries) can match bays created by installed modules. + all_bays = dict(device_bays) + for scope_bays in module_scoped_bays.values(): + all_bays.update(scope_bays) + + for item in top_items: + row = self._build_row(item, index_map, all_bays, module_types, depth=0) + parent_idx = len(table_data) + table_data.append(row) + + # Determine which bays sub-components should match against: + # If parent matched a bay with an installed module, use that module's child bays. + # If parent matched a bay but it's NOT installed, children can't be installed + # individually (parent must be installed first to create child bays). + parent_module_id = None + parent_bay_matched_but_uninstalled = False + if row.get("module_bay_id"): + matched_bay = all_bays.get(row["module_bay"]) + if matched_bay and hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + parent_module_id = matched_bay.installed_module.pk + else: + # Parent matched a bay but it's not installed yet + parent_bay_matched_but_uninstalled = True + + if parent_bay_matched_but_uninstalled: + # Empty dict: children can't match any bay individually + child_bays = {} + elif parent_module_id: + child_bays = module_scoped_bays.get(parent_module_id, {}) + else: + child_bays = device_bays + + # Find sub-components with a model name (transceivers, converters, etc.) + # Track bay scope per depth level so nested modules use correct bays + bays_by_depth = {0: child_bays} + sub_items = self._get_sub_components(item["entPhysicalIndex"], inventory_data) + for depth, sub_item in sub_items: + scope_bays = bays_by_depth.get(depth, child_bays) + sub_row = self._build_row(sub_item, index_map, scope_bays, module_types, depth=depth) + table_data.append(sub_row) + + # If this sub-item matched an installed module, deeper items use its bays + if sub_row.get("module_bay_id"): + matched_sub_bay = scope_bays.get(sub_row["module_bay"]) + if ( + matched_sub_bay + and hasattr(matched_sub_bay, "installed_module") + and matched_sub_bay.installed_module + ): + sub_module_id = matched_sub_bay.installed_module.pk + bays_by_depth[depth + 1] = module_scoped_bays.get(sub_module_id, {}) + + # Mark parent if any child is installable + if sub_row.get("can_install"): + table_data[parent_idx]["has_installable_children"] = True + + # When parent is installable but children can't match bays yet + # (parent module not installed), enable "Install Branch" if children + # have matching module types (branch install handles bay creation) + if ( + parent_bay_matched_but_uninstalled + and row.get("can_install") + and not table_data[parent_idx].get("has_installable_children") + ): + for _depth, sub_item in sub_items: + sub_model = (sub_item.get("entPhysicalModelName") or "").strip() + if sub_model and ( + sub_model in module_types + or apply_normalization_rules( + sub_model, + "module_type", + manufacturer=getattr(self, "_device_manufacturer", None), + ) + in module_types + ): + table_data[parent_idx]["has_installable_children"] = True + break + + # Sort top-level groups by status, keeping children after their parent + table_data = self._sort_with_hierarchy(table_data) + + table = self.get_table(table_data, obj) + table.configure(request) + + cache_ttl = getattr(cache, "ttl", lambda k: None)(self.get_cache_key(obj, "inventory")) + cache_expiry = timezone.now() + timezone.timedelta(seconds=cache_ttl) if cache_ttl is not None else None + + return { + "table": table, + "object": obj, + "cache_expiry": cache_expiry, + } + + def _merge_transceiver_data(self, inventory_data): + """Merge transceiver API data with entity inventory. + + For vendors like Nokia that don't expose SFPs in ENTITY-MIB, + the transceiver API provides SFP model, serial, and type info. + + Strategy: + - For transceivers matching existing inventory items by entity_physical_index: + supplement entPhysicalModelName if empty + - For transceivers NOT in inventory: create synthetic inventory items + so they appear in the modules table + """ + success, transceivers = self.librenms_api.get_device_transceivers(self.librenms_id) + if not success or not transceivers: + return inventory_data + + # Build lookup of existing inventory items by index and serial + inv_by_index = {item["entPhysicalIndex"]: item for item in inventory_data} + inv_serials = { + (item.get("entPhysicalSerialNum") or "").strip() + for item in inventory_data + if (item.get("entPhysicalSerialNum") or "").strip() + } + + # Build port_id β†’ ifName lookup for better synthetic item naming + port_name_map = self._build_port_name_map(transceivers) + + # Types that are containers, not real transceiver modules + SKIP_TYPES = {"Port Container", "Port", ""} + + for txr in transceivers: + ent_idx = txr.get("entity_physical_index") + if not ent_idx: + continue + + model = (txr.get("model") or "").strip() + serial = (txr.get("serial") or "").strip() + txr_type = (txr.get("type") or "").strip() + + # Skip containers and entries with no useful data + if txr_type in SKIP_TYPES and not model and not serial: + continue + + # Use transceiver type as model fallback (e.g., "CFP2/QSFP28") + display_model = model or (txr_type if txr_type not in SKIP_TYPES else "") + + if ent_idx in inv_by_index: + # Supplement existing inventory item if model is missing + existing = inv_by_index[ent_idx] + if not (existing.get("entPhysicalModelName") or "").strip() and display_model: + existing["entPhysicalModelName"] = display_model + if not (existing.get("entPhysicalSerialNum") or "").strip() and serial: + existing["entPhysicalSerialNum"] = serial + else: + # Skip if serial already exists in ENTITY-MIB data (avoid duplicates) + if serial and serial in inv_serials: + continue + # Create synthetic inventory item for SFPs not in entity inventory + port_id = txr.get("port_id", 0) + ifname = port_name_map.get(port_id) + if ifname: + name = ifname + elif port_id: + name = f"Transceiver (port {port_id})" + else: + name = f"Transceiver {ent_idx}" + + synthetic = { + "entPhysicalIndex": ent_idx, + "entPhysicalName": name, + "entPhysicalClass": "port", + "entPhysicalModelName": display_model, + "entPhysicalSerialNum": serial, + "entPhysicalDescr": txr_type, + "entPhysicalContainedIn": 0, + "_from_transceiver_api": True, + } + inventory_data.append(synthetic) + + return inventory_data + + def _build_port_name_map(self, transceivers): + """Build port_id β†’ ifName mapping for transceiver ports. + + Fetches port data from LibreNMS to resolve port IDs to interface names, + enabling better bay matching for synthetic transceiver items (e.g., + Nokia 1/1/c1 instead of opaque port IDs). + """ + port_ids = {txr.get("port_id") for txr in transceivers if txr.get("port_id")} + if not port_ids: + return {} + + success, ports_data = self.librenms_api.get_ports(self.librenms_id) + if not success or not isinstance(ports_data, dict): + return {} + + return { + p["port_id"]: p["ifName"] + for p in ports_data.get("ports", []) + if p.get("port_id") in port_ids and p.get("ifName") + } + + def _get_sub_components(self, parent_idx, inventory_data): + """Find descendant items with a model name (real hardware, not empty containers). + + Returns list of (depth, item) tuples. + """ + results = [] + self._collect_descendants(parent_idx, inventory_data, depth=1, results=results) + return results + + def _collect_descendants(self, parent_idx, inventory_data, depth, results): + """Recursively collect descendant items that have a model name.""" + children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] + for child in children: + model = (child.get("entPhysicalModelName") or "").strip() + if model and model not in _GENERIC_CONTAINER_MODELS: + results.append((depth, child)) + # Continue looking for deeper components (e.g., SFPs inside converters) + self._collect_descendants(child["entPhysicalIndex"], inventory_data, depth + 1, results) + else: + # Skip generic/empty items, but check their children + self._collect_descendants(child["entPhysicalIndex"], inventory_data, depth, results) + + def _sort_with_hierarchy(self, table_data): + """Sort table keeping children grouped under their parent.""" + status_order = {"Installed": 0, "Serial Mismatch": 1, "Matched": 2, "No Type": 3, "No Bay": 4, "Unmatched": 5} + + # Group into top-level items with their children + groups = [] + current_group = None + for row in table_data: + if row.get("depth", 0) == 0: + current_group = {"parent": row, "children": []} + groups.append(current_group) + elif current_group is not None: + current_group["children"].append(row) + + # Sort groups by parent status + groups.sort(key=lambda g: status_order.get(g["parent"]["status"], 99)) + + # Flatten back + result = [] + for group in groups: + result.append(group["parent"]) + result.extend(group["children"]) + return result + + def _get_module_bays(self, obj): + """Get module bays for the device, organized by scope. + + Returns: + tuple: (device_bays, module_bays) where: + - device_bays: {name: bay} for device-level bays (module=None) + - module_bays: {module_id: {name: bay}} for bays created by installed modules + """ + from dcim.models import ModuleBay + + bays = ModuleBay.objects.filter(device=obj).select_related("installed_module__module_type") + device_bays = {} + module_scoped_bays = {} + for bay in bays: + if bay.module_id: + module_scoped_bays.setdefault(bay.module_id, {})[bay.name] = bay + else: + device_bays[bay.name] = bay + return device_bays, module_scoped_bays + + def _get_module_types(self): + """Get all module types, indexed by model (part_number), with ModuleTypeMapping checked first.""" + from dcim.models import ModuleType + + from netbox_librenms_plugin.models import ModuleTypeMapping + + # Build base lookup from NetBox module types + types = ModuleType.objects.all().select_related("manufacturer") + result = {} + for mt in types: + result[mt.model] = mt + if mt.part_number and mt.part_number != mt.model: + result[mt.part_number] = mt + + # Overlay with explicit ModuleTypeMapping entries (take priority) + for mapping in ModuleTypeMapping.objects.select_related("netbox_module_type__manufacturer"): + result[mapping.librenms_model] = mapping.netbox_module_type + + return result + + def _find_parent_container_name(self, item, index_map): + """Resolve the parent container name for an inventory item.""" + contained_in = item.get("entPhysicalContainedIn", 0) + if contained_in == 0: + return None + parent = index_map.get(contained_in) + if parent: + return parent.get("entPhysicalName", "") + return None + + def _match_module_bay(self, item, index_map, module_bays): + """ + Try to match an inventory item to a NetBox ModuleBay. + Checks ModuleBayMapping table first (exact then regex), then falls back + to exact parent name match, then positional matching. + """ + import re + + from netbox_librenms_plugin.models import ModuleBayMapping + + parent_name = self._find_parent_container_name(item, index_map) + item_name = item.get("entPhysicalName", "") + item_descr = item.get("entPhysicalDescr", "") + phys_class = item.get("entPhysicalClass", "") + + # Build candidate names: parent, item name, item description + candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + + # Check ModuleBayMapping table for each candidate (exact match) + for name in candidate_names: + filters = {"librenms_name": name, "is_regex": False} + if phys_class: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + if not mapping: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + else: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + + if mapping and mapping.netbox_bay_name in module_bays: + return module_bays[mapping.netbox_bay_name] + + # Regex pattern matching on all candidate names + for name in candidate_names: + bay = self._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + if bay and self._fpc_slot_matches(name, bay): + return bay + + # Fallback: exact match on candidate names against bay dict + for name in candidate_names: + if name in module_bays: + return module_bays[name] + + # Positional fallback: determine slot number from container sibling order + # Handles SFPs inside converters where containers are unnamed + bay = self._match_bay_by_position(item, index_map, module_bays) + if bay: + return bay + + return None + + @staticmethod + def _fpc_slot_matches(candidate_name, bay): + """Validate that a regex-matched bay's parent slot position is consistent with + a positional descriptor like 'Model @ FPC/pic/port'. + + Returns True if the descriptor has no FPC reference, or if the bay's parent + module slot position matches the FPC number in the descriptor. Prevents + orphaned top-level items (e.g. QSFP @ 1/1/1 when FPC1 is not installed) + from incorrectly matching bays belonging to a different FPC's module. + """ + import re as _re + + match = _re.search(r"@\s+(\d+)/", candidate_name) + if not match: + return True + expected_fpc = match.group(1) + module = getattr(bay, "module", None) + if not module: + return True + parent_bay = getattr(module, "module_bay", None) + if not parent_bay: + return True + return parent_bay.position == expected_fpc + + @staticmethod + def _lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping): + """Try regex ModuleBayMapping patterns against a name. + + Returns matched module bay or None. + """ + regex_filters = {"is_regex": True} + if phys_class: + regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class=phys_class)) + list( + ModuleBayMapping.objects.filter(**regex_filters, librenms_class="") + ) + else: + regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class="")) + + for mapping in regex_mappings: + try: + match = re.fullmatch(mapping.librenms_name, name) + except re.error: + continue + if match: + resolved_bay = match.expand(mapping.netbox_bay_name) + if resolved_bay in module_bays: + bay = module_bays[resolved_bay] + if BaseModuleTableView._fpc_slot_matches(name, bay): + return bay + return None + + @staticmethod + def _match_bay_by_position(item, index_map, module_bays): + """Match bay by item's positional order among container siblings. + + When an item is inside a container (no model), walk up to find the + nearest ancestor with a model, count which container slot the item + occupies, and match to the bay by number (e.g., SFP 1, SFP 2). + """ + # Walk up through modelless containers to find the parent with a model + current_idx = item.get("entPhysicalContainedIn", 0) + container_idx = None + for _ in range(5): + if not current_idx or current_idx not in index_map: + return None + ancestor = index_map[current_idx] + model = (ancestor.get("entPhysicalModelName") or "").strip() + if model: + # Found the parent with a model; container_idx is the intermediate container + break + container_idx = current_idx + current_idx = ancestor.get("entPhysicalContainedIn", 0) + else: + return None + + if not container_idx: + return None + + # Determine position: count siblings of the container under the parent + parent_with_model_idx = current_idx + siblings = sorted( + [i for i in index_map.values() if i.get("entPhysicalContainedIn") == parent_with_model_idx], + key=lambda x: x.get("entPhysicalParentRelPos", 0), + ) + slot_num = None + for i, sib in enumerate(siblings): + if sib["entPhysicalIndex"] == container_idx: + slot_num = i + 1 + break + + if slot_num is None: + return None + + # Try common bay naming patterns + for pattern in [f"SFP {slot_num}", f"Slot {slot_num}", f"Bay {slot_num}", f"Port {slot_num}"]: + if pattern in module_bays: + return module_bays[pattern] + + return None + + def _build_row(self, item, index_map, module_bays, module_types, depth=0): + """Build a single table row from a LibreNMS inventory item.""" + from netbox_librenms_plugin.utils import ( + apply_normalization_rules, + has_nested_name_conflict, + module_type_uses_module_path, + supports_module_path, + ) + + model_name = item.get("entPhysicalModelName", "") or "" + serial = item.get("entPhysicalSerialNum", "") or "" + phys_class = item.get("entPhysicalClass", "") + name = item.get("entPhysicalName", "") or "-" + description = item.get("entPhysicalDescr", "") or "" + + # Match to NetBox module bay + matched_bay = self._match_module_bay(item, index_map, module_bays) + + # Match to NetBox module type (direct lookup, then normalization fallback) + matched_type = module_types.get(model_name) if model_name else None + if not matched_type and model_name: + normalized = apply_normalization_rules( + model_name, "module_type", manufacturer=getattr(self, "_device_manufacturer", None) + ) + if normalized != model_name: + matched_type = module_types.get(normalized) + + # Check {module_path} compatibility + needs_module_path = matched_type and module_type_uses_module_path(matched_type) + module_path_blocked = needs_module_path and not supports_module_path() + + # Check for nested module naming conflicts + name_conflict = ( + matched_type + and matched_bay + and not module_path_blocked + and has_nested_name_conflict(matched_type, matched_bay) + ) + + # Determine status + status = self._determine_status(matched_bay, matched_type, serial, module_path_blocked) + + row = { + "name": name, + "model": model_name or "-", + "serial": serial or "-", + "description": description, + "item_class": phys_class, + "module_bay": matched_bay.name if matched_bay else "-", + "module_type": matched_type.model if matched_type else "-", + "status": status, + "row_class": "", + "can_install": False, + "module_bay_id": matched_bay.pk if matched_bay else None, + "module_type_id": matched_type.pk if matched_type else None, + "depth": depth, + "ent_physical_index": item.get("entPhysicalIndex"), + "has_installable_children": False, + } + + if module_path_blocked: + row["row_class"] = "table-warning" + row["module_path_warning"] = ( + "This module type uses {module_path} in its interface template " + "but the running NetBox does not support it yet." + ) + + if name_conflict: + row["row_class"] = "table-warning" + row["name_conflict_warning"] = ( + "This module type uses {module} in its interface template. " + "Installing multiple siblings will create duplicate interface names. " + "An interface naming plugin with a rewrite rule for this module type can resolve this." + ) + + # Add URLs for matched objects + if matched_bay: + row["module_bay_url"] = matched_bay.get_absolute_url() + # Check if a module is already installed in this bay + if hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + installed = matched_bay.installed_module + row["installed_module"] = installed + row["module_url"] = installed.get_absolute_url() + # Check serial match + if serial and installed.serial and installed.serial.strip() == serial.strip(): + status = "Installed" + row["row_class"] = "table-success" + elif serial and installed.serial and installed.serial.strip() != serial.strip(): + status = "Serial Mismatch" + row["row_class"] = "table-danger" + else: + status = "Installed" + row["row_class"] = "table-success" + row["status"] = status + elif matched_type and not module_path_blocked: + # Bay exists, type matched, no module installed β†’ can install + row["can_install"] = True + + if matched_type: + row["module_type_url"] = matched_type.get_absolute_url() + + return row + + def _determine_status(self, matched_bay, matched_type, serial, module_path_blocked=False): + """Determine the sync status for an inventory item.""" + if module_path_blocked: + return "Requires Upgrade" + if matched_bay and matched_type: + return "Matched" + if not matched_bay: + return "No Bay" + if not matched_type: + return "No Type" + return "Unmatched" + + +class InstallModuleView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, View): + """Install a NetBox Module into a ModuleBay from LibreNMS inventory data.""" + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + module_bay_id = request.POST.get("module_bay_id") + module_type_id = request.POST.get("module_type_id") + serial = request.POST.get("serial", "").strip() + + if not module_bay_id or not module_type_id: + messages.error(request, "Missing module bay or module type.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + module_bay = get_object_or_404(ModuleBay, pk=module_bay_id, device=device) + module_type = get_object_or_404(ModuleType, pk=module_type_id) + + # Block install if module type uses {module_path} and NetBox doesn't support it + from netbox_librenms_plugin.utils import module_type_uses_module_path, supports_module_path + + if module_type_uses_module_path(module_type) and not supports_module_path(): + messages.error( + request, + f"Cannot install {module_type.model}: its interface templates use " + f"{{module_path}} which this NetBox version does not support.", + ) + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Check if bay already has a module installed + if hasattr(module_bay, "installed_module") and module_bay.installed_module: + messages.warning(request, f"Module bay '{module_bay.name}' already has a module installed.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + with transaction.atomic(): + module = Module( + device=device, + module_bay=module_bay, + module_type=module_type, + serial=serial, + status="active", + ) + module.full_clean() + module.save() + + messages.success( + request, f"Installed {module_type.model} in {module_bay.name} (serial: {serial or 'N/A'})." + ) + except Exception as e: + messages.error(request, f"Failed to install module: {e}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + +class InstallBranchView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): + """Install a module and all its installable descendants from LibreNMS inventory.""" + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + parent_index = request.POST.get("parent_index") + + if not parent_index: + messages.error(request, "Missing parent inventory index.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + parent_index = int(parent_index) + except ValueError: + messages.error(request, "Invalid parent inventory index.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Get cached inventory data + cached_data = cache.get(self.get_cache_key(device, "inventory")) + if not cached_data: + messages.error(request, "No cached inventory data. Please refresh modules first.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Build index map and collect the branch to install + index_map = {item["entPhysicalIndex"]: item for item in cached_data} + branch_items = self._collect_branch(parent_index, cached_data) + + if not branch_items: + messages.warning(request, "No installable items found in this branch.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Load module types (with mappings) + module_types = self._get_module_types() + + # Install top-down: each install may create new child bays + installed = [] + skipped = [] + failed = [] + + try: + with transaction.atomic(): + for item in branch_items: + result = self._install_single( + device, + item, + index_map, + module_types, + ModuleBay, + ModuleType, + Module, + ) + if result["status"] == "installed": + installed.append(result["name"]) + elif result["status"] == "skipped": + skipped.append(f"{result['name']}: {result['reason']}") + else: + failed.append(f"{result['name']}: {result['reason']}") + except Exception as e: + messages.error(request, f"Branch install failed: {e}") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Report results + if installed: + messages.success(request, f"Installed {len(installed)} module(s): {', '.join(installed)}") + if skipped: + messages.info(request, f"Skipped {len(skipped)}: {'; '.join(skipped)}") + if failed: + messages.warning(request, f"Failed {len(failed)}: {'; '.join(failed)}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + def _collect_branch(self, parent_index, inventory_data): + """Collect all items in a branch depth-first, parent first. + + Returns items in install order (parent before children). + """ + items = [] + parent = next((i for i in inventory_data if i["entPhysicalIndex"] == parent_index), None) + if parent: + model = (parent.get("entPhysicalModelName") or "").strip() + if model: + items.append(parent) + self._collect_children(parent_index, inventory_data, items) + return items + + def _collect_children(self, parent_idx, inventory_data, items): + """Recursively collect children with models, depth-first.""" + children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] + for child in children: + model = (child.get("entPhysicalModelName") or "").strip() + if model: + items.append(child) + # Always recurse to find deeper items (containers may lack models) + self._collect_children(child["entPhysicalIndex"], inventory_data, items) + + def _get_module_types(self): + """Get all module types indexed by model, with mappings applied.""" + from dcim.models import ModuleType + + from netbox_librenms_plugin.models import ModuleTypeMapping + + types = ModuleType.objects.all().select_related("manufacturer") + result = {} + for mt in types: + result[mt.model] = mt + if mt.part_number and mt.part_number != mt.model: + result[mt.part_number] = mt + for mapping in ModuleTypeMapping.objects.select_related("netbox_module_type__manufacturer"): + result[mapping.librenms_model] = mapping.netbox_module_type + return result + + def _install_single(self, device, item, index_map, module_types, ModuleBay, ModuleType, Module): + """Try to install a single inventory item. + + Re-fetches module bays each time since parent installs create new ones. + Scopes bay lookup to the correct parent module to handle duplicate bay names. + """ + from netbox_librenms_plugin.models import ModuleBayMapping + from netbox_librenms_plugin.utils import ( + apply_normalization_rules, + module_type_uses_module_path, + supports_module_path, + ) + + model_name = (item.get("entPhysicalModelName") or "").strip() + serial = (item.get("entPhysicalSerialNum") or "").strip() + name = item.get("entPhysicalName", "") or model_name + + # Match module type (direct, then normalization fallback) + matched_type = module_types.get(model_name) + if not matched_type and model_name: + manufacturer = getattr(getattr(device, "device_type", None), "manufacturer", None) + normalized = apply_normalization_rules(model_name, "module_type", manufacturer=manufacturer) + if normalized != model_name: + matched_type = module_types.get(normalized) + if not matched_type: + return {"status": "skipped", "name": name, "reason": "no matching type"} + + # Check {module_path} compatibility + if module_type_uses_module_path(matched_type) and not supports_module_path(): + return {"status": "skipped", "name": name, "reason": "requires {module_path}"} + + # Re-fetch module bays (parent install creates new child bays) + bays = ModuleBay.objects.filter(device=device).select_related("installed_module__module_type") + + # Determine if this item belongs under an installed module + # by tracing its LibreNMS parent hierarchy to an installed item + parent_module_id = self._find_parent_module_id(item, index_map, device, ModuleBay) + + if parent_module_id: + bay_dict = {bay.name: bay for bay in bays if bay.module_id == parent_module_id} + else: + bay_dict = {bay.name: bay for bay in bays if not bay.module_id} + + # Match module bay using mapping table + matched_bay = self._match_bay(item, index_map, bay_dict, ModuleBayMapping) + if not matched_bay: + return {"status": "skipped", "name": name, "reason": "no matching bay"} + + # Check if already installed + if hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + return {"status": "skipped", "name": name, "reason": "bay already occupied"} + + # Install + try: + with transaction.atomic(): # savepoint: failure here won't abort parent tx + module = Module( + device=device, + module_bay=matched_bay, + module_type=matched_type, + serial=serial, + status="active", + ) + module.full_clean() + module.save() + except Exception as e: + error_msg = str(e) + if "dcim_interface_unique_device_name" in error_msg: + error_msg = ( + "duplicate interface name β€” this module type's interface template " + "uses {module} which resolves to the same name for all siblings. " + "An interface naming plugin with a rewrite rule for this module type can fix this." + ) + return {"status": "failed", "name": name, "reason": error_msg} + + return {"status": "installed", "name": f"{matched_type.model} β†’ {matched_bay.name}"} + + @staticmethod + def _find_parent_module_id(item, index_map, device, ModuleBay): + """Find the NetBox module ID for the installed parent of this inventory item. + + Walks up the LibreNMS hierarchy to find an ancestor whose name matches + an installed module bay on the device. + """ + from netbox_librenms_plugin.models import ModuleBayMapping + + current = item + for _ in range(10): # max depth guard + parent_idx = current.get("entPhysicalContainedIn", 0) + if not parent_idx or parent_idx not in index_map: + return None + parent = index_map[parent_idx] + parent_name = parent.get("entPhysicalName", "") + parent_descr = parent.get("entPhysicalDescr", "") + + # Check if this parent matches an installed module bay on the device + device_bays = ModuleBay.objects.filter(device=device, module_id__isnull=True).select_related( + "installed_module" + ) + + for bay in device_bays: + if hasattr(bay, "installed_module") and bay.installed_module: + if bay.name == parent_name or (parent_descr and bay.name == parent_descr): + return bay.installed_module.pk + + # Also check ModuleBayMapping for indirect matches + for name in [parent_name, parent_descr]: + if not name: + continue + mapping = ModuleBayMapping.objects.filter(librenms_name=name).first() + if mapping: + bay = ( + ModuleBay.objects.filter(device=device, name=mapping.netbox_bay_name, module_id__isnull=True) + .select_related("installed_module") + .first() + ) + if bay and hasattr(bay, "installed_module") and bay.installed_module: + return bay.installed_module.pk + + current = parent + return None + + @staticmethod + def _match_bay(item, index_map, module_bays, ModuleBayMapping): + """Match an inventory item to a module bay (same logic as BaseModuleTableView).""" + import re + + # Resolve parent name + contained_in = item.get("entPhysicalContainedIn", 0) + parent_name = None + if contained_in: + parent = index_map.get(contained_in) + if parent: + parent_name = parent.get("entPhysicalName", "") + + item_name = item.get("entPhysicalName", "") + item_descr = item.get("entPhysicalDescr", "") + phys_class = item.get("entPhysicalClass", "") + + # Build candidate names: parent, item name, item description + candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + + # Check mapping for each candidate (exact match) + for name in candidate_names: + filters = {"librenms_name": name, "is_regex": False} + if phys_class: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + if not mapping: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + else: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + if mapping and mapping.netbox_bay_name in module_bays: + return module_bays[mapping.netbox_bay_name] + + # Regex pattern matching on all candidate names + for name in candidate_names: + bay = BaseModuleTableView._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + if bay: + return bay + + # Fallback: exact match on candidate names against bay dict + for name in candidate_names: + if name in module_bays: + return module_bays[name] + + # Positional fallback for items inside converters + return BaseModuleTableView._match_bay_by_position(item, index_map, module_bays) diff --git a/netbox_librenms_plugin/views/imports/__init__.py b/netbox_librenms_plugin/views/imports/__init__.py index b6d22d0e1d..3a46ab3625 100644 --- a/netbox_librenms_plugin/views/imports/__init__.py +++ b/netbox_librenms_plugin/views/imports/__init__.py @@ -4,6 +4,7 @@ BulkImportConfirmView, BulkImportDevicesView, DeviceClusterUpdateView, + DeviceConflictActionView, DeviceRackUpdateView, DeviceRoleUpdateView, DeviceValidationDetailsView, @@ -16,6 +17,7 @@ "BulkImportConfirmView", "BulkImportDevicesView", "DeviceClusterUpdateView", + "DeviceConflictActionView", "DeviceRackUpdateView", "DeviceRoleUpdateView", "DeviceValidationDetailsView", diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 46b10b117a..a1aa2db102 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -707,12 +707,75 @@ def get(self, request, device_id): "validation": validation, } + # Add sync comparison data for existing devices + existing = validation.get("existing_device") + if existing: + context["sync_info"] = self._build_sync_info(libre_device, existing) + return render( request, "netbox_librenms_plugin/htmx/device_validation_details.html", context, ) + @staticmethod + def _build_sync_info(libre_device, existing_device): + """Build sync comparison data between LibreNMS device and existing NetBox device.""" + librenms_serial = libre_device.get("serial") or "-" + librenms_os = libre_device.get("os") or "-" + librenms_hardware = libre_device.get("hardware") or "-" + + # Serial comparison + serial_synced = existing_device.serial == librenms_serial or librenms_serial == "-" + + # Platform comparison + platform_info = { + "netbox_platform": getattr(existing_device, "platform", None), + "librenms_os": librenms_os, + "platform_exists": False, + "matching_platform": None, + } + if librenms_os and librenms_os != "-": + from netbox_librenms_plugin.utils import find_matching_platform + + match_result = find_matching_platform(librenms_os) + if match_result["found"]: + platform_info["platform_exists"] = True + platform_info["matching_platform"] = match_result["platform"] + + netbox_platform = platform_info["netbox_platform"] + matching_platform = platform_info["matching_platform"] + platform_synced = librenms_os == "-" or ( + netbox_platform and matching_platform and netbox_platform.pk == matching_platform.pk + ) + + # Device type comparison + device_type_synced = True + librenms_device_type = None + if librenms_hardware and librenms_hardware != "-": + from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type + + hw_match = match_librenms_hardware_to_device_type(librenms_hardware) + if hw_match.get("matched"): + librenms_device_type = hw_match["device_type"] + if not existing_device.device_type or existing_device.device_type.pk != librenms_device_type.pk: + device_type_synced = False + else: + device_type_synced = False + + all_synced = serial_synced and platform_synced and device_type_synced + + return { + "librenms_serial": librenms_serial, + "serial_synced": serial_synced, + "platform_info": platform_info, + "platform_synced": platform_synced, + "librenms_hardware": librenms_hardware, + "librenms_device_type": librenms_device_type, + "device_type_synced": device_type_synced, + "all_synced": all_synced, + } + class DeviceRoleUpdateView(LibreNMSPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View): """HTMX view to update a table row when a role is selected.""" @@ -753,6 +816,195 @@ def post(self, request, device_id): return self.render_device_row(request, libre_device, validation, selections) +class DeviceConflictActionView(LibreNMSPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View): + """HTMX view to resolve device conflicts (link, update, update serial).""" + + def post(self, request, device_id): + """Resolve a device conflict by linking, updating, or syncing serial.""" + if error := self.require_write_permission(): + return error + + from dcim.models import Device + + action = request.POST.get("action") + existing_device_id = request.POST.get("existing_device_id") + + if not action or not existing_device_id: + return HttpResponse("Missing action or existing_device_id", status=400) + + try: + existing_device = Device.objects.get(pk=int(existing_device_id)) + except (Device.DoesNotExist, ValueError): + return HttpResponse("Existing device not found", status=404) + + libre_device, validation, selections = self.get_validated_device_with_selections(device_id, request) + if not libre_device: + return HttpResponse("LibreNMS device not found", status=404) + + # Require force flag when device type mismatches + force = request.POST.get("force") == "on" + if validation.get("device_type_mismatch") and not force: + return HttpResponse( + "Device type mismatch detected. Check the force checkbox to proceed.", + status=400, + ) + + # When force is used with device_type_mismatch, update device type to LibreNMS value + librenms_device_type = None + if validation.get("device_type_mismatch") and force: + librenms_device_type = validation.get("device_type", {}).get("device_type") + + librenms_id = libre_device.get("device_id") + + try: + librenms_id_int = int(librenms_id) + except (TypeError, ValueError): + return HttpResponse("Invalid or missing LibreNMS device_id", status=400) + + if action == "link": + # Link to LibreNMS and update name from LibreNMS data + use_sysname = request.POST.get("use-sysname-toggle") == "on" + strip_domain = request.POST.get("strip-domain-toggle") == "on" + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + existing_device.custom_field_data["librenms_id"] = librenms_id_int + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + existing_device.save() + logger.info(f"Linked device '{existing_device.name}' to LibreNMS ID {librenms_id}") + + elif action == "update": + # Update hostname, serial, and link to LibreNMS + use_sysname = request.POST.get("use-sysname-toggle") == "on" + strip_domain = request.POST.get("strip-domain-toggle") == "on" + incoming_serial = libre_device.get("serial") or "" + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + existing_device.custom_field_data["librenms_id"] = librenms_id_int + if incoming_serial and incoming_serial != "-": + conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + if conflict_device: + return HttpResponse( + f"Serial conflict: '{incoming_serial}' is already assigned to device " + f"'{conflict_device.name}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + existing_device.save() + logger.info( + f"Updated device '{existing_device.name}': serial={incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" + ) + + elif action == "update_serial": + # Update only the serial and link to LibreNMS + incoming_serial = libre_device.get("serial") or "" + existing_device.custom_field_data["librenms_id"] = librenms_id_int + if incoming_serial and incoming_serial != "-": + conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + if conflict_device: + return HttpResponse( + f"Serial conflict: '{incoming_serial}' is already assigned to device " + f"'{conflict_device.name}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + if librenms_device_type: + existing_device.device_type = librenms_device_type + existing_device.save() + logger.info( + f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" + ) + + elif action == "sync_name": + # Sync device name from LibreNMS (e.g., IP β†’ sysName) + use_sysname = request.POST.get("use-sysname-toggle") == "on" + strip_domain = request.POST.get("strip-domain-toggle") == "on" + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + existing_device.name = hostname + existing_device.save() + logger.info(f"Synced name on device '{existing_device.name}' from LibreNMS") + + elif action == "update_type": + # Update device type from LibreNMS (requires force for mismatch) + if librenms_device_type: + existing_device.device_type = librenms_device_type + existing_device.save() + logger.info(f"Updated device type on '{existing_device.name}' to {librenms_device_type}") + else: + return HttpResponse("No LibreNMS device type available to update", status=400) + + elif action == "sync_serial": + # Sync serial number from LibreNMS + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + # Check for serial ownership conflict + conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + if conflict_device: + logger.warning( + f"Serial sync blocked: '{incoming_serial}' already assigned to " + f"'{conflict_device.name}' (pk={conflict_device.pk})" + ) + return HttpResponse( + f"Serial conflict: '{incoming_serial}' is already assigned to device " + f"'{conflict_device.name}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + existing_device.save() + logger.info(f"Synced serial on '{existing_device.name}' to {incoming_serial}") + else: + return HttpResponse("No valid serial from LibreNMS", status=400) + + elif action == "sync_platform": + # Sync platform from LibreNMS OS + from netbox_librenms_plugin.utils import find_matching_platform + + librenms_os = libre_device.get("os") or "" + if librenms_os and librenms_os != "-": + match_result = find_matching_platform(librenms_os) + if match_result["found"]: + existing_device.platform = match_result["platform"] + existing_device.save() + logger.info(f"Synced platform on '{existing_device.name}' to {match_result['platform']}") + else: + return HttpResponse(f"Platform '{librenms_os}' not found in NetBox", status=400) + else: + return HttpResponse("No OS info from LibreNMS", status=400) + + elif action == "sync_device_type": + # Sync device type from LibreNMS hardware (non-mismatch case) + from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type + + hardware = libre_device.get("hardware") or "" + hw_match = match_librenms_hardware_to_device_type(hardware) + if hw_match.get("matched"): + existing_device.device_type = hw_match["device_type"] + existing_device.save() + logger.info(f"Synced device type on '{existing_device.name}' to {hw_match['device_type']}") + else: + return HttpResponse(f"No matching device type for '{hardware}'", status=400) + + else: + return HttpResponse(f"Unknown action: {action}", status=400) + + # Clear cached validation so re-validation picks up the changes + cache_key = get_import_device_cache_key(device_id, self.librenms_api.server_key) + cache.delete(cache_key) + + # Re-validate and render updated row + libre_device, validation, selections = self.get_validated_device_with_selections(device_id, request) + if not libre_device: + return HttpResponse("Device not found after action", status=404) + + response = self.render_device_row(request, libre_device, validation, selections) + response["HX-Trigger"] = "closeModal" + return response + + class SaveUserPrefView(LibreNMSPermissionMixin, View): """Save a user preference via POST. Used by JS toggle handlers.""" diff --git a/netbox_librenms_plugin/views/mapping_views.py b/netbox_librenms_plugin/views/mapping_views.py index b1fcec9c77..55ff7bd658 100644 --- a/netbox_librenms_plugin/views/mapping_views.py +++ b/netbox_librenms_plugin/views/mapping_views.py @@ -1,14 +1,44 @@ from netbox.views import generic from utilities.views import register_model_view -from netbox_librenms_plugin.filters import InterfaceTypeMappingFilterSet +from netbox_librenms_plugin.filters import ( + DeviceTypeMappingFilterSet, + InterfaceTypeMappingFilterSet, + ModuleBayMappingFilterSet, + ModuleTypeMappingFilterSet, + NormalizationRuleFilterSet, +) from netbox_librenms_plugin.forms import ( + DeviceTypeMappingFilterForm, + DeviceTypeMappingForm, + DeviceTypeMappingImportForm, InterfaceTypeMappingFilterForm, InterfaceTypeMappingForm, InterfaceTypeMappingImportForm, + ModuleBayMappingFilterForm, + ModuleBayMappingForm, + ModuleBayMappingImportForm, + ModuleTypeMappingFilterForm, + ModuleTypeMappingForm, + ModuleTypeMappingImportForm, + NormalizationRuleFilterForm, + NormalizationRuleForm, + NormalizationRuleImportForm, +) +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) +from netbox_librenms_plugin.tables.mappings import ( + DeviceTypeMappingTable, + InterfaceTypeMappingTable, + ModuleBayMappingTable, + ModuleTypeMappingTable, + NormalizationRuleTable, ) -from netbox_librenms_plugin.models import InterfaceTypeMapping -from netbox_librenms_plugin.tables.mappings import InterfaceTypeMappingTable from netbox_librenms_plugin.views.mixins import LibreNMSPermissionMixin @@ -84,3 +114,243 @@ class InterfaceTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectC """ queryset = InterfaceTypeMapping.objects.all() + + +# --- DeviceTypeMapping views --- + + +class DeviceTypeMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + table = DeviceTypeMappingTable + filterset = DeviceTypeMappingFilterSet + filterset_form = DeviceTypeMappingFilterForm + template_name = "netbox_librenms_plugin/devicetypemapping_list.html" + + +class DeviceTypeMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + form = DeviceTypeMappingForm + + +@register_model_view(DeviceTypeMapping, "bulk_import", path="import", detail=False) +class DeviceTypeMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + model_form = DeviceTypeMappingImportForm + + +class DeviceTypeMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +class DeviceTypeMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + form = DeviceTypeMappingForm + + +class DeviceTypeMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +class DeviceTypeMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + table = DeviceTypeMappingTable + + +class DeviceTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +# --- ModuleTypeMapping views --- + + +class ModuleTypeMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + table = ModuleTypeMappingTable + filterset = ModuleTypeMappingFilterSet + filterset_form = ModuleTypeMappingFilterForm + template_name = "netbox_librenms_plugin/moduletypemapping_list.html" + + +class ModuleTypeMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + form = ModuleTypeMappingForm + + +@register_model_view(ModuleTypeMapping, "bulk_import", path="import", detail=False) +class ModuleTypeMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + model_form = ModuleTypeMappingImportForm + + +class ModuleTypeMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +class ModuleTypeMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + form = ModuleTypeMappingForm + + +class ModuleTypeMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +class ModuleTypeMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + table = ModuleTypeMappingTable + + +class ModuleTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +# --- ModuleBayMapping views --- + + +class ModuleBayMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + table = ModuleBayMappingTable + filterset = ModuleBayMappingFilterSet + filterset_form = ModuleBayMappingFilterForm + template_name = "netbox_librenms_plugin/modulebaymapping_list.html" + + +class ModuleBayMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + form = ModuleBayMappingForm + + +@register_model_view(ModuleBayMapping, "bulk_import", path="import", detail=False) +class ModuleBayMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + model_form = ModuleBayMappingImportForm + + +class ModuleBayMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +class ModuleBayMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + form = ModuleBayMappingForm + + +class ModuleBayMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +class ModuleBayMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + table = ModuleBayMappingTable + + +class ModuleBayMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +# --- NormalizationRule views --- + + +class NormalizationRuleListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + table = NormalizationRuleTable + filterset = NormalizationRuleFilterSet + filterset_form = NormalizationRuleFilterForm + template_name = "netbox_librenms_plugin/normalizationrule_list.html" + + +class NormalizationRuleCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + form = NormalizationRuleForm + + +@register_model_view(NormalizationRule, "bulk_import", path="import", detail=False) +class NormalizationRuleBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + model_form = NormalizationRuleImportForm + + +class NormalizationRuleView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + + +class NormalizationRuleEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + form = NormalizationRuleForm + + +class NormalizationRuleDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + + +class NormalizationRuleBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + table = NormalizationRuleTable + + +class NormalizationRuleChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() diff --git a/netbox_librenms_plugin/views/object_sync/__init__.py b/netbox_librenms_plugin/views/object_sync/__init__.py index e9893cf7d8..f025cb2a7b 100644 --- a/netbox_librenms_plugin/views/object_sync/__init__.py +++ b/netbox_librenms_plugin/views/object_sync/__init__.py @@ -5,6 +5,7 @@ DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceVLANTableView, SaveVlanGroupOverridesView, SingleInterfaceVerifyView, diff --git a/netbox_librenms_plugin/views/object_sync/devices.py b/netbox_librenms_plugin/views/object_sync/devices.py index 97584f8319..72f63a094f 100644 --- a/netbox_librenms_plugin/views/object_sync/devices.py +++ b/netbox_librenms_plugin/views/object_sync/devices.py @@ -17,6 +17,7 @@ LibreNMSInterfaceTable, VCInterfaceTable, ) +from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable from netbox_librenms_plugin.utils import ( get_interface_name_field, get_missing_vlan_warning, @@ -29,6 +30,7 @@ from ..base.interfaces_view import BaseInterfaceTableView from ..base.ip_addresses_view import BaseIPAddressTableView from ..base.librenms_sync_view import BaseLibreNMSSyncView +from ..base.modules_view import BaseModuleTableView from ..base.vlan_table_view import BaseVLANTableView from ..mixins import CacheMixin, LibreNMSPermissionMixin @@ -63,6 +65,12 @@ def get_vlan_context(self, request, obj): vlan_table_view.request = request return vlan_table_view.get_vlan_context(request, obj) + def get_module_context(self, request, obj): + """Return module sync context for the device.""" + module_table_view = DeviceModuleTableView() + module_table_view.request = request + return module_table_view.get_context_data(request, obj) + class DeviceInterfaceTableView(BaseInterfaceTableView): """Interface synchronization table for Devices.""" @@ -375,3 +383,15 @@ class DeviceVLANTableView(BaseVLANTableView): """VLAN synchronization table view for Devices.""" model = Device + + +class DeviceModuleTableView(BaseModuleTableView): + """Module/inventory synchronization view for Devices.""" + + model = Device + + def get_table(self, data, obj): + """Return the module sync table.""" + table = LibreNMSModuleTable(data, device=obj) + table.htmx_url = f"{self.request.path}?tab=modules" + return table diff --git a/netbox_librenms_plugin/views/sync/cables.py b/netbox_librenms_plugin/views/sync/cables.py index 0e52f3b017..27dc5943c2 100644 --- a/netbox_librenms_plugin/views/sync/cables.py +++ b/netbox_librenms_plugin/views/sync/cables.py @@ -1,3 +1,5 @@ +import logging + from dcim.models import Cable, Device, Interface from django.contrib import messages from django.core.cache import cache @@ -9,6 +11,8 @@ from netbox_librenms_plugin.views.mixins import CacheMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin +logger = logging.getLogger(__name__) + class SyncCablesView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): """Create NetBox cables using cached LibreNMS link data.""" @@ -42,7 +46,11 @@ def get_cached_links_data(self, request, obj): return cached_data.get("links", []) def create_cable(self, local_interface, remote_interface, request): - """Create a cable between local and remote interfaces.""" + """Create a cable between local and remote interfaces. + + Returns: + True on success, False on failure. + """ try: Cable.objects.create( a_terminations=[local_interface], @@ -81,13 +89,12 @@ def process_single_interface(self, interface, cached_links): link_data = next(link for link in cached_links if link["local_port"] == interface["interface"]) return self.handle_cable_creation(link_data, interface) except StopIteration: - return {"status": "invalid"} + return {"status": "invalid", "interface": interface.get("interface", "")} def verify_cable_creation_requirements(self, link_data): """Return True if all required NetBox IDs are present in link data.""" required_fields = [ "netbox_local_interface_id", - "netbox_remote_device_id", "netbox_remote_interface_id", ] @@ -113,13 +120,21 @@ def handle_cable_creation(self, link_data, interface): return {"status": "missing_remote", "interface": interface["interface"]} def process_interface_sync(self, selected_interfaces, cached_links): - """Process cable sync for all selected interfaces and return results.""" + """Process cable sync for all selected interfaces and return results. + + Each interface is processed in its own atomic block so individual + failures roll back only that cable without affecting others. + """ results = {"valid": [], "invalid": [], "duplicate": [], "missing_remote": []} - with transaction.atomic(): - for interface in selected_interfaces: - result = self.process_single_interface(interface, cached_links) + for interface in selected_interfaces: + try: + with transaction.atomic(): + result = self.process_single_interface(interface, cached_links) results[result["status"]].append(result.get("interface", "")) + except Exception: + logger.exception("Failed to sync cable for interface %s", interface.get("interface", "")) + results["invalid"].append(interface.get("interface", "")) return results diff --git a/netbox_librenms_plugin/views/sync/device_fields.py b/netbox_librenms_plugin/views/sync/device_fields.py index 952afdefa9..d0a7141709 100644 --- a/netbox_librenms_plugin/views/sync/device_fields.py +++ b/netbox_librenms_plugin/views/sync/device_fields.py @@ -1,7 +1,7 @@ from dcim.models import Device, Manufacturer, Platform from django.contrib import messages from django.core.exceptions import ValidationError -from django.db import IntegrityError +from django.db import IntegrityError, transaction from django.shortcuts import get_object_or_404, redirect from django.views import View @@ -9,6 +9,53 @@ from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin +class UpdateDeviceNameView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, LibreNMSAPIMixin, View): + """Update NetBox device name from LibreNMS sysName.""" + + required_object_permissions = { + "POST": [("change", Device)], + } + + def post(self, request, pk): + """Sync the device name from LibreNMS sysName.""" + if error := self.require_all_permissions("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + self.librenms_id = self.librenms_api.get_librenms_id(device) + + if not self.librenms_id: + messages.error(request, "Device not found in LibreNMS") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + success, device_info = self.librenms_api.get_device_info(self.librenms_id) + + if not success or not device_info: + messages.error(request, "Failed to retrieve device info from LibreNMS") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + sys_name = device_info.get("sysName") + + if not sys_name: + messages.warning(request, "No sysName available in LibreNMS") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + old_name = device.name + device.name = sys_name + try: + device.full_clean() + device.save() + except (ValidationError, IntegrityError) as e: + device.name = old_name + error_msg = e.message_dict if hasattr(e, "message_dict") else str(e) + messages.error(request, f"Failed to update device name to '{sys_name}': {error_msg}") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + messages.success(request, f"Device name updated from '{old_name}' to '{sys_name}'") + + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + class UpdateDeviceSerialView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, LibreNMSAPIMixin, View): """Update NetBox device serial number from LibreNMS.""" @@ -231,26 +278,28 @@ def post(self, request, pk): pass try: - platform = Platform.objects.create( - name=platform_name, - manufacturer=manufacturer, - ) - except IntegrityError: - messages.error( - request, - f"Platform '{platform_name}' could not be created (slug collision). Try a different name.", - ) + with transaction.atomic(): + platform = Platform.objects.create( + name=platform_name, + manufacturer=manufacturer, + ) + + device.platform = platform + device.full_clean() + device.save() + except IntegrityError as e: + error_str = str(e) + if "platform" in error_str.lower() or "slug" in error_str.lower(): + messages.error( + request, + f"Platform '{platform_name}' could not be created (slug collision). Try a different name.", + ) + else: + messages.error(request, f"Failed to assign platform '{platform_name}': {error_str}") return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) - - old_platform = device.platform - device.platform = platform - try: - device.full_clean() - device.save() - except (ValidationError, IntegrityError) as e: - device.platform = old_platform + except ValidationError as e: error_msg = e.message_dict if hasattr(e, "message_dict") else str(e) - messages.error(request, f"Failed to assign platform '{platform}': {error_msg}") + messages.error(request, f"Failed to assign platform '{platform_name}': {error_msg}") return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) messages.success( diff --git a/netbox_librenms_plugin/views/sync/devices.py b/netbox_librenms_plugin/views/sync/devices.py index da0f9af5b0..eb45b375ff 100644 --- a/netbox_librenms_plugin/views/sync/devices.py +++ b/netbox_librenms_plugin/views/sync/devices.py @@ -26,7 +26,7 @@ def get_object(self, object_id): try: return Device.objects.get(pk=object_id) except Device.DoesNotExist: - return VirtualMachine.objects.get(pk=object_id) + return get_object_or_404(VirtualMachine, pk=object_id) def post(self, request, object_id): """Add a device to LibreNMS using the submitted SNMP form.""" diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index a19629cccf..c00989b45d 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -163,26 +163,9 @@ def sync_interface(self, obj, librenms_interface, exclude_columns, interface_nam interface_name_field, ) - if "enabled" not in exclude_columns: - interface.enabled = ( - True - if librenms_interface.get("ifAdminStatus") is None - else ( - librenms_interface["ifAdminStatus"].lower() == "up" - if isinstance(librenms_interface["ifAdminStatus"], str) - else bool(librenms_interface["ifAdminStatus"]) - ) - ) - # Sync VLANs if not excluded - vlan_synced = False if "vlans" not in exclude_columns: self._sync_interface_vlans(interface, librenms_interface, interface_name) - vlan_synced = True - - # Skip redundant save when _sync_interface_vlans already saved (via _update_interface_vlan_assignment) - if not vlan_synced: - interface.save() def get_netbox_interface_type(self, librenms_interface): """Return the NetBox interface type mapped from LibreNMS type and speed.""" @@ -248,8 +231,17 @@ def update_interface_attributes( if "librenms_id" in interface.cf: interface.custom_field_data["librenms_id"] = librenms_interface.get("port_id") - ifPhysAddress = librenms_interface.get("ifPhysAddress") - self.handle_mac_address(interface, ifPhysAddress) + if "enabled" not in exclude_columns: + admin_status = librenms_interface.get("ifAdminStatus") + interface.enabled = ( + True + if admin_status is None + else (admin_status.lower() == "up" if isinstance(admin_status, str) else bool(admin_status)) + ) + + if "mac_address" not in exclude_columns: + ifPhysAddress = librenms_interface.get("ifPhysAddress") + self.handle_mac_address(interface, ifPhysAddress) interface.save() diff --git a/netbox_librenms_plugin/views/sync/locations.py b/netbox_librenms_plugin/views/sync/locations.py index 915ed61941..58f58f9490 100644 --- a/netbox_librenms_plugin/views/sync/locations.py +++ b/netbox_librenms_plugin/views/sync/locations.py @@ -46,10 +46,6 @@ def get_queryset(self): if self.request.GET and self.filterset: return self.filterset(self.request.GET, queryset=sync_data).qs - if "q" in self.request.GET: - query = self.request.GET.get("q", "").lower() - sync_data = [item for item in sync_data if query in item.netbox_site.name.lower()] - return sync_data def get_librenms_locations(self): diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000000..62c68691ec --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,6 @@ +"""Conftest for e2e tests β€” no Django initialization needed.""" + +import os + +# Prevent pytest-django from trying to initialize Django +os.environ.pop("DJANGO_SETTINGS_MODULE", None) diff --git a/tests/e2e/test_module_install.py b/tests/e2e/test_module_install.py new file mode 100644 index 0000000000..cee05eb904 --- /dev/null +++ b/tests/e2e/test_module_install.py @@ -0,0 +1,330 @@ +"""End-to-end Playwright tests for LibreNMS plugin module sync workflow. + +These tests exercise the full import β†’ modules β†’ install flow against a +live NetBox + LibreNMS instance inside the devcontainer. + +Prerequisites: + - NetBox running at NETBOX_URL (default http://172.22.0.4:8000) + - LibreNMS server configured in plugin settings + - Device 15 (WS-C4900M) exists and is linked to LibreNMS + - Playwright installed: pip install playwright && playwright install chromium + +Run: + cd /home/mzieba/workspace/netbox-librenms-plugin + HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ + no_proxy=localhost,127.0.0.1,172.22.0.4 \ + python -m pytest tests/e2e/test_module_install.py -v -s +""" + +import os +import subprocess +import time + +import pytest + +NETBOX_URL = os.environ.get("NETBOX_URL", "http://172.22.0.4:8000") +NETBOX_USER = os.environ.get("NETBOX_USER", "admin") +NETBOX_PASS = os.environ.get("NETBOX_PASS", "admin") +CONTAINER_NAME = None + + +def _get_container(): + """Find the devcontainer name.""" + global CONTAINER_NAME + if CONTAINER_NAME: + return CONTAINER_NAME + result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + for name in result.stdout.strip().split("\n"): + if "devcontainer-devcontainer" in name: + CONTAINER_NAME = name + return name + pytest.skip("No devcontainer found") + + +def _netbox_shell(code): + """Run Python code in NetBox's Django shell.""" + import shlex + + container = _get_container() + escaped = shlex.quote(code) + result = subprocess.run( + [ + "docker", + "exec", + container, + "bash", + "-c", + f"cd /opt/netbox/netbox && python3 manage.py shell -c {escaped}", + ], + capture_output=True, + text=True, + env={"PATH": "/usr/bin:/bin", "HOME": "/root"}, + ) + # Filter out config loading lines + lines = [line for line in result.stdout.strip().split("\n") if not line.startswith(("🧬", "156 objects"))] + return "\n".join(lines).strip() + + +def _delete_device_modules(device_id): + """Remove all modules from a device.""" + _netbox_shell( + f"from dcim.models import Module; " + f"deleted = Module.objects.filter(device_id={device_id}).delete(); " + f"print(f'Deleted {{deleted}}')" + ) + + +def _get_interfaces(device_id): + """Get interface names for a device.""" + output = _netbox_shell( + f"from dcim.models import Interface; " + f'[print(f\'{{i.name}}|{{i.module.module_type.model if i.module else "-"}}|' + f'{{i.module.module_bay.name if i.module else "-"}}\')' + f" for i in Interface.objects.filter(device_id={device_id}).order_by('name')]" + ) + results = [] + for line in output.split("\n"): + if "|" in line: + name, mod_type, bay = line.split("|") + results.append({"name": name, "module_type": mod_type, "bay": bay}) + return results + + +@pytest.fixture(scope="module") +def browser(): + """Launch browser for the test module.""" + from playwright.sync_api import sync_playwright + + pw = sync_playwright().start() + b = pw.chromium.launch(headless=True) + yield b + b.close() + pw.stop() + + +@pytest.fixture +def page(browser): + """Create a new page and log in to NetBox.""" + ctx = browser.new_context(ignore_https_errors=True) + pg = ctx.new_page() + + pg.goto(f"{NETBOX_URL}/login/", timeout=10000) + pg.fill("#id_username", NETBOX_USER) + pg.fill("#id_password", NETBOX_PASS) + pg.click("button[type=submit]") + pg.wait_for_load_state("networkidle") + yield pg + ctx.close() + + +class TestModuleInstallWorkflow: + """Test the full module sync and install workflow on device 15 (WS-C4900M).""" + + DEVICE_ID = 15 + + def _goto_modules_tab(self, page): + """Navigate to the modules sync tab and refresh data.""" + page.goto(f"{NETBOX_URL}/dcim/devices/{self.DEVICE_ID}/librenms-sync/?tab=modules") + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Click Refresh Modules + btn = page.query_selector('button:has-text("Refresh Modules")') + assert btn is not None, "Refresh Modules button not found" + btn.click() + time.sleep(8) + + def _get_table_rows(self, page): + """Parse the module sync table into dicts.""" + pane = page.query_selector("#modules") + assert pane is not None, "Modules pane not found" + + rows = [] + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if len(cells) >= 8: + rows.append( + { + "name": cells[0].inner_text().strip(), + "model": cells[1].inner_text().strip(), + "serial": cells[2].inner_text().strip(), + "bay": cells[5].inner_text().strip(), + "type": cells[6].inner_text().strip(), + "status": cells[7].inner_text().strip(), + } + ) + return rows + + def test_clean_state_shows_install_buttons(self, page): + """After deleting all modules, table shows Install buttons.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + rows = self._get_table_rows(page) + assert len(rows) > 0, "No rows in module sync table" + + # Top-level items with matched bays should show Matched status + supervisor = [r for r in rows if "Supervisor(slot 1)" in r["name"]] + assert len(supervisor) == 1, f"Expected 1 Supervisor row, got {len(supervisor)}" + assert supervisor[0]["status"] == "Matched", f"Expected Matched, got {supervisor[0]['status']}" + + def test_single_install(self, page): + """Installing a single top-level module works.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Install FanTray 1 + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "FanTray 1" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install"):not(:has-text("Branch"))') + if btn: + btn.click() + time.sleep(5) + break + + # Verify via DB + output = _netbox_shell( + f"from dcim.models import Module; " + f"m = Module.objects.filter(device_id={self.DEVICE_ID}, module_bay__name='Fan Tray 1').first(); " + f"print(m.module_type.model if m else 'NONE')" + ) + assert "WS-X4992" in output, f"FanTray not installed: {output}" + + def test_branch_install_supervisor(self, page): + """Branch install creates supervisor + X2 transceivers with correct names.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Click Install Branch on Supervisor(slot 1) + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + assert btn is not None, "Install Branch button not found for Supervisor" + btn.click() + break + + # Wait for branch install to complete (creates many modules + signals) + time.sleep(20) + page.wait_for_load_state("networkidle") + time.sleep(5) + + # Verify interfaces have correct names (not bare position numbers) + interfaces = _get_interfaces(self.DEVICE_ID) + x2_interfaces = [i for i in interfaces if i["module_type"] in ("X2-10GB-LR", "X2-10GB-SR")] + + assert len(x2_interfaces) > 0, "No X2 transceiver interfaces created" + + for iface in x2_interfaces: + assert iface["name"].startswith("TenGigabitEthernet"), ( + f"Interface '{iface['name']}' in {iface['bay']} " + f"should start with 'TenGigabitEthernet' (INR rule not applied?)" + ) + + def test_branch_install_no_duplicate_errors(self, page): + """Branch install handles already-occupied bays gracefully.""" + # Don't delete modules β€” some should already be installed + self._goto_modules_tab(page) + + # Click Install Branch on Supervisor(slot 1) again + pane = page.query_selector("#modules") + branch_btn = None + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + branch_btn = tr.query_selector('button:has-text("Install Branch")') + break + + if branch_btn: + branch_btn.click() + time.sleep(10) + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Check for error messages β€” should only have skips, no failures + body_text = page.query_selector("body").inner_text() + assert "Branch install failed" not in body_text, ( + "Branch install crashed instead of handling errors gracefully" + ) + + def test_child_bays_hidden_when_parent_not_installed(self, page): + """Children show 'No Bay' when parent module is not installed.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + rows = self._get_table_rows(page) + + # Children of Supervisor(slot 1) should show "No matching bay" + # since Supervisor isn't installed, its child bays don't exist yet + children = [r for r in rows if r["name"].startswith("└─") and "TenGigabitEthernet1/" in r["name"]] + for child in children: + assert "No matching bay" in child["bay"], ( + f"Child '{child['name']}' should show 'No matching bay' when parent not installed, got '{child['bay']}'" + ) + + def test_full_workflow(self, page): + """Full workflow: clean β†’ install individuals β†’ branch install β†’ verify.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Step 1: Install PSUs and FanTray individually + for label in ["FanTray 1", "Power Supply 1", "Power Supply 2"]: + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and label in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install"):not(:has-text("Branch"))') + if btn: + btn.click() + time.sleep(4) + break + + # Step 2: Branch install Supervisor + transceivers + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + if btn: + btn.click() + time.sleep(10) + break + + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Step 3: Branch install Linecard + self._goto_modules_tab(page) + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Linecard(slot 3)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + if btn: + btn.click() + time.sleep(10) + break + + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Verify: all installable modules should be installed + self._goto_modules_tab(page) + rows = self._get_table_rows(page) + + matched_but_not_installed = [r for r in rows if r["status"] == "Matched" and not r["name"].startswith("└─")] + assert len(matched_but_not_installed) == 0, ( + f"Top-level items still 'Matched' after full workflow: {[r['name'] for r in matched_but_not_installed]}" + ) + + # Verify interface naming + interfaces = _get_interfaces(self.DEVICE_ID) + for iface in interfaces: + assert iface["name"] != "1", "Interface with bare name '1' found β€” INR rule not applied"