diff --git a/.devcontainer/.env.example b/.devcontainer/.env.example index 1437c2ed67..912cd19fd9 100644 --- a/.devcontainer/.env.example +++ b/.devcontainer/.env.example @@ -31,3 +31,17 @@ SKIP_SUPERUSER=false # .devcontainer/plugin-config.py.example β†’ .devcontainer/plugin-config.py # Advanced NetBox configuration (optional): # .devcontainer/extra-configuration.py.example β†’ .devcontainer/extra-configuration.py + +# Proxy Configuration (optional, for corporate networks with MITM proxies) +# Uncomment and set these if you're behind a proxy +# HTTP_PROXY=http://proxy.example.com:8080 +# HTTPS_PROXY=http://proxy.example.com:8080 +# NO_PROXY=localhost,127.0.0.1,postgres,redis +# REQUESTS_CA_BUNDLE=/path/to/ca-bundle.crt +# SSL_CERT_FILE=/path/to/ca-bundle.crt +# CURL_CA_BUNDLE=/path/to/ca-bundle.crt + +# Git SSL verification override (default: false) +# Only set to true if behind a MITM proxy and you cannot provide a CA bundle. +# Prefer placing a ca-bundle.crt in the workspace root instead. +# ALLOW_GIT_SSL_DISABLE=false diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 98c4213cef..28e119cf6f 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -65,6 +65,20 @@ If you need to test with a LibreNMS instance on a private network (local lab, co - **GitHub CLI**: Automatically configured for easy PR submission - **Logs**: Use `netbox-logs` to debug issues in real-time + +### πŸ“‘ LibreNMS Server Configuration + +You need a LibreNMS instance to use this plugin. Configure your LibreNMS server(s) in `plugin-config.py`: + +1. Copy the example config: + + ```bash + cp .devcontainer/config/plugin-config.py.example .devcontainer/config/plugin-config.py + ``` + +2. Edit it with your LibreNMS server URL(s) and API token(s) +3. Restart NetBox: `netbox-restart` + ## Out-of-the-box defaults Below are the dev container defaults. The field name to change these defaults is listed below each line. @@ -84,6 +98,8 @@ Below are the dev container defaults. The field name to change these defaults is - Plugin loader: enabled; reads `.devcontainer/config/plugin-config.py` if present - If `plugin-config.py` is missing: plugin is enabled with empty config (features won’t work until configured) + + ## πŸ”§ Configuration ### NetBox Version and Environment (use .devcontainer/.env) @@ -139,6 +155,84 @@ You might experience issues with database schemas and migrations when changing N - Database: `DB_HOST`, `DB_NAME`, `DB_USER`, `DB_PASSWORD` - Redis: `REDIS_HOST`, `REDIS_PASSWORD` - Superuser: `SUPERUSER_NAME`, `SUPERUSER_EMAIL`, `SUPERUSER_PASSWORD`, `SKIP_SUPERUSER` +- Proxy: `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `REQUESTS_CA_BUNDLE`, `SSL_CERT_FILE`, `CURL_CA_BUNDLE` + +### 🌐 Proxy Configuration (MITM Proxies) + +If you're behind a corporate proxy or MITM proxy (like Zscaler, BlueCoat, etc.), you need to configure proxy at two levels: the Docker client (for building) and the container runtime (for package installation inside the container). + +**Step 1: Configure Docker client proxy** (`~/.docker/config.json`) + +This is **required** so that `apt-get`, `curl`, etc. work during the container image build (e.g., when installing devcontainer features like `git` and `github-cli`). + +Create or edit `~/.docker/config.json`: + +```json +{ + "proxies": { + "default": { + "httpProxy": "http://proxy.example.com:8080", + "httpsProxy": "http://proxy.example.com:8080", + "noProxy": "localhost,127.0.0.1,postgres,redis" + } + } +} +``` + +Docker automatically injects these as environment variables into every `RUN` instruction during `docker build`. No VS Code restart is needed β€” this takes effect immediately. + +> **Docker Desktop users:** You can configure the same settings via Docker Desktop Settings β†’ Resources β†’ Proxies, which writes this file for you. + +**Step 2: Create `.devcontainer/.env`** (for container runtime) + +```bash +cp .devcontainer/.env.example .devcontainer/.env +``` + +Add your proxy settings to `.devcontainer/.env`: + +```bash +# Proxy Configuration +HTTP_PROXY=http://proxy.example.com:8080 +HTTPS_PROXY=http://proxy.example.com:8080 +NO_PROXY=localhost,127.0.0.1,postgres,redis + +# CA Certificate Bundle (if your proxy uses a custom CA) +# Place your CA cert in the workspace, e.g., /workspaces/netbox-librenms-plugin/ca-bundle.crt +# REQUESTS_CA_BUNDLE=/workspaces/netbox-librenms-plugin/ca-bundle.crt +# SSL_CERT_FILE=/workspaces/netbox-librenms-plugin/ca-bundle.crt +# CURL_CA_BUNDLE=/workspaces/netbox-librenms-plugin/ca-bundle.crt +``` + +**Step 3: Add your CA certificate** (optional, only if your proxy intercepts TLS): + - Export your proxy's CA certificate (usually available from your IT department or browser) + - Save it as `ca-bundle.crt` in the root of your workspace + - Uncomment and update the `*_CA_BUNDLE` / `SSL_CERT_FILE` lines in `.env` + +**Step 4: Rebuild the container**: + - VS Code: Ctrl+Shift+P β†’ "Dev Containers: Rebuild Container" + +**What gets configured:** +- `~/.docker/config.json` β†’ proxy for Docker build steps (devcontainer features, apt in Dockerfile) +- `.devcontainer/.env` β†’ proxy for running containers (apt, pip, curl at runtime) +- `setup.sh` auto-configures apt proxy and git SSL settings inside the container + +**Important Notes:** +- The `.env` file is ignored by git, so your proxy credentials stay private +- `~/.docker/config.json` is a per-user file outside the repo +- Add internal service names to `NO_PROXY` to avoid routing internal Docker traffic through the proxy +- **Proxy authentication:** Embedding credentials directly in the proxy URL (e.g., `http://username:password@proxy.example.com:8080`) is insecure β€” credentials can be visible in process listings, environment dumps, `docker inspect` output, and logs. Prefer safer alternatives such as Docker's `config.json` with `credsStore` or a secret manager for storing proxy credentials securely. + +**Common Issues:** + +*"Could not connect to archive.ubuntu.com" during build* +- β†’ `~/.docker/config.json` is missing or has wrong proxy URL + +*"SSL certificate errors" during build* +- β†’ Your proxy uses a MITM certificate. Export it and add it to the system trust store, or set `SSL_CERT_FILE` in `.env` + +*Container builds but apt/pip fails inside* +- β†’ .env file is missing or has wrong proxy settings. Check .env matches Docker Desktop settings After any `.env` change, rebuild the dev container to apply environment updates. diff --git a/.devcontainer/config/plugin-config.py.example b/.devcontainer/config/plugin-config.py.example index c9648174cc..5e48833a48 100644 --- a/.devcontainer/config/plugin-config.py.example +++ b/.devcontainer/config/plugin-config.py.example @@ -13,13 +13,13 @@ PLUGINS = [ "netbox_librenms_plugin", ] -# Sample configuration with three example servers +# Sample configuration with example servers PLUGINS_CONFIG = { "netbox_librenms_plugin": { "servers": { "production": { "display_name": "Production LibreNMS", - "librenms_url": "https://librenms-prod.exampel.com", + "librenms_url": "https://librenms-prod.example.com", "api_token": "your-prod-token", "cache_timeout": 300, "verify_ssl": True, diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 77c4c8cccc..63a1a9b471 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -34,12 +34,18 @@ "SUPERUSER_NAME": "${localEnv:SUPERUSER_NAME:admin}", "SUPERUSER_EMAIL": "${localEnv:SUPERUSER_EMAIL:admin@example.com}", "SUPERUSER_PASSWORD": "${localEnv:SUPERUSER_PASSWORD:admin}", - "SKIP_SUPERUSER": "${localEnv:SKIP_SUPERUSER:false}" - }, - "features": { - "ghcr.io/devcontainers/features/git:1": {}, - "ghcr.io/devcontainers/features/github-cli:1": {} + "SKIP_SUPERUSER": "${localEnv:SKIP_SUPERUSER:false}", + "HTTP_PROXY": "${localEnv:HTTP_PROXY}", + "HTTPS_PROXY": "${localEnv:HTTPS_PROXY}", + "http_proxy": "${localEnv:HTTP_PROXY}", + "https_proxy": "${localEnv:HTTPS_PROXY}", + "NO_PROXY": "${localEnv:NO_PROXY}", + "no_proxy": "${localEnv:NO_PROXY}", + "REQUESTS_CA_BUNDLE": "${localEnv:REQUESTS_CA_BUNDLE}", + "SSL_CERT_FILE": "${localEnv:SSL_CERT_FILE}", + "CURL_CA_BUNDLE": "${localEnv:CURL_CA_BUNDLE}" }, + "features": {}, "customizations": { "vscode": { "extensions": [ diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index d1b7ed03f4..50e1d278bb 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -22,6 +22,16 @@ services: SUPERUSER_EMAIL: ${SUPERUSER_EMAIL:-admin@example.com} SUPERUSER_PASSWORD: ${SUPERUSER_PASSWORD:-admin} SKIP_SUPERUSER: ${SKIP_SUPERUSER:-false} + # Proxy settings (optional) + HTTP_PROXY: ${HTTP_PROXY:-} + HTTPS_PROXY: ${HTTPS_PROXY:-} + http_proxy: ${HTTP_PROXY:-} + https_proxy: ${HTTPS_PROXY:-} + NO_PROXY: ${NO_PROXY:-} + no_proxy: ${NO_PROXY:-} + REQUESTS_CA_BUNDLE: ${REQUESTS_CA_BUNDLE:-} + SSL_CERT_FILE: ${SSL_CERT_FILE:-} + CURL_CA_BUNDLE: ${CURL_CA_BUNDLE:-} depends_on: postgres: condition: service_healthy diff --git a/.devcontainer/scripts/load-aliases.sh b/.devcontainer/scripts/load-aliases.sh index 8628966f55..fc8a979d0c 100755 --- a/.devcontainer/scripts/load-aliases.sh +++ b/.devcontainer/scripts/load-aliases.sh @@ -61,4 +61,7 @@ alias rq-jobs="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && alias rq-failed="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell -c \"from django_rq import get_failed_queue; q = get_failed_queue(); print(f'Failed jobs: {len(q)}'); [print(f' {job.id[:8]}: {job.func_name}') for job in q.jobs[:10]]\"" alias rq-recent="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell -c \"from core.models import Job; jobs = Job.objects.all().order_by('-created')[:10]; [print(f'{j.id}: {j.name[:50]} - {getattr(j.status, \\\"value\\\", j.status)} ({j.user})') for j in jobs]\"" -echo "βœ… Aliases loaded! Try: rq-status, rq-stats, rq-recent" +# Help +alias dev-help='echo "🎯 NetBox LibreNMS Plugin Development Commands:"; echo ""; echo "πŸ“Š NetBox Server Management:"; echo " netbox-run-bg : Start NetBox in background"; echo " netbox-run : Start NetBox in foreground (for debugging)"; echo " netbox-stop : Stop NetBox and RQ worker"; echo " netbox-restart : Restart NetBox and RQ worker"; echo " netbox-reload : Reinstall plugin and restart NetBox"; echo " netbox-status : Check if NetBox and RQ worker are running"; echo " netbox-logs : View NetBox server logs"; echo ""; echo "βš™οΈ Background Jobs (RQ Worker):"; echo " rq-status : Check if RQ worker is running"; echo " rq-logs : View RQ worker logs"; echo " rq-stats : Show RQ queue statistics"; echo " rq-jobs : List jobs in default queue"; echo " rq-failed : List failed jobs"; echo " rq-recent : Show recent NetBox jobs"; echo ""; echo "πŸ› οΈ Development Tools:"; echo " netbox-shell : Open NetBox Django shell"; echo " netbox-test : Run plugin tests"; echo " netbox-manage : Run Django management commands"; echo " plugin-install : Reinstall plugin in development mode"; echo ""; echo "🧹 Code Quality:"; echo " ruff-check : Check code with Ruff"; echo " ruff-format : Format code with Ruff"; echo " ruff-fix : Auto-fix code issues with Ruff"; echo ""; echo "πŸ”Ž Diagnostics:"; echo " diagnose : Run startup diagnostics"; echo " dev-help : Show this help message"; echo ""; echo "πŸ“– NetBox available at: http://localhost:8000 (admin/admin)"; echo ""' + +echo "βœ… Aliases loaded! Try: rq-status, rq-stats, rq-recent, dev-help" diff --git a/.devcontainer/scripts/setup.sh b/.devcontainer/scripts/setup.sh old mode 100644 new mode 100755 index 18da8d24da..0b69b2bf2f --- a/.devcontainer/scripts/setup.sh +++ b/.devcontainer/scripts/setup.sh @@ -7,6 +7,74 @@ echo "πŸ‘€ Current user: $(whoami)" NETBOX_VERSION=${NETBOX_VERSION:-"latest"} echo "πŸ“¦ Using NetBox Docker image: netboxcommunity/netbox:${NETBOX_VERSION}" +# --------------------------------------------------------------------------- +# Detect plugin workspace directory (must contain pyproject.toml). +# Prints the resolved path to stdout on success, or an empty string on +# failure. Always exits 0 β€” callers must check for an empty result. +# --------------------------------------------------------------------------- +detect_plugin_workspace() { + if [ -f "$PWD/pyproject.toml" ]; then + echo "$PWD" + elif [ -d "/workspaces/netbox-librenms-plugin" ] && [ -f "/workspaces/netbox-librenms-plugin/pyproject.toml" ]; then + echo "/workspaces/netbox-librenms-plugin" + else + local candidate + candidate=$(find /workspaces -maxdepth 2 -type f -name pyproject.toml 2>/dev/null | head -n1 | xargs -r dirname || true) + if [ -n "$candidate" ] && [ -f "$candidate/pyproject.toml" ]; then + echo "$candidate" + else + echo "" + fi + fi +} + +# Configure proxy for apt and pip if proxy environment variables are set +if [ -n "$HTTP_PROXY" ] || [ -n "$HTTPS_PROXY" ]; then + echo "🌐 Configuring proxy settings..." + + # Configure apt proxy + if [ -n "$HTTP_PROXY" ]; then + echo "Acquire::http::Proxy \"$HTTP_PROXY\";" > /etc/apt/apt.conf.d/80proxy + echo " βœ“ apt HTTP proxy: $HTTP_PROXY" + fi + if [ -n "$HTTPS_PROXY" ]; then + echo "Acquire::https::Proxy \"$HTTPS_PROXY\";" >> /etc/apt/apt.conf.d/80proxy + echo " βœ“ apt HTTPS proxy: $HTTPS_PROXY" + fi + + # Configure pip proxy via environment (already set, but ensure it's exported) + export HTTP_PROXY HTTPS_PROXY http_proxy https_proxy NO_PROXY no_proxy + + # Install custom CA certificate into the system trust store (for MITM proxies) + PLUGIN_WS_DIR_EARLY="$(detect_plugin_workspace)" + [ -z "$PLUGIN_WS_DIR_EARLY" ] && PLUGIN_WS_DIR_EARLY="/workspaces/netbox-librenms-plugin" + CA_BUNDLE_SRC="$PLUGIN_WS_DIR_EARLY/ca-bundle.crt" + if [ -f "$CA_BUNDLE_SRC" ]; then + echo "πŸ” Installing custom CA certificate into system trust store..." + mkdir -p /usr/local/share/ca-certificates + cp "$CA_BUNDLE_SRC" /usr/local/share/ca-certificates/proxy-ca-bundle.crt + update-ca-certificates 2>/dev/null + echo " βœ“ CA certificate installed into system trust store" + # Point environment variables to the system bundle + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt + else + echo " ℹ️ No ca-bundle.crt found at $CA_BUNDLE_SRC, skipping CA install" + # Only disable git SSL verification if explicitly opted-in via ALLOW_GIT_SSL_DISABLE. + # Silently disabling SSL is a security risk; prefer providing a CA bundle instead. + if [ "${ALLOW_GIT_SSL_DISABLE:-false}" = "true" ]; then + git config --global http.sslVerify false + echo " ⚠️ git SSL verification disabled globally (ALLOW_GIT_SSL_DISABLE=true)" + else + echo " ⚠️ No CA bundle found and git SSL verification was NOT disabled." + echo " If you need to disable it, set ALLOW_GIT_SSL_DISABLE=true in .devcontainer/.env" + echo " Preferred: provide a ca-bundle.crt in the workspace root instead." + fi + fi +fi + # Verify NetBox virtual environment exists if [ ! -f "/opt/netbox/venv/bin/activate" ]; then echo "❌ NetBox virtual environment not found at /opt/netbox/venv/" @@ -27,23 +95,37 @@ fi # Install dev tools echo "πŸ”§ Installing development dependencies..." apt-get update -qq -apt-get install -y -qq net-tools +apt-get install -y -qq net-tools git $PIP_CMD install pytest pytest-django ruff pre-commit -# Detect plugin workspace directory (must contain pyproject.toml) -if [ -f "$PWD/pyproject.toml" ]; then - PLUGIN_WS_DIR="$PWD" -elif [ -d "/workspaces/netbox-librenms-plugin" ] && [ -f "/workspaces/netbox-librenms-plugin/pyproject.toml" ]; then - PLUGIN_WS_DIR="/workspaces/netbox-librenms-plugin" -else - CANDIDATE_DIR=$(find /workspaces -maxdepth 2 -type f -name pyproject.toml 2>/dev/null | head -n1 | xargs dirname || true) - if [ -n "$CANDIDATE_DIR" ] && [ -f "$CANDIDATE_DIR/pyproject.toml" ]; then - PLUGIN_WS_DIR="$CANDIDATE_DIR" - else - echo "❌ Could not locate plugin workspace directory (pyproject.toml not found)." - echo " Checked: $PWD and /workspaces/*" - exit 1 - fi +# Install GitHub CLI (gh) +# NOTE: The chained && commands below mean a partial failure (e.g. wget succeeds +# but apt-get install gh fails) may leave artifacts (keyring, sources list, temp +# file). This is acceptable here because it only runs during container build β€” +# a rebuild will retry from scratch. If this block is ever moved to a runtime +# script, consider adding a trap or explicit cleanup on error. +if ! command -v gh >/dev/null 2>&1; then + echo "πŸ”§ Installing GitHub CLI..." + (type -p wget >/dev/null || apt-get install -y -qq wget) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && out=$(mktemp) \ + && wget -qO "$out" https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + && cat "$out" | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update -qq \ + && apt-get install -y -qq gh \ + && rm -f "$out" \ + && echo " βœ“ GitHub CLI installed: $(gh --version | head -1)" \ + || echo "⚠️ GitHub CLI installation failed (non-fatal)" +fi + +# Detect plugin workspace directory using the shared helper +PLUGIN_WS_DIR="$(detect_plugin_workspace)" +if [ -z "$PLUGIN_WS_DIR" ]; then + echo "❌ Could not locate plugin workspace directory (pyproject.toml not found)." + echo " Checked: $PWD and /workspaces/*" + exit 1 fi echo "πŸ“‚ Plugin workspace: $PLUGIN_WS_DIR" @@ -160,42 +242,27 @@ python manage.py collectstatic --noinput >/dev/null 2>&1 || true # Set up pre-commit hooks echo "πŸͺ Installing pre-commit hooks..." cd "$PLUGIN_WS_DIR" +git config --global --add safe.directory "$PLUGIN_WS_DIR" pre-commit install --install-hooks 2>/dev/null || echo "⚠️ Pre-commit hook installation failed (may already be installed)" # Ensure scripts are executable chmod +x "$PLUGIN_WS_DIR/.devcontainer/scripts/start-netbox.sh" || true chmod +x "$PLUGIN_WS_DIR/.devcontainer/scripts/diagnose.sh" || true +chmod +x "$PLUGIN_WS_DIR/.devcontainer/scripts/load-aliases.sh" || true -# Aliases for convenience -cat >> ~/.bashrc << EOF -# NetBox LibreNMS Plugin Development Aliases -export PATH="/opt/netbox/venv/bin:\$PATH" -export DEBUG="\${DEBUG:-True}" -PLUGIN_DIR="$PLUGIN_WS_DIR" -alias netbox-run-bg="\$PLUGIN_DIR/.devcontainer/scripts/start-netbox.sh --background" -alias netbox-run="\$PLUGIN_DIR/.devcontainer/scripts/start-netbox.sh" -alias netbox-restart="netbox-stop && sleep 1 && netbox-run-bg" -alias netbox-reload="cd \$PLUGIN_DIR && (command -v uv >/dev/null 2>&1 && uv pip install -e . || pip install -e .) && netbox-restart" -alias netbox-stop="([ -f /tmp/netbox.pid ] && kill \\\$(cat /tmp/netbox.pid) && rm /tmp/netbox.pid && echo 'NetBox stopped' || echo 'NetBox not running'); ([ -f /tmp/rqworker.pid ] && kill \\\$(cat /tmp/rqworker.pid) && rm /tmp/rqworker.pid && echo 'RQ worker stopped' || echo 'RQ worker not running')" -alias netbox-logs="tail -f /tmp/netbox.log" -alias netbox-status="[ -f /tmp/netbox.pid ] && kill -0 \\\$(cat /tmp/netbox.pid) 2>/dev/null && echo 'NetBox is running (PID: '\\\$(cat /tmp/netbox.pid)')' || echo 'NetBox is not running'; [ -f /tmp/rqworker.pid ] && kill -0 \\\$(cat /tmp/rqworker.pid) 2>/dev/null && echo 'RQ worker is running (PID: '\\\$(cat /tmp/rqworker.pid)')' || echo 'RQ worker is not running'" -alias rq-logs="tail -f /tmp/rqworker.log" -alias rq-status="[ -f /tmp/rqworker.pid ] && kill -0 \\\$(cat /tmp/rqworker.pid) 2>/dev/null && echo 'RQ worker is running (PID: '\\\$(cat /tmp/rqworker.pid)')' || echo 'RQ worker is not running'" -alias netbox-shell="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py shell" -alias netbox-test="cd \$PLUGIN_DIR && source /opt/netbox/venv/bin/activate && python -m pytest" -alias netbox-manage="cd /opt/netbox/netbox && source /opt/netbox/venv/bin/activate && python manage.py" -alias plugin-install="cd \$PLUGIN_DIR && (command -v uv >/dev/null 2>&1 && uv pip install -e . || pip install -e .)" -alias ruff-check="cd \$PLUGIN_DIR && ruff check ." -alias ruff-format="cd \$PLUGIN_DIR && ruff format ." -alias ruff-fix="cd \$PLUGIN_DIR && ruff check --fix ." -alias diagnose="\$PLUGIN_DIR/.devcontainer/scripts/diagnose.sh" -alias plugins-install='if [ -f "$PLUGIN_DIR/.devcontainer/extra-requirements.txt" ]; then source /opt/netbox/venv/bin/activate && pip install -r "$PLUGIN_DIR/.devcontainer/extra-requirements.txt"; else echo "No .devcontainer/extra-requirements.txt found"; fi' -# Help -alias dev-help='echo "🎯 NetBox LibreNMS Plugin Development Commands:"; echo ""; echo "πŸ“Š NetBox Server Management:"; echo " netbox-run-bg : Start NetBox in background"; echo " netbox-run : Start NetBox in foreground (for debugging)"; echo " netbox-stop : Stop NetBox and RQ worker"; echo " netbox-restart : Restart NetBox and RQ worker"; echo " netbox-reload : Reinstall plugin and restart NetBox"; echo " netbox-status : Check if NetBox and RQ worker are running"; echo " netbox-logs : View NetBox server logs"; echo ""; echo "βš™οΈ Background Jobs (RQ Worker):"; echo " rq-status : Check if RQ worker is running"; echo " rq-logs : View RQ worker logs"; echo ""; echo "πŸ› οΈ Development Tools:"; echo " netbox-shell : Open NetBox Django shell"; echo " netbox-test : Run plugin tests"; echo " netbox-manage : Run Django management commands"; echo " plugin-install : Reinstall plugin in development mode"; echo ""; echo "🧹 Code Quality:"; echo " ruff-check : Check code with Ruff"; echo " ruff-format : Format code with Ruff"; echo " ruff-fix : Auto-fix code issues with Ruff"; echo ""; echo "πŸ”Ž Diagnostics:"; echo " diagnose : Run startup diagnostics"; echo " dev-help : Show this help message"; echo ""; echo "πŸ“– NetBox available at: http://localhost:8000 (admin/admin)"; echo ""' +# Load aliases and welcome message from the canonical source (load-aliases.sh). +# Appended to .bashrc so every interactive shell gets them automatically. +# Guard with a sentinel so rerunning setup.sh doesn't create duplicate entries. +BASHRC_SENTINEL="# NetBox LibreNMS Plugin β€” source aliases from the single canonical file" +if ! grep -qF "$BASHRC_SENTINEL" ~/.bashrc 2>/dev/null; then + cat >> ~/.bashrc << EOF +$BASHRC_SENTINEL +source "$PLUGIN_WS_DIR/.devcontainer/scripts/load-aliases.sh" # Show welcome message for new terminals -bash $PLUGIN_DIR/.devcontainer/scripts/welcome.sh +bash "$PLUGIN_WS_DIR/.devcontainer/scripts/welcome.sh" EOF +fi # Fix Git remote URLs for dev container compatibility echo "πŸ”§ Checking Git remote configuration..." @@ -221,4 +288,5 @@ else echo "⚠️ Warning: Plugin may not be properly installed" fi +echo "" echo "πŸš€ NetBox LibreNMS Plugin Dev Environment Ready!" diff --git a/.devcontainer/scripts/welcome.sh b/.devcontainer/scripts/welcome.sh index c9b811e62c..2a6552f93d 100755 --- a/.devcontainer/scripts/welcome.sh +++ b/.devcontainer/scripts/welcome.sh @@ -46,6 +46,7 @@ fi echo "" echo "πŸš€ Quick start:" echo " β€’ Type 'netbox-run' to start the development server" +echo " β€’ Type 'netbox-restart' to restart NetBox (after config changes)" echo " β€’ Type 'dev-help' to see all available commands" echo " β€’ Edit code in the workspace - auto-reload is enabled" echo "" diff --git a/.gitignore b/.gitignore index dd2caaa5f7..538eb6f7e9 100644 --- a/.gitignore +++ b/.gitignore @@ -285,3 +285,7 @@ cython_debug/ .devcontainer/config/plugin-config.py .devcontainer/config/extra-configuration.py .devcontainer/config/extra-plugins.py +# Proxy CA certificates (keep local) +ca-bundle.crt +*.pem +.github/hooks/ diff --git a/docs/usage_tips/custom_field.md b/docs/usage_tips/custom_field.md index 9934fbd232..80a50b2a7e 100644 --- a/docs/usage_tips/custom_field.md +++ b/docs/usage_tips/custom_field.md @@ -4,6 +4,9 @@ To enhance device identification and synchronization between NetBox and LibreNMS, this plugin supports using a custom field `librenms_id` on Device, Virtual Machine and Interface objects. While the plugin works without it, using this custom field is recommended for LibreNMS API lookups, and to assist with matching the remote device and remote interfaces for cable creation in Netbox. It can also be entered manually if no primary IP or FQDN is available. +!!! info "Automatic Creation" + As of version 0.4.2, the plugin **automatically creates** the `librenms_id` custom field when migrations are run. You no longer need to create it manually. The field is created for Device, Virtual Machine, Interface, and VM Interface objects. + For the Device and Virtual Machine objects the plugin will automatically populate the LibreNMS ID custom field when opening the LibreNMS Sync page if the device has been found in LibreNMS. For the Interface object, the plugin will automatically populate the LibreNMS ID custom field when the interface data is synced from LibreNMS. @@ -15,7 +18,10 @@ For the Interface object, the plugin will automatically populate the LibreNMS ID - **Efficient Synchronization:** Enhances the reliability of API lookups. - **Cable creation:** Allows better device identification for the creation of cables between NetBox devices. -## Suggested Custom Field Setup +## Manual Custom Field Setup (Legacy) + +!!! note + This section is only needed if you are running an older version of the plugin that does not auto-create the field, or if you need to recreate it after deletion. Follow these steps to create the `librenms_id` custom field in NetBox: diff --git a/netbox_librenms_plugin/__init__.py b/netbox_librenms_plugin/__init__.py index f1720c85d6..5a2c2532c2 100644 --- a/netbox_librenms_plugin/__init__.py +++ b/netbox_librenms_plugin/__init__.py @@ -28,6 +28,7 @@ def ready(self): super().ready() from django.conf import settings + from django.db.models.signals import post_migrate plugin_config = getattr(settings, "PLUGINS_CONFIG", {}).get(self.name, {}) @@ -37,6 +38,12 @@ def ready(self): else: self._validate_legacy_config(plugin_config) + # Auto-create the librenms_id custom field after migrations complete + post_migrate.connect( + _ensure_librenms_id_custom_field, + dispatch_uid="netbox_librenms_plugin_ensure_cf", + ) + def _validate_multi_server_config(self, servers_config): """Validate multi-server configuration.""" if not servers_config or not isinstance(servers_config, dict): @@ -61,4 +68,63 @@ def _validate_legacy_config(self, plugin_config): ) +def _ensure_librenms_id_custom_field(sender, **kwargs): + """ + Auto-create the 'librenms_id' custom field if it doesn't exist. + Runs after migrations via post_migrate signal to ensure tables exist. + Uses dispatch_uid to avoid duplicate connections. + """ + # Only run once per migrate invocation (post_migrate fires per-app). + # The _executed flag is intentionally never reset: migrations are expected to + # run in short-lived CLI processes (manage.py migrate) where the flag is + # naturally cleared on exit. Long-running processes (e.g. gunicorn workers) + # should not rely on this handler re-executing after startup. + if getattr(_ensure_librenms_id_custom_field, "_executed", False): + return + _ensure_librenms_id_custom_field._executed = True # not reset; see comment above + + try: + from django.contrib.contenttypes.models import ContentType + + from extras.models import CustomField + + cf, created = CustomField.objects.get_or_create( + name="librenms_id", + defaults={ + "type": "integer", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Ensure the field is assigned to the required object types + from dcim.models import Device, Interface + from virtualization.models import VirtualMachine, VMInterface + + required_models = [Device, VirtualMachine, Interface, VMInterface] + current_types = set(cf.object_types.values_list("pk", flat=True)) + + for model in required_models: + ct = ContentType.objects.get_for_model(model) + if ct.pk not in current_types: + cf.object_types.add(ct) + + if created: + import logging + + logging.getLogger("netbox_librenms_plugin").info( + "Auto-created 'librenms_id' custom field for Device, VirtualMachine, Interface, VMInterface" + ) + except Exception as e: + # Don't break startup if custom field creation fails (e.g., during initial migration), + # but log the error so it's not silently swallowed. + import logging + + logging.getLogger("netbox_librenms_plugin").exception("Failed to auto-create 'librenms_id' custom field: %s", e) + + config = LibreNMSSyncConfig diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index e58cf26153..9a5fc3488b 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -37,11 +37,31 @@ def __init__(self, server_key=None): # Default to 'default' if still no server_key server_key = server_key or "default" - self.server_key = server_key # Get server configuration servers_config = get_plugin_config("netbox_librenms_plugin", "servers") + # If the requested server_key doesn't exist but there are configured servers, + # only fall back to the first available server when using the auto-default key. + # If a specific (non-default) server_key was requested but not found, raise + # immediately to avoid silently using the wrong LibreNMS instance. + if servers_config and isinstance(servers_config, dict) and server_key not in servers_config: + if server_key != "default": + available = list(servers_config.keys()) + raise KeyError( + f"Server '{server_key}' not found in LibreNMS plugin configuration. Available servers: {available}" + ) + first_key = next(iter(servers_config), None) + if first_key: + logger.info( + "Server '%s' not found in config, falling back to '%s'", + server_key, + first_key, + ) + server_key = first_key + + self.server_key = server_key + if servers_config and isinstance(servers_config, dict) and server_key in servers_config: # Multi-server configuration config = servers_config[server_key] diff --git a/netbox_librenms_plugin/tests/test_init.py b/netbox_librenms_plugin/tests/test_init.py new file mode 100644 index 0000000000..ed5612f81d --- /dev/null +++ b/netbox_librenms_plugin/tests/test_init.py @@ -0,0 +1,170 @@ +"""Tests for netbox_librenms_plugin.__init__ module. + +Covers the _ensure_librenms_id_custom_field post_migrate signal handler. +""" + +from unittest.mock import MagicMock, patch + + +# ============================================================================= +# TestEnsureLibreNMSIdCustomField - 6 tests +# ============================================================================= + + +class TestEnsureLibreNMSIdCustomField: + """Test _ensure_librenms_id_custom_field signal handler.""" + + def setup_method(self): + """Reset the _executed flag before each test for consistent isolation.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = False + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_creates_custom_field_when_missing( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """Custom field is created with correct defaults when it does not exist.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [] + MockCustomField.objects.get_or_create.return_value = (mock_cf, True) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + + MockCustomField.objects.get_or_create.assert_called_once_with( + name="librenms_id", + defaults={ + "type": "integer", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Should have added content types for all 4 models + assert mock_cf.object_types.add.call_count == 4 + + # Should log when created + mock_get_logger.assert_called_with("netbox_librenms_plugin") + + def test_skips_when_already_executed(self): + """Handler is a no-op on second invocation (per-migrate dedup).""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = True + + with patch("extras.models.CustomField") as MockCustomField: + _ensure_librenms_id_custom_field(sender=None) + MockCustomField.objects.get_or_create.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_existing_field_not_recreated( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When custom field already exists, it is not recreated but types are checked.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + _ensure_librenms_id_custom_field(sender=None) + + # All pks already present, no types should be added + mock_cf.object_types.add.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_adds_missing_content_types( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When some content types are missing, only those are added.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + ct_existing = MagicMock() + ct_existing.pk = 1 + ct_new = MagicMock() + ct_new.pk = 99 + MockContentType.objects.get_for_model.side_effect = [ct_existing, ct_existing, ct_new, ct_new] + + _ensure_librenms_id_custom_field(sender=None) + + assert mock_cf.object_types.add.call_count == 2 + mock_cf.object_types.add.assert_any_call(ct_new) + + @patch("extras.models.CustomField") + def test_exception_does_not_propagate(self, MockCustomField): + """Exceptions during custom field creation are caught and logged.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + MockCustomField.objects.get_or_create.side_effect = Exception("DB not ready") + + with patch("logging.getLogger") as mock_get_logger: + # Should not raise + _ensure_librenms_id_custom_field(sender=None) + + # Verify the exception was logged + logger_instance = mock_get_logger.return_value + logger_instance.exception.assert_called_once() + call_args = logger_instance.exception.call_args + assert "librenms_id" in call_args[0][0] + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_no_log_when_field_already_exists( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """No log message when the custom field already existed.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + # When the field already exists (created=False), the info log should + # not be emitted. We verify via the logger instance rather than + # asserting getLogger was never called, which is fragile. + logger_instance = mock_get_logger.return_value + logger_instance.info.assert_not_called()