SDK | Structured logging and progress bars

gateway/config/settings/base.py

@@ -8,6 +8,7 @@
 from urllib.parse import urlparse
 
 from celery.schedules import crontab
+from celery.schedules import schedule
 from environs import env
 
 from config.settings.logs import ColoredFormatter
@@ -543,7 +544,7 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str:
     },
     "monitor-services-health": {
         "task": "sds_gateway.monitoring.tasks.monitor_services_health",
-        "schedule": crontab(minute="*/1"),
+        "schedule": schedule(run_every=60),
         "options": {"expires": 50},
     },
 }

gateway/docs/changelog.md

@@ -74,13 +74,13 @@
 
 - Fixes:
     - [**Added M2M relationships for assets to
-        datasets**](https://github.com/spectrumx/sds-code/pull/226): Currently the
-        foreign key (FK) relationships only allow assets to be connected to a single
-        dataset. This PR expands the schema (with future contraction planned) to correct
-        the schema issues.
+      datasets**](https://github.com/spectrumx/sds-code/pull/226): Currently the
+      foreign key (FK) relationships only allow assets to be connected to a single
+      dataset. This PR expands the schema (with future contraction planned) to correct
+      the schema issues.
         - Requires a data migration for existing datasets (and captures) from the FK
-            field to M2M field. Run management command `migrate_fk_to_m2m` to transfer
-            foreign key model relationship to many-to-many field (only adds, does NOT
-            nullify FK).
+          field to M2M field. Run management command `migrate_fk_to_m2m` to transfer
+          foreign key model relationship to many-to-many field (only adds, does NOT
+          nullify FK).
         - This is meant to be run on systems with data under the current schema that
-            require migrations to new schema.
+          require migrations to new schema.

gateway/docs/dev-notes.md

@@ -45,22 +45,22 @@ migrations, or before upgrading Postgres versions.
 Up to one snapshot per day of:
 
 + [x] **PostgreSQL database** - Complete database dump via `pg_dumpall` (all schemas,
-    tables, and data)
+  tables, and data)
 + [x] **OpenSearch indices** - Snapshots of all indices and their data
 + [x] **Gateway secrets** - Environment files (`.envs/`, `.env.sh`, etc.) and
-    configuration files
+  configuration files
 + [x] **Git repository state** - Branch info, commit SHA, uncommitted changes, staged
-    changes, and commit archive.
+  changes, and commit archive.
 
 ### What is NOT backed up
 
 + ❌ **MinIO objects** - S3-compatible storage data is not included in snapshots (see
-    warning below). When configuring MinIO, you can configure erasure coding for data
-    redundancy across multiple drives or nodes. Consider using MinIO's own backup and
-    replication features for protecting this data instead of this snapshot procedure.
+  warning below). When configuring MinIO, you can configure erasure coding for data
+  redundancy across multiple drives or nodes. Consider using MinIO's own backup and
+  replication features for protecting this data instead of this snapshot procedure.
 + ❌ **Docker volumes** - Any data stored in Docker volumes outside of the database
-    container is not included, such as temporary zip files, `uv` cache, virtual
-    environments, generated static files.
+  container is not included, such as temporary zip files, `uv` cache, virtual
+  environments, generated static files.
 
 ### Creating Backups
 
@@ -69,17 +69,17 @@ Up to one snapshot per day of:
 3. Run `just snapshot` to create a _daily_ backup snapshot of Postgres database.
     + See `scripts/create-snapshot.sh` for details of what is executed.
     + Database credentials are automatically loaded from the postgres container, so the
-        user running the snapshot must have privileges to manage the database container,
-        and the container must be running with those env variables set.
+      user running the snapshot must have privileges to manage the database container,
+      and the container must be running with those env variables set.
     + The Postgres snapshot is created with `pg_dumpall`. See the docs for details:
-        [Postgres docs](https://postgresql.org/docs/current/app-pg-dumpall.html).
+      [Postgres docs](https://postgresql.org/docs/current/app-pg-dumpall.html).
     + The snapshot creation is non-interactive, so it can be run from automated scripts;
       but interaction might be required for backup exfiltration.
     + **Note:** MinIO objects are not included in this snapshot. See [What is NOT backed
       up](#what-is-not-backed-up) above.
 4. Access the created files in `./data/backups/`.
     + Note you can set up a remote server in `.env.sh` to automatically copy the backups
-    there after creation using `rsync` over `ssh`.
+   there after creation using `rsync` over `ssh`.
 
 ### Backups Restoration
 
@@ -95,7 +95,7 @@ READ THIS ENTIRE SECTION BEFORE RESTORING A BACKUP.
 What can be automatically restored:
 
 + [x] **PostgreSQL database** - Full restoration of the database from the latest
-    snapshot. See [Backups Restoration](#backups-restoration) below.
+  snapshot. See [Backups Restoration](#backups-restoration) below.
 
 Everything else (OpenSearch indices, secrets, git state) can be manually restored from
 the snapshot files, but there is no automated restoration procedure for them yet.
@@ -104,11 +104,11 @@ Note that, while the database can be restored, it will contain references to oth
 sources that might exist or not. Most notably:
 
 + OpenSearch indices - The restored database might contain references to documents in
-    OpenSearch that do not exist if those indices were not manually restored.
+  OpenSearch that do not exist if those indices were not manually restored.
 + MinIO objects - The restored database might contain references to objects in MinIO
-    that do not exist if those objects were not manually restored. This is the main
-    challenge when restoring a production backup to a staging/QA machine, as most
-    objects will not exist there due to the commonly large size of object data.
+  that do not exist if those objects were not manually restored. This is the main
+  challenge when restoring a production backup to a staging/QA machine, as most
+  objects will not exist there due to the commonly large size of object data.
 
 Run `scripts/restore-snapshot.sh` when you are ready to restore the most recent snapshot.
 
@@ -123,9 +123,9 @@ production environment.
 ## MinIO Configuration
 
 + MinIO config for SDS: [see the production deploy
-    instructions](./detailed-deploy.md#production-deploy).
+  instructions](./detailed-deploy.md#production-deploy).
 + [MinIO reference
-    document](https://github.com/minio/minio/blob/master/docs/config/README.md).
+  document](https://github.com/minio/minio/blob/master/docs/config/README.md).
 
 ## OpenSearch Cluster Maintenance
 

gateway/sds_gateway/monitoring/tests/test_celery_schedule.py

@@ -1,4 +1,4 @@
-from celery.schedules import crontab
+from celery.schedules import schedule
 from django.conf import settings
 from django.utils import timezone
 
@@ -12,7 +12,7 @@ def test_monitoring_task_is_in_celery_beat_schedule() -> None:
         schedule_config["task"]
         == "sds_gateway.monitoring.tasks.monitor_services_health"
     )
-    assert isinstance(schedule_config["schedule"], crontab)
+    assert isinstance(schedule_config["schedule"], schedule)
     _is_due, next_run_seconds = schedule_config["schedule"].is_due(timezone.now())
     check_period_sec: int = 60
     expected_expires_sec = check_period_sec - 10

sdk/docs/README.md

@@ -338,3 +338,28 @@ If this is needed, SDK users have a few options:
 
 One writer (an SDK client that creates, updates, and/or deletes contents) and multiple
 readers are generally safe.
+
+## Structured Logging
+
+The SDK writes structured JSONL logs to `~/.local/state/spectrumx/logs/YYYY-MM-DD.jsonl`.
+Each line is a JSON object with fields for timestamp, severity, category, and message.
+Filter them with `jq`:
+
+```bash
+# Show only filesystem operations
+jq 'select(.cat == "filesystem")' ~/.local/state/spectrumx/logs/*.jsonl
+
+# Show only download operations
+jq 'select(.cat == "download")' ~/.local/state/spectrumx/logs/*.jsonl
+
+# Show warnings and errors
+jq 'select(.lvl == "WARNING" or .lvl == "ERROR")' ~/.local/state/spectrumx/logs/*.jsonl
+
+# Compact view per line
+jq -r '"\(.ts[11:19]) [\(.lvl)] [\(.cat)] \(.msg)"' ~/.local/state/spectrumx/logs/*.jsonl
+
+# Count messages per category
+jq -r '.cat' ~/.local/state/spectrumx/logs/*.jsonl | sort | uniq -c | sort -rn
+```
+
+See the [full structured logging docs](https://sds.crc.nd.edu/sdk/advanced/structured-logging/) for more examples.

sdk/docs/mkdocs.yaml

@@ -25,6 +25,7 @@ nav:
     - "Changelog": "changelog.md"
     - "Advanced":
           - "Concurrent Access": "advanced/concurrent-access.md"
+          - "Structured Logging": "advanced/structured-logging.md"
     - "API Reference":
           - "Client": "api/client.md"
           - "Errors": "api/errors.md"

sdk/docs/mkdocs/advanced/structured-logging.md

@@ -0,0 +1,101 @@
+# Structured Logging
+
+The SDK writes structured JSONL logs to `~/.local/state/spectrumx/logs/YYYY-MM-DD.jsonl`.
+Each line is a JSON object with fields like `ts`, `pid`, `lvl`, `cat`, and `msg`.
+
+## Filtering with jq
+
+Use [`jq`](https://jqlang.org/) to filter and analyze log files on the command line.
+
+### By category
+
+```bash
+# Filesystem operations only
+jq 'select(.cat == "filesystem")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Network operations
+jq 'select(.cat == "network")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Upload-specific messages
+jq 'select(.cat == "upload")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Download-specific messages
+jq 'select(.cat == "download")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Auth messages
+jq 'select(.cat == "auth")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Config messages
+jq 'select(.cat == "config")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+```
+
+### By severity
+
+```bash
+# Warnings and errors only
+jq 'select(.lvl == "WARNING" or .lvl == "ERROR")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Errors only with full exception info
+jq 'select(.lvl == "ERROR")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+```
+
+### Compound filters
+
+```bash
+# Filesystem warnings and errors
+jq 'select(.cat == "filesystem" and (.lvl == "WARNING" or .lvl == "ERROR"))' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Everything except generic "log" category
+jq 'select(.cat != "log")' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Messages with exception details
+jq 'select(.exc_info != null)' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+```
+
+### Aggregations
+
+```bash
+# Count messages per category
+jq -r '.cat' ~/.local/state/spectrumx/logs/2026-06-29.jsonl | sort | uniq -c | sort -rn
+
+# Count messages per severity level
+jq -r '.lvl' ~/.local/state/spectrumx/logs/2026-06-29.jsonl | sort | uniq -c | sort -rn
+
+# Extract just the messages for a category
+jq -r 'select(.cat == "network") | .msg' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+```
+
+### Pretty-print and format
+
+```bash
+# Colorized output (jq highlights strings by default)
+jq . ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Compact view: timestamp + level + category + message
+jq -r '"\(.ts[11:19]) [\(.lvl)] [\(.cat)] \(.msg)"' ~/.local/state/spectrumx/logs/2026-06-29.jsonl
+
+# Tabular output with header
+jq -r '["TS","LVL","CAT","MSG"], ["---","---","---","---"], (.[] | [.ts[11:19], .lvl, .cat, .msg]) | @tsv' ~/.local/state/spectrumx/logs/2026-06-29.jsonl | column -t -s $'\t'
+```
+
+### Watch in real time
+
+```bash
+# Tail and filter: show only filesystem logs as they arrive
+tail -f ~/.local/state/spectrumx/logs/2026-06-29.jsonl | jq 'select(.cat == "filesystem")'
+
+# Tail with compact format
+tail -f ~/.local/state/spectrumx/logs/2026-06-29.jsonl | jq -r '"\(.ts[11:19]) [\(.lvl)] [\(.cat)] \(.msg)"'
+```
+
+## Log categories reference
+
+| Category     | Description                     | Examples                              |
+|--------------|---------------------------------|---------------------------------------|
+| `log`        | Generic/default messages        | Dry-run notices, user-facing messages |
+| `config`     | SDK configuration               | Config loading, env variable warnings |
+| `auth`       | Authentication operations       | Token validation, auth success/failure|
+| `network`    | HTTP/network operations         | Response info, connection errors      |
+| `filesystem` | File and capture CRUD           | Upload/download paths, capture ops    |
+| `download`   | File download operations        | Download skip/reason, dry-run notices, path resolution |
+| `upload`     | File upload metadata operations | Upload status, metadata updates       |

sdk/justfile

@@ -116,6 +116,7 @@ pyrefly *args:
 test python=python_version *pytest_args:
     #!/usr/bin/env bash
     set -euo pipefail
+    export COLUMNS=88
     echo -e "\n\t\033[34mRunning tests against Python {{ python }}\033[0m\n"
 
     # no coverage if -k is used (likely a focused test run)
@@ -145,6 +146,7 @@ test python=python_version *pytest_args:
 test-all *args:
     #!/usr/bin/env bash
     set -euo pipefail
+    COLUMNS=80
     echo -e "\n\t\033[34mRunning all local (non-integration) tests\033[0m\n"
 
     # run higher bound tests against default python version
@@ -169,7 +171,7 @@ test-integration python=python_version workers="auto" *pytest_args:
     @echo -e "\n\t\033[33mRunning integration tests in parallel\033[0m"
     @echo -e "\t\033[33m  Workers: {{ workers }}\033[0m"
     @echo -e "\t\033[33m  Pass -n 1 or use 'just test-integration-sequential' for sequential execution\033[0m\n"
-    uv run --resolution highest -p '{{ python }}' pytest \
+    COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \
         -m '{{ test_marker_integration }}' \
         -n {{ workers }} \
         --dist loadgroup \
@@ -181,7 +183,7 @@ test-integration python=python_version workers="auto" *pytest_args:
 test-integration-sequential python=python_version *pytest_args:
     @echo -e "\n\t\033[33mRunning integration tests sequentially\033[0m"
     @echo -e "\t\033[33m  Use 'just test-integration' for parallel execution\033[0m\n"
-    uv run --resolution highest -p '{{ python }}' pytest \
+    COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \
         -m '{{ test_marker_integration }}' \
         {{ pytest_args }} \
         tests
@@ -190,15 +192,15 @@ test-integration-sequential python=python_version *pytest_args:
 [group('qa')]
 test-lowest python=python_version *pytest_args:
     @echo "Running lowest dep resolution tests for Python '{{ python }}'"
-    uv run --resolution lowest-direct -p '{{ python }}' pytest \
+    COLUMNS=88 uv run --resolution lowest-direct -p '{{ python }}' pytest \
         -m '{{ test_marker_default }}' \
         {{ pytest_args }} \
         tests
 
 # runs local tests in verbose mode with stdout capture; useful for debugging test failures
 [group('qa')]
 test-verbose python=python_version *pytest_args:
-    uv run --resolution highest -p '{{ python }}' pytest \
+    COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \
         -vvv \
         -m '{{ test_marker_default }}' \
         --show-capture=stdout \
@@ -213,7 +215,7 @@ test-verbose python=python_version *pytest_args:
 test-integration-verbose python=python_version *pytest_args:
     @echo -e "\n\t\033[33mRunning integration tests: make sure you set up the web application\033[0m"
     @echo -e "\t\033[33mand configure 'tests/integration/integration.env'\033[0m\n"
-    uv run --resolution highest -p '{{ python }}' pytest \
+    COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \
         -m '{{ test_marker_integration }}' \
         -vvv \
         --capture=no \
@@ -226,7 +228,7 @@ test-integration-quick python=python_version workers="auto" *pytest_args:
     @echo -e "\n\t\033[33mRunning quick integration tests in parallel (skipping heavy tests)\033[0m"
     @echo -e "\t\033[33m  Workers: {{ workers }}\033[0m"
     @echo -e "\t\033[33m  Marker: {{ test_marker_integration }} and not heavy\033[0m\n"
-    uv run --resolution highest -p '{{ python }}' pytest \
+    COLUMNS=88 uv run --resolution highest -p '{{ python }}' pytest \
         -m '{{ test_marker_integration }} and not heavy' \
         -n {{ workers }} \
         --dist loadgroup \
@@ -258,3 +260,8 @@ publish:
 update:
     uv sync --upgrade --dev
     uv run prek autoupdate
+
+# runs the progress bars demo with the given file count and rate
+[group('development')]
+demo-progress-bars count="9" rate="10":
+    uv run scripts/demo_progress_bars.py --file-count {{ count }} --rate {{ rate }}

sdk/pyproject.toml

@@ -419,6 +419,9 @@
             # Tests commonly use boolean fixtures as positional args — intentional
             "sdk/tests/**" = ["FBT001", "FBT002"]
 
+            # Demo scripts intentionally use print(), lazy imports, private member access for mocking
+            "sdk/scripts/**" = ["T201", "PLC0415", "SLF001", "S108", "RUF001"]
+
     [tool.ruff.format]
         indent-style              = "space"
         line-ending               = "auto"