User-dataset collection + admission gatekeeper

.github/workflows/ci.yml

@@ -69,7 +69,7 @@ jobs:
       # tests need. numpy is required because constants.py imports it; requests
       # because Authlib's Flask client imports it.
       - name: Install test deps
-        run: python -m pip install --upgrade pip pytest werkzeug SQLAlchemy Flask-SQLAlchemy alembic argon2-cffi Authlib requests flask-cors numpy python-dotenv
+        run: python -m pip install --upgrade pip pytest werkzeug SQLAlchemy Flask-SQLAlchemy alembic argon2-cffi Authlib requests flask-cors numpy nibabel python-dotenv
 
       # Catches syntax errors anywhere in the backend (api_blueprint.py etc.)
       # without installing the heavy runtime deps - nothing else in CI parses
@@ -80,6 +80,11 @@ jobs:
       - name: Path-safety unit tests
         run: python -m pytest tests/unit/test_path_safety.py -v
 
+      # User-dataset admission gatekeeper: quotas, dedup, CT/segmentation validity,
+      # promotion into the PanTS-mirroring layout.
+      - name: User-dataset gatekeeper unit tests
+        run: python -m pytest tests/unit/test_user_dataset.py -v
+
       # Chunked-upload staging area: which chunks the server still holds (what a
       # resuming client checks its cursor against) and the stale-upload sweep.
       - name: Chunk-store unit tests

flask-server/api/api_blueprint.py

@@ -1032,6 +1032,11 @@ def _start_auto_segmentation(session_id, model_name, ct_file=None, server_input_
     user = current_user()
     if user is None:
         return jsonify({"error": "Sign in to run inference"}), 401
+    # Captured in the request context so the background worker (which has none) can
+    # attribute the scan for per-IP quotas in the user-dataset gatekeeper. Use
+    # remote_addr (set by the trusted reverse proxy) rather than a client-supplied
+    # X-Forwarded-For, which an attacker could rotate per request to defeat the cap.
+    _collector_ip = request.remote_addr or ""
     blocked = plan_store.check_inference(user["id"], model_name)
     if blocked is not None:
         # 402 Payment Required: the request is well-formed and the user is
@@ -1118,6 +1123,20 @@ def _on_gpu_slot():
             _set_inference_job(session_id, status="completed", error=None,
                                zip_path=zip_path, output_mask_dir=output_mask_dir)
             print(f"✅ Finished segmentation and zipping for session {session_id}")
+
+            # Non-blocking: offer this scan+mask to the user-dataset gatekeeper,
+            # which decides (async) whether it's worth keeping. The result is
+            # already delivered above; this never affects the user, and is a no-op
+            # unless USER_DATASET_PATH is configured.
+            try:
+                from services.user_dataset import collect_user_scan_async
+                collect_user_scan_async(
+                    ct_path=input_path, output_mask_dir=output_mask_dir,
+                    model=model_name, user_id=user.get("id"), ip=_collector_ip,
+                    session_id=session_id,
+                )
+            except Exception as _ude:
+                print(f"[user_dataset] hook error (non-fatal): {_ude}")
         except Exception as e:
             # A killed subprocess surfaces here as CalledProcessError/RuntimeError;
             # if the user cancelled, keep "cancelled" rather than reporting failure.

flask-server/constants.py

@@ -33,6 +33,11 @@ class Constants:
     CANCERVERSE_PATH = os.environ.get('CANCERVERSE_PATH')
     CANCERVERSE_LOWRES_PATH = os.environ.get('CANCERVERSE_LOWRES_PATH', '/home/visitor/cancerverse_lowres')
     DATASET_PREFIXES = {'PanTS': 'PanTS', 'CancerVerse': 'CV'}
+    # Where accepted user scans (CT + mask + sublabels) are collected, in a
+    # PanTS-mirroring layout (image_only/, mask_only/). Unset => the collection
+    # gatekeeper (services/user_dataset.py) is a no-op. Point it at a writable
+    # staging dir now; relocate beside PanTS/CancerVerse once write access lands.
+    USER_DATASET_PATH = os.environ.get('USER_DATASET_PATH')
     PERMISSIONS_DIR = os.environ.get('PERMISSIONS_DIR', "/home/visitor/data")
     MESH_PATH = PERMISSIONS_DIR + "/render_only"
     CASE_QUALITY_MANIFEST = os.environ.get('BODYMAPS_CASE_QUALITY_MANIFEST')