diff --git a/.github/workflows/dify-plugin.yml b/.github/workflows/dify-plugin.yml index d26d582971..ea53eea923 100644 --- a/.github/workflows/dify-plugin.yml +++ b/.github/workflows/dify-plugin.yml @@ -4,11 +4,17 @@ on: pull_request: paths: - integrations/dify/** + - sdk/python/** + - scripts/gates/dify* + - tests/unit/test_dify_application_guards.py - .github/workflows/dify-plugin.yml push: branches: [main] paths: - integrations/dify/** + - sdk/python/** + - scripts/gates/dify* + - tests/unit/test_dify_application_guards.py - .github/workflows/dify-plugin.yml permissions: @@ -38,3 +44,51 @@ jobs: with: name: loom-dify-plugin path: loom-0.1.0.difypkg + + application-smoke: + name: Dify CE native application smoke + needs: check + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + UV_NO_CACHE: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version: "0.12.19" + enable-cache: false + - name: Install the candidate's locked server dependencies + run: uv sync --locked --python "$(which python)" --extra dev --extra code + - name: Build the exact candidate SDK wheel + run: uv build sdk/python --out-dir dify-sdk-wheel + - name: Install the candidate wheel for fixture setup + run: | + wheels=(dify-sdk-wheel/clickhouse_loom-*.whl) + test "${#wheels[@]}" -eq 1 + uv pip install --python "$PWD/.venv/bin/python" "${wheels[0]}" + - name: Pure mocked ownership and failure guards + run: .venv/bin/python -m pytest -q tests/unit/test_dify_application_guards.py + - name: Package the candidate plugin with the checked official CLI + run: | + curl --fail --location https://github.com/langgenius/dify-plugin-daemon/releases/download/0.6.15/dify-plugin-linux-amd64 --output "$RUNNER_TEMP/dify" + echo "d0a1e3b25bd6c5f6cd749cad2cf04817ea808344e6cc07a2c49bb0108019ab93 $RUNNER_TEMP/dify" | sha256sum --check + chmod +x "$RUNNER_TEMP/dify" + "$RUNNER_TEMP/dify" plugin package integrations/dify -o loom-0.1.0.difypkg + - name: Exercise the owned Dify application + run: .venv/bin/python scripts/gates/dify_application_smoke.py + - name: Verify owned cleanup even when the smoke fails + if: always() + run: .venv/bin/python scripts/gates/dify_application_smoke.py --cleanup-only + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: dify-application-evidence + path: dify-application-evidence/ + if-no-files-found: error + retention-days: 7 diff --git a/scripts/gates/dify-loopback.override.yaml b/scripts/gates/dify-loopback.override.yaml new file mode 100644 index 0000000000..e83ec41291 --- /dev/null +++ b/scripts/gates/dify-loopback.override.yaml @@ -0,0 +1,17 @@ +services: + nginx: + ports: !override + - target: 80 + published: "17881" + host_ip: "127.0.0.1" + protocol: tcp + - target: 443 + published: "17882" + host_ip: "127.0.0.1" + protocol: tcp + plugin_daemon: + ports: !override + - target: 5003 + published: "17883" + host_ip: "127.0.0.1" + protocol: tcp diff --git a/scripts/gates/dify_application_scenario.py b/scripts/gates/dify_application_scenario.py new file mode 100644 index 0000000000..2eab75d606 --- /dev/null +++ b/scripts/gates/dify_application_scenario.py @@ -0,0 +1,449 @@ +"""Exercise native tools through normal Dify console setup and workflow routes.""" + +import base64 +import json +import secrets +import subprocess +import threading +import time +from collections import deque +from copy import deepcopy +from urllib.parse import quote + +import httpx + +FACT = "The application proof user prefers aisle seats." +DOCUMENT = "The shared application travel guide describes aisle seats." + + +class Console: + def __init__(self, ctx): + self.ctx = ctx + self.client = httpx.Client( + base_url="http://127.0.0.1:17881/console/api/", timeout=60, trust_env=False + ) + ctx.cleanup_callbacks.append(self.client.close) + + def refusal(self, response, label): + response.read() + try: + body = response.json() + except ValueError: + body = {"message": response.text} + details = { + name: self.ctx.redact(value)[:2000] + for name in ("code", "message", "error") + if isinstance(body, dict) and isinstance(value := body.get(name), str) + } + evidence = {"label": label, "status": response.status_code, **details} + self.ctx.record("console-refusal-" + secrets.token_hex(4) + ".json", evidence) + raise RuntimeError(self.ctx.redact(json.dumps(evidence))) + + def request(self, method, path, *, payload=None, allowed=(200, 201)): + csrf = self.client.cookies.get("csrf_token") + headers = {"X-CSRF-Token": csrf} if csrf else {} + response = self.client.request(method, path, json=payload, headers=headers) + for cookie in self.client.cookies.jar: + self.ctx.secret(cookie.value) + if response.status_code not in allowed: + self.refusal(response, method + " " + path) + return response + + def json(self, method, path, payload=None): + return self.request(method, path, payload=payload).json() + + def ready(self): + deadline = time.monotonic() + 240 + last_response = None + last_connection_error = None + while time.monotonic() < deadline: + try: + response = self.client.get("setup") + if response.status_code == 200: + if response.json()["step"] != "not_started": + raise RuntimeError("Fresh Dify instance is unexpectedly initialized") + return + if response.status_code not in (502, 503, 504): + self.refusal(response, "GET setup during startup") + last_response = response + last_connection_error = None + except httpx.ConnectError as error: + last_response = None + last_connection_error = self.ctx.redact(str(error))[:2000] + time.sleep(1) + if last_response is not None: + self.refusal(last_response, "GET setup startup deadline exceeded") + evidence = { + "label": "GET setup startup deadline exceeded", + "status": None, + "connection_error": last_connection_error, + } + self.ctx.record("startup-deadline.json", evidence) + raise RuntimeError(self.ctx.redact(json.dumps(evidence))) + + +def mappings(value): + if isinstance(value, dict): + yield value + for item in value.values(): + yield from mappings(item) + elif isinstance(value, list): + for item in value: + yield from mappings(item) + + +def validate_tools(tools): + expected = { + "remember": { + "user_id": ("form", "string", True), + "memory_key": ("llm", "string", True), + "value": ("llm", "string", True), + }, + "recall": { + "user_id": ("form", "string", True), + "query": ("llm", "string", True), + "top_k": ("form", "number", False), + }, + "search_knowledge": { + "user_id": ("form", "string", True), + "query": ("llm", "string", True), + "top_k": ("form", "number", False), + }, + } + if len(tools) != len(expected) or {tool["name"] for tool in tools} != set(expected): + raise RuntimeError("Registered native tool names differ") + for tool in tools: + parameters = tool["parameters"] + actual = {p["name"]: (p["form"], p["type"], p["required"]) for p in parameters} + if len(actual) != len(parameters) or actual != expected[tool["name"]]: + raise RuntimeError("Registered native parameter schema differs: " + tool["name"]) + + +def has_shared_document(output): + return any( + item.get("source_uri") == "application-guide" and item.get("content") == DOCUMENT + for item in mappings(output) + ) + + +def build_graph(defaults, provider, tool, user_id, parameters): + start = deepcopy(defaults["start"]) + end = deepcopy(defaults["end"]) + start.update(type="start", title="Start", desc="", variables=[]) + end.update( + type="end", + title="End", + desc="", + outputs=[ + { + "variable": "result", + "value_selector": ["tool", "json"], + "value_type": "array[object]", + } + ], + ) + data = dict( + type="tool", + title=tool, + desc="", + provider_type="builtin", + provider_id=provider["id"], + provider_name=provider["name"], + tool_name=tool, + tool_label=tool, + tool_node_version="2", + tool_configurations={"user_id": {"type": "constant", "value": user_id}}, + tool_parameters={ + name: {"type": "mixed", "value": value} for name, value in parameters.items() + }, + ) + if tool != "remember": + data["tool_configurations"]["top_k"] = {"type": "constant", "value": 5} + for name in ("plugin_id", "plugin_unique_identifier"): + if provider.get(name): + data[name] = provider[name] + nodes = [ + { + "id": name, + "type": "custom", + "data": payload, + "position": {"x": index * 300, "y": 0}, + "sourcePosition": "right", + "targetPosition": "left", + } + for index, (name, payload) in enumerate([("start", start), ("tool", data), ("end", end)]) + ] + edges = [ + { + "id": source + "-" + target, + "source": source, + "target": target, + "sourceHandle": "source", + "targetHandle": "target", + "type": "custom", + "data": {}, + } + for source, target in [("start", "tool"), ("tool", "end")] + ] + return {"nodes": nodes, "edges": edges} + + +def workflow(console, ctx, app_id, defaults, provider, tool, user_id, parameters, label): + path = "apps/" + app_id + "/workflows/draft" + draft = console.request("GET", path, allowed=(200, 404)) + current = draft.json() if draft.status_code == 200 else {} + graph = build_graph(defaults, provider, tool, user_id, parameters) + console.json( + "POST", + path, + { + "graph": graph, + "features": current.get("features") or {}, + "hash": current.get("hash"), + "conversation_variables": [], + }, + ) + events = [] + csrf = console.client.cookies.get("csrf_token") + with console.client.stream( + "POST", path + "/run", json={"inputs": {}, "files": []}, headers={"X-CSRF-Token": csrf} + ) as response: + if response.status_code != 200: + console.refusal(response, "POST " + path + "/run") + for line in response.iter_lines(): + if line.startswith("data: "): + event = json.loads(line[6:]) + if event.get("event") in ("workflow_finished", "node_finished", "error"): + data = event.get("data") or {} + events.append( + { + "event": event["event"], + "node_type": data.get("node_type"), + "node_id": data.get("node_id"), + "status": data.get("status"), + "outputs": data.get("outputs"), + "failure": { + level: { + key: ctx.redact(value)[:2000] + for key in ("code", "message", "error") + if isinstance(value := source.get(key), str) + } + for level, source in [("event", event), ("data", data)] + }, + } + ) + ctx.record(label + "-workflow.json", events) + terminal = [e for e in events if e["event"] == "workflow_finished"] + tools = [e for e in events if e["event"] == "node_finished" and e["node_type"] == "tool"] + if ( + len(terminal) != 1 + or terminal[0]["status"] != "succeeded" + or len(tools) != 1 + or tools[0]["status"] != "succeeded" + ): + raise RuntimeError( + "Native tool workflow did not succeed: " + + label + + "; " + + ctx.redact(json.dumps(events[-4:])) + ) + return tools[0]["outputs"] + + +def run(ctx): + import clickhouse_connect + + console = Console(ctx) + console.ready() + console.json("POST", "init", {"password": ctx.env["INIT_PASSWORD"]}) + password = ctx.secret("Aa7!" + secrets.token_hex(10)) + email = "proof-" + secrets.token_hex(4) + "@example.com" + console.json( + "POST", + "setup", + {"email": email, "name": "CI plugin proof", "password": password, "language": "en-US"}, + ) + encoded = ctx.secret(base64.b64encode(password.encode()).decode()) + console.json("POST", "login", {"email": email, "password": encoded, "remember_me": False}) + if not console.client.cookies.get("csrf_token"): + raise RuntimeError("Normal login supplied no CSRF token") + debug = console.json("GET", "workspaces/current/plugin/debugging-key") + key = ctx.secret(debug["key"]) + if debug["host"] != "127.0.0.1" or int(debug["port"]) != 17883: + raise RuntimeError("Workspace advertised a different debug endpoint") + plugin_env = dict( + ctx.env, + INSTALL_METHOD="remote", + REMOTE_INSTALL_URL="127.0.0.1:17883", + REMOTE_INSTALL_KEY=key, + ) + logs = deque(maxlen=1000) + child = subprocess.Popen( + [str(ctx.plugin_python), "-m", "main"], + cwd=ctx.plugin_dir, + env=plugin_env, + start_new_session=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + ctx.track_plugin(child) + + def collect(): + for line in child.stdout: + logs.append(ctx.redact(line[:4096].rstrip())) + + reader = threading.Thread(target=collect, daemon=True) + reader.start() + + def save_plugin_log(): + reader.join(timeout=10) + if reader.is_alive(): + raise RuntimeError("Native plugin log reader did not finish") + ctx.record("native-plugin-log.json", list(logs)) + + ctx.cleanup_callbacks.append(save_plugin_log) + deadline = time.monotonic() + 60 + providers = [] + while time.monotonic() < deadline: + if child.poll() is not None: + raise RuntimeError("Native remote plugin exited before registration") + providers = console.json("GET", "workspaces/current/tool-providers") + matches = [ + p + for p in providers + if p.get("author") == "clickhouse" + and (p.get("name") == "loom" or p.get("name", "").endswith("/loom")) + ] + if len(matches) == 1: + break + if len(matches) > 1: + raise RuntimeError("Ambiguous installed Loom provider") + time.sleep(1) + else: + raise RuntimeError("Native plugin registration deadline exceeded") + (provider,) = matches + if provider["type"] != "builtin": + raise RuntimeError("Unexpected native provider type") + provider_path = quote(provider["id"], safe="/") + tools = console.json( + "GET", "workspaces/current/tool-provider/builtin/" + provider_path + "/tools" + ) + validate_tools(tools) + org, namespace, agent = ctx.org, ctx.namespace, ctx.agent + token = ctx.token + profile = ctx.env + + credentials = { + "base_url": "http://127.0.0.1:7871", + "api_key": token, + "org": org, + "namespace": namespace, + "agent": agent, + } + console.json( + "POST", + "workspaces/current/tool-provider/builtin/" + provider_path + "/add", + {"credentials": credentials, "name": "CI proof", "type": "api-key"}, + ) + app = console.json( + "POST", "apps", {"name": "Loom native application proof", "mode": "workflow"} + ) + app_id = app["id"] + configs = console.json("GET", "apps/" + app_id + "/workflows/default-workflow-block-configs") + defaults = {item["type"]: item["config"] for item in configs} + for name in ("start", "end"): + if name not in defaults: + raise RuntimeError("Application supplied no default block: " + name) + public_provider = { + name: provider.get(name) + for name in ("id", "name", "type", "plugin_id", "plugin_unique_identifier") + } + ctx.record( + "registration.json", + {"provider": public_provider, "tools": ["remember", "recall", "search_knowledge"]}, + ) + workflow( + console, + ctx, + app_id, + defaults, + provider, + "remember", + "user-a", + {"memory_key": "seat", "value": FACT}, + "remember", + ) + own = workflow( + console, + ctx, + app_id, + defaults, + provider, + "recall", + "user-a", + {"query": "aisle seats"}, + "recall-own", + ) + if FACT not in json.dumps(own): + raise RuntimeError("Same-user native recall did not return the stored fact") + other = workflow( + console, + ctx, + app_id, + defaults, + provider, + "recall", + "user-b", + {"query": "aisle seats"}, + "recall-other", + ) + result_lists = [item["results"] for item in mappings(other) if "results" in item] + if result_lists != [[]]: + raise RuntimeError("Other-user native recall was not empty") + with httpx.Client(timeout=30, trust_env=False) as loom: + response = loom.post( + "http://127.0.0.1:7871/v1/knowledge.upsert", + headers={"Authorization": "Bearer " + token}, + json={ + "org": org, + "namespace": namespace, + "agent": agent, + "source_uri": "application-guide", + "content": DOCUMENT, + }, + ) + if response.status_code != 200: + raise RuntimeError("Owned knowledge fixture write failed") + knowledge = workflow( + console, + ctx, + app_id, + defaults, + provider, + "search_knowledge", + "user-b", + {"query": "aisle seats"}, + "knowledge-shared", + ) + if not has_shared_document(knowledge): + raise RuntimeError("Native knowledge workflow did not return the shared document") + ch = clickhouse_connect.get_client( + host="127.0.0.1", + port=18471, + username=profile.get("LOOM_CH_USER", "loom"), + password=profile.get("LOOM_CH_PASSWORD", "loomdev"), + database=profile.get("LOOM_CH_DATABASE", "loom"), + ) + try: + rows = ch.query( + "SELECT org,namespace,agent,user_id,session_id,memory_key,value FROM loom.memory FINAL " + "WHERE org=%(org)s AND namespace=%(ns)s AND tombstoned_at IS NULL", + parameters={"org": org, "ns": namespace}, + ).result_rows + if rows != [(org, namespace, agent, "user-a", "-", "seat", FACT)]: + raise RuntimeError("Stored Loom identity/value differs from native workflow input") + ctx.record("stored-identity.json", {"rows": rows}) + finally: + ch.close() diff --git a/scripts/gates/dify_application_smoke.py b/scripts/gates/dify_application_smoke.py new file mode 100644 index 0000000000..2a19d2fad9 --- /dev/null +++ b/scripts/gates/dify_application_smoke.py @@ -0,0 +1,1350 @@ +"""Run one Dify application scenario on its own GitHub-hosted VM.""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +import os +import secrets +import shutil +import signal +import socket +import subprocess +import sys +import threading +import time +import uuid +import zipfile +from collections import deque +from datetime import UTC, datetime +from pathlib import Path + +import httpx + +DIFY = "8e689e868b63b6dd2b449d0eb568ca5cc7127759" +SERVICES = { + "agent_backend", + "agent_ssrf_proxy", + "api", + "db_postgres", + "init_permissions", + "local_sandbox", + "nginx", + "pgvector", + "plugin_daemon", + "redis", + "sandbox", + "ssrf_proxy", + "web", + "worker", + "worker_beat", +} +LOCAL_DOCKER = "unix:///var/run/docker.sock" +EXPECTED_PLUGIN_FILES = frozenset( + ( + ".difyignore", + "LICENSE", + "PRIVACY.md", + "README.md", + "_assets/icon.svg", + "client.py", + "main.py", + "manifest.yaml", + "provider/__init__.py", + "provider/loom.py", + "provider/loom.yaml", + "requirements.txt", + "tools/__init__.py", + "tools/recall.py", + "tools/recall.yaml", + "tools/remember.py", + "tools/remember.yaml", + "tools/search_knowledge.py", + "tools/search_knowledge.yaml", + ) +) +GIB = 1024**3 +PORTS = (17881, 17882, 17883, 18471, 18472, 7871) + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def available(): + return ( + int( + next( + line.split()[1] + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith("MemAvailable:") + ) + ) + * 1024 + ) + + +def birth(pid): + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + return int(fields[19]), int(fields[2]), int(fields[3]) + except FileNotFoundError: + return None + + +def initial_password(): + """The pinned normal /init model accepts at most thirty characters.""" + return "Aa7" + secrets.token_hex(12) + + +def verify_docker_selection(environ, resolve_endpoint): + require( + not any(environ.get(name) for name in ("DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG")), + "Docker host/context/config overrides are forbidden", + ) + endpoint = resolve_endpoint() + require(endpoint == LOCAL_DOCKER, "Effective Docker context is not the hosted local socket") + return endpoint + + +def docker_arguments(args, endpoint): + require(endpoint == LOCAL_DOCKER, "Docker endpoint is unverified") + return ["docker", "--host", endpoint, *args] + + +def validate_archive_members(members): + names = [item.filename for item in members] + require(not any(item.is_dir() for item in members), "Plugin directory entry is unreviewed") + require( + len(names) == len(set(names)) and set(names) == EXPECTED_PLUGIN_FILES, + "Plugin archive members differ from reviewed set", + ) + + +def image_volume_mounts(service_name, service, image): + covered = {mount["target"] for mount in service.get("volumes", [])} + missing = sorted(set(image.get("Config", {}).get("Volumes") or {}) - covered) + mounts, volumes = [], {} + for index, target in enumerate(missing): + require( + target.startswith("/") and target != "/" and ".." not in Path(target).parts, + "Image volume target is invalid", + ) + name = f"image-{service_name}-{index}" + mounts.append({"type": "volume", "source": name, "target": target}) + volumes[name] = {} + return mounts, volumes + + +def read_session(pid): + snapshot = {} + for item in Path("/proc").iterdir(): + if item.name.isdigit() and (identity := birth(int(item.name))): + if int(item.name) == pid or identity[2] == pid: + try: + snapshot[int(item.name)] = (*identity, item.stat().st_uid) + except FileNotFoundError: + pass + return snapshot + + +def authorized_session(saved, observed, uid): + """An absent leader permits only previously saved exact member generations.""" + pid, started = saved["pid"], saved["start"] + leader = observed.get(pid) + if leader is not None: + require(leader == (started, pid, pid, uid), "Session leader generation changed") + known = {int(key): tuple(value) for key, value in saved["members"].items()} + for member, identity in observed.items(): + require(identity[2:] == (pid, uid), "Session member owner changed") + if leader is None: + require(known.get(member) == identity, "Orphan member generation is unproven") + return observed + + +def stop_owned_session( + saved, + *, + observe, + current_birth, + send, + pause, + now, + uid, + remember=lambda: None, + reap=lambda: None, +): + for sig, seconds in ((signal.SIGTERM, 15), (signal.SIGKILL, 5)): + members = authorized_session(saved, observe(saved["pid"]), uid) + if not members: + return + saved["members"].update(members) + remember() + for pid, identity in members.items(): + if current_birth(pid) == identity[:3]: + try: + send(pid, sig) + except ProcessLookupError: + pass + end = now() + seconds + while now() < end: + reap() + if not authorized_session(saved, observe(saved["pid"]), uid): + return + pause(0.1) + require( + not authorized_session(saved, observe(saved["pid"]), uid), + "Owned session remains after cleanup", + ) + + +def stop_monitor(runtime): + runtime.finished.set() + if runtime.monitor_thread is not None: + runtime.monitor_thread.join(timeout=5) + require(not runtime.monitor_thread.is_alive(), "Resource monitor did not stop") + + +def final_failure(failure, monitor_error): + return failure or monitor_error + + +def wait_fixture_ready(client, url, seconds, *, now=time.monotonic, pause=time.sleep): + end = now() + seconds + observations = [] + while now() < end: + try: + status = client.get(url).status_code + except httpx.TransportError as error: + observations.append({"error_type": type(error).__name__}) + else: + observations.append({"status": status}) + require(status not in (401, 403), "Fixture readiness permission refused") + if status == 200: + return observations + pause(1) + raise RuntimeError("Owned fixture readiness failed: " + json.dumps(observations[-3:])) + + +class Runtime: + def __init__(self): + require( + os.environ.get("GITHUB_ACTIONS") == "true" + and os.environ.get("RUNNER_ENVIRONMENT") == "github-hosted" + and os.environ.get("RUNNER_OS") == "Linux", + "GitHub-hosted Linux job required", + ) + + self.workspace = Path(os.environ["GITHUB_WORKSPACE"]).resolve() + sys.path.insert(0, str(self.workspace)) + self.root = Path(os.environ["RUNNER_TEMP"]).resolve() / ("dify-smoke-" + uuid.uuid4().hex) + self.root.mkdir(mode=0o700) + self.evidence = self.workspace / "dify-application-evidence" + self.evidence.mkdir(mode=0o700) + self.project = "loom-dify-" + uuid.uuid4().hex[:16] + self.path = self.root / "dify" + self.plugin_dir = self.root / "plugin" + self.plugin_python = self.root / "plugin-venv/bin/python" + self.env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("LOOM_", "UV_", "PYTHON")) + } + self.env.update( + COMPOSE_PROFILES="postgresql,pgvector", + VECTOR_STORE="pgvector", + DB_TYPE="postgresql", + EXPOSE_PLUGIN_DEBUGGING_HOST="127.0.0.1", + EXPOSE_PLUGIN_DEBUGGING_PORT="17883", + ) + self.env["PATH"] = str(self.workspace / ".venv/bin") + os.pathsep + self.env["PATH"] + self.secrets = [] + for key in ( + "SECRET_KEY", + "DB_PASSWORD", + "REDIS_PASSWORD", + "PLUGIN_DAEMON_KEY", + "PLUGIN_DIFY_INNER_API_KEY", + "DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN", + "DIFY_AGENT_API_TOKEN", + "PGVECTOR_POSTGRES_PASSWORD", + "SANDBOX_API_KEY", + ): + self.env[key] = self.secret("Aa7" + secrets.token_hex(24)) + self.env["INIT_PASSWORD"] = self.secret(initial_password()) + self.env["CODE_EXECUTION_API_KEY"] = self.env["SANDBOX_API_KEY"] + self.env["DIFY_AGENT_SERVER_SECRET_KEY"] = self.secret( + base64.urlsafe_b64encode(secrets.token_bytes(32)).decode().rstrip("=") + ) + self.env.update( + LOOM_ENV_FILE="", + LOOM_CH_HOST="127.0.0.1", + LOOM_CH_PORT="18471", + LOOM_CH_USER="loom", + LOOM_CH_READER_USER="loom_reader", + LOOM_CH_PASSWORD="loomdev", + LOOM_CH_DATABASE="loom", + LOOM_CH_SECURE="false", + LOOM_AUTH_REQUIRED="true", + LOOM_JWT_SECRET=self.secret(secrets.token_hex(32)), + LOOM_BASE_URL="http://127.0.0.1:7871", + LOOM_LLM_PROVIDER="fake", + LOOM_EMBEDDING_MODEL="fake-ngram-1024", + LOOM_BACKGROUND_WORKERS="off", + LOOM_INSTANCE_LABEL="ci:dify-application", + LOOM_WEB_WORKERS="1", + LOOM_RUNTIME_DIR=str(self.root / "loom-runtime"), + PORT="7871", + ) + self.primary_profile = self.env + self.processes, self.cleanup_callbacks = [], [] + self.containers, self.volumes, self.networks = {}, {}, {} + self.expected_images = {} + self.finished = threading.Event() + self.monitor_error = None + self.monitor_thread = None + self.docker_endpoint = None + self.cleaning = False + self.deadline = time.monotonic() + 1800 + self.phase = "admission" + self.created = False + self.started = datetime.now(UTC).isoformat() + self.before = {} + self.threads = [] + self.state_path = Path(os.environ["RUNNER_TEMP"]) / "dify-smoke-ownership.json" + require(not self.state_path.exists(), "Ownership state already exists") + self.persist() + + def persist(self): + state = { + "run_id": os.environ["GITHUB_RUN_ID"], + "attempt": os.environ["GITHUB_RUN_ATTEMPT"], + "root": str(self.root), + "project": self.project, + "started": self.started, + "before": {key: sorted(value) for key, value in self.before.items()}, + "images": self.expected_images, + "processes": [ + {"pid": item["child"].pid, "start": item["birth"], "members": item["members"]} + for item in self.processes + ], + "ch_attempted": getattr(self, "ch_attempted", False), + "ch_image": getattr(self, "ch_image", None), + "compose_attempted": self.created, + "docker_endpoint": self.docker_endpoint, + } + temporary = self.state_path.with_suffix(".new") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w") as output: + json.dump(state, output) + temporary.replace(self.state_path) + + def secret(self, value): + self.secrets.append(value) + return value + + def redact(self, value): + text = str(value) + for secret in sorted(set(self.secrets), key=len, reverse=True): + text = text.replace(secret, "[REDACTED]") + return text + + def record(self, name, value): + # Only explicit metadata reaches this method; never env, config or inspect dumps. + text = self.redact(json.dumps(value, default=str, indent=2)) + require(len(text) <= 256_000, "Evidence record exceeds limit") + with (self.evidence / name).open("x") as target: + target.write(text + "\n") + + def launch(self, args, *, cwd=None, env=None, logged=False): + child = subprocess.Popen( + args, + cwd=cwd or self.workspace, + env=env or self.env, + start_new_session=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + identity = birth(child.pid) + require( + identity is not None and identity[1:] == (child.pid, child.pid), + "Child session identity unavailable", + ) + entry = { + "child": child, + "birth": identity[0], + "logs": deque(maxlen=200), + "logged": logged, + "members": {child.pid: (*identity, os.getuid())}, + } + self.processes.append(entry) + self.persist() + if logged: + + def drain(): + for line in iter(child.stdout.readline, ""): + entry["logs"].append(self.redact(line[:4096])) + + thread = threading.Thread(target=drain, daemon=True) + thread.start() + self.threads.append(thread) + return child + + def command(self, args, *, timeout=120, cwd=None, check=True, docker_metadata=False): + if args[0] == "docker": + if docker_metadata: + require( + args + == [ + "docker", + "context", + "inspect", + "--format", + "{{json .Endpoints.docker.Host}}", + ], + "Unexpected Docker metadata request", + ) + else: + args = docker_arguments(args[1:], self.docker_endpoint) + child = self.launch(args, cwd=cwd) + entry = next(item for item in self.processes if item["child"] is child) + try: + output, _ = child.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + self.stop_group(entry) + raise RuntimeError("Command timeout in " + self.phase) from None + self.retire(entry) + if check and child.returncode: + # Commands may print environment-derived secrets; redact before retaining a tail. + raise RuntimeError( + "Command failed in " + self.phase + ": " + self.redact(output)[-2000:] + ) + return subprocess.CompletedProcess(args, child.returncode, output, "") + + def docker_json(self, *args): + return json.loads(self.command(["docker", *args]).stdout) + + def ids(self, kind, *filters): + return set(self.command(["docker", kind, "ls", "-q", *filters]).stdout.split()) + + def compose(self, *args, timeout=120): + files = ["docker-compose.yaml", "task-loopback.override.yaml"] + if (self.path / "docker/task-resource.override.json").exists(): + files.append("task-resource.override.json") + command = ["docker", "compose", "--project-name", self.project] + for file in files: + command += ["-f", file] + return self.command(command + list(args), timeout=timeout, cwd=self.path / "docker") + + def retire(self, entry): + saved = {"pid": entry["child"].pid, "start": entry["birth"], "members": entry["members"]} + require( + not authorized_session(saved, read_session(saved["pid"]), os.getuid()), + "Completed command session is not empty", + ) + self.processes.remove(entry) + self.persist() + + def stop_group(self, entry): + child = entry["child"] + saved = {"pid": child.pid, "start": entry["birth"], "members": entry["members"]} + stop_owned_session( + saved, + observe=read_session, + current_birth=birth, + send=os.kill, + pause=time.sleep, + now=time.monotonic, + uid=os.getuid(), + remember=self.persist, + reap=child.poll, + ) + child.wait(timeout=2) + self.retire(entry) + + def admit(self): + self.docker_endpoint = verify_docker_selection( + self.env, + lambda: json.loads( + self.command( + ["docker", "context", "inspect", "--format", "{{json .Endpoints.docker.Host}}"], + docker_metadata=True, + ).stdout + ), + ) + # Also binds the existing image resolver's internal Docker commands to this same endpoint. + self.env["DOCKER_HOST"] = self.docker_endpoint + self.persist() + require(available() >= 12 * GIB, "CI admission requires 12 GiB MemAvailable") + require(shutil.disk_usage(self.root).free >= 10 * GIB, "CI admission requires 10 GiB disk") + require((os.cpu_count() or 0) >= 4, "CI admission requires four logical CPUs") + version = ( + self.command(["docker", "compose", "version", "--short"]).stdout.strip().lstrip("v") + ) + parts = version.split(".") + require( + len(parts) >= 3 + and all(part.isdigit() for part in parts[:3]) + and tuple(map(int, parts[:3])) >= (2, 35, 0), + "Docker Compose 2.35.0+ required", + ) + for port in PORTS: + with socket.socket() as sock: + sock.bind(("127.0.0.1", port)) + self.before = { + kind: self.ids( + kind, + *( + ["-a", "--no-trunc"] + if kind == "container" + else ["--no-trunc"] + if kind == "network" + else [] + ), + ) + for kind in ("container", "network", "volume") + } + self.persist() + # This job owns its VM; a pre-existing workload is not adopted or removed. + require(not self.before["container"], "Unexpected pre-existing container") + self.record( + "admission.json", + { + "candidate": self.command(["git", "rev-parse", "HEAD"]).stdout.strip(), + "dify_commit": DIFY, + "project": self.project, + "mem_available": available(), + "disk_free": shutil.disk_usage(self.root).free, + "cpu_count": os.cpu_count(), + "ci_limits": { + "start_memory_gib": 12, + "start_disk_gib": 10, + "stop_memory_gib": 2, + "stop_disk_gib": 3, + "seconds": 1800, + }, + }, + ) + self.monitor_thread = threading.Thread(target=self.monitor, daemon=True) + self.monitor_thread.start() + + def monitor(self): + while not self.finished.wait(1): + try: + require(available() >= 2 * GIB, "CI memory reserve crossed") + require(shutil.disk_usage(self.root).free >= 3 * GIB, "CI disk reserve crossed") + require(time.monotonic() < self.deadline, "CI controller deadline crossed") + if getattr(self, "loom_pid", None): + rss = 0 + for item in Path("/proc").iterdir(): + if item.name.isdigit() and (ident := birth(int(item.name))): + if ident[1:] == (self.loom_pid, self.loom_pid): + try: + rss += ( + int( + next( + line.split()[1] + for line in (item / "status") + .read_text() + .splitlines() + if line.startswith("VmRSS:") + ) + ) + * 1024 + ) + except (FileNotFoundError, StopIteration): + pass + require(rss <= 2 * GIB, "Owned Loom RSS cap crossed") + except Exception as error: # noqa: BLE001 — Stop owned work on a failed resource observation. + self.monitor_error = str(error) + os.kill(os.getpid(), signal.SIGTERM) + return + + def platform(self): + self.phase = "official-source" + self.command(["git", "init", str(self.path)]) + self.command( + [ + "git", + "-C", + str(self.path), + "fetch", + "--depth=1", + "https://github.com/langgenius/dify.git", + DIFY, + ], + timeout=180, + ) + self.command(["git", "-C", str(self.path), "checkout", "--detach", "FETCH_HEAD"]) + require( + self.command(["git", "-C", str(self.path), "rev-parse", "HEAD"]).stdout.strip() == DIFY, + "Official source commit differs", + ) + # Official root .env is a service env_file, not just Compose interpolation. + # This private file contains only this job's fresh values and is never an artifact. + env_keys = ( + "COMPOSE_PROFILES", + "VECTOR_STORE", + "DB_TYPE", + "SECRET_KEY", + "DB_PASSWORD", + "REDIS_PASSWORD", + "PLUGIN_DAEMON_KEY", + "PLUGIN_DIFY_INNER_API_KEY", + "DIFY_AGENT_LOCAL_SANDBOX_AUTH_TOKEN", + "DIFY_AGENT_API_TOKEN", + "DIFY_AGENT_SERVER_SECRET_KEY", + "PGVECTOR_POSTGRES_PASSWORD", + "SANDBOX_API_KEY", + "CODE_EXECUTION_API_KEY", + "INIT_PASSWORD", + ) + descriptor = os.open(self.path / "docker/.env", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w") as output: + output.write("".join(key + "=" + self.env[key] + "\n" for key in env_keys)) + shutil.copyfile( + self.workspace / "scripts/gates/dify-loopback.override.yaml", + self.path / "docker/task-loopback.override.yaml", + ) + self.cfg = json.loads(self.compose("config", "--format", "json").stdout) + require(set(self.cfg["services"]) == SERVICES, "Official service closure differs") + require( + str( + self.cfg["services"]["plugin_daemon"]["environment"]["FORCE_VERIFYING_SIGNATURE"] + ).lower() + == "true", + "Signature verification differs", + ) + published = [] + for service in self.cfg["services"].values(): + for mount in service.get("volumes", []): + if mount["type"] == "bind": + require( + Path(mount["source"]).resolve().is_relative_to(self.path), "Foreign bind" + ) + for port in service.get("ports", []): + require(port.get("host_ip") == "127.0.0.1", "Non-loopback publication") + published.append(int(port["published"])) + require(sorted(published) == [17881, 17882, 17883], "Unexpected published port") + self.phase = "official-image-pull" + self.compose("pull", timeout=600) + override = {"services": {}, "volumes": {}} + for name, service in self.cfg["services"].items(): + (image,) = self.docker_json("image", "inspect", service["image"]) + mounts, volumes = image_volume_mounts(name, service, image) + require( + not (volumes.keys() & self.cfg.get("volumes", {}).keys()), "Volume name collision" + ) + override["volumes"].update(volumes) + self.expected_images[name] = image["Id"] + cap = ( + 2 * GIB + if name in ("api", "worker") + else GIB + if name in ("web", "agent_backend", "plugin_daemon") + else GIB // 2 + ) + override["services"][name] = {"image": image["Id"], "mem_limit": cap, "cpus": 1.0} + if mounts: + override["services"][name]["volumes"] = mounts + (self.path / "docker/task-resource.override.json").write_text(json.dumps(override)) + self.cfg = json.loads(self.compose("config", "--format", "json").stdout) + for name, service in self.cfg["services"].items(): + (image,) = self.docker_json("image", "inspect", service["image"]) + covered = {mount["target"] for mount in service.get("volumes", [])} + require(image["Id"] == self.expected_images[name], "Pinned image changed") + require( + not (set(image.get("Config", {}).get("Volumes") or {}) - covered), + "Unreviewed anonymous image volume", + ) + self.record("platform-caps.json", override) + self.created = True + self.persist() + self.phase = "official-platform-start" + self.compose("up", "-d", "--pull", "never", timeout=300) + self.discover() + require( + {row["service"] for row in self.containers.values()} >= SERVICES, + "Missing official service container", + ) + + def platform_ready(self): + end = time.monotonic() + 180 + while time.monotonic() < end: + ready = True + for ident, owner in self.containers.items(): + (row,) = self.docker_json("container", "inspect", ident) + state = row["State"] + require(not state.get("OOMKilled"), "Official service was OOM-killed") + if owner["service"] == "init_permissions": + require(state["Status"] in ("created", "running", "exited"), "Init failed") + ready &= state["Status"] == "exited" and state["ExitCode"] == 0 + else: + require(state["Status"] in ("created", "running"), "Official service exited") + ready &= ( + state["Status"] == "running" + and state.get("Health", {}).get("Status", "healthy") == "healthy" + ) + if ready: + return + time.sleep(2) + raise RuntimeError("Official service readiness deadline exceeded") + + def discover(self): + for ident in self.ids( + "container", + "-a", + "--no-trunc", + "--filter", + "label=com.docker.compose.project=" + self.project, + ): + (data,) = self.docker_json("container", "inspect", ident) + labels = data["Config"].get("Labels") or {} + service = labels.get("com.docker.compose.service") + require( + ident not in self.before["container"] + and data["Id"] == ident + and labels.get("com.docker.compose.project") == self.project + and service in SERVICES + and data["Created"] >= self.started + and data["Image"] == self.expected_images[service], + "Container ownership differs", + ) + expected_cap = ( + 2 * GIB + if service in ("api", "worker") + else GIB + if service in ("web", "agent_backend", "plugin_daemon") + else GIB // 2 + ) + require(data["HostConfig"]["Memory"] == expected_cap, "Container memory cap differs") + for mount in data["Mounts"]: + if mount["Type"] == "bind": + require( + Path(mount["Source"]).resolve().is_relative_to(self.path), + "Foreign runtime bind", + ) + elif mount["Type"] == "volume": + (volume,) = self.docker_json("volume", "inspect", mount["Name"]) + require( + volume["Name"] not in self.before["volume"] + and (volume.get("Labels") or {}).get("com.docker.compose.project") + == self.project, + "Volume ownership differs", + ) + self.volumes[volume["Name"]] = volume["CreatedAt"] + else: + require(mount["Type"] == "tmpfs", "Unexpected mount type") + self.containers[ident] = {"service": service, "created": data["Created"]} + for name in self.ids( + "volume", "--filter", "label=com.docker.compose.project=" + self.project + ): + (row,) = self.docker_json("volume", "inspect", name) + require( + name not in self.before["volume"] + and (row.get("Labels") or {}).get("com.docker.compose.project") == self.project + and datetime.fromisoformat(row["CreatedAt"].replace("Z", "+00:00")) + >= datetime.fromisoformat(self.started).replace(microsecond=0), + "Declared volume differs", + ) + self.volumes[name] = row["CreatedAt"] + for ident in self.ids( + "network", "--no-trunc", "--filter", "label=com.docker.compose.project=" + self.project + ): + (data,) = self.docker_json("network", "inspect", ident) + require( + ident not in self.before["network"] + and data["Created"] >= self.started + and (data.get("Labels") or {}).get("com.docker.compose.project") == self.project, + "Network ownership differs", + ) + self.networks[ident] = data["Created"] + + def fixture(self): + self.phase = "loom-fixture" + image = self.command(["bash", "dev/clickhouse_release.sh"], timeout=120).stdout.strip() + self.command(["docker", "pull", image], timeout=300) + (data,) = self.docker_json("image", "inspect", image) + self.ch_image = data["Id"] + args = [ + "docker", + "run", + "-d", + "--pull", + "never", + "--name", + self.project + "-ch", + "--label", + "loom.dify-smoke=" + self.project, + "--memory", + "1536m", + "--cpus", + "1", + "-p", + "127.0.0.1:18471:8123", + "-p", + "127.0.0.1:18472:9000", + "-e", + "CLICKHOUSE_DB=loom", + "-e", + "CLICKHOUSE_USER=loom", + "-e", + "CLICKHOUSE_PASSWORD=loomdev", + "-e", + "CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1", + "--ulimit", + "nofile=262144:262144", + ] + mounts = [("deploy/lemmagen3", "/var/lib/clickhouse/lemmagen3")] + mounts += [ + ("deploy/clickhouse-config/" + name, target + name) + for name, target in ( + ("lemmatizers.xml", "/etc/clickhouse-server/config.d/"), + ("nlp_user_settings.xml", "/etc/clickhouse-server/users.d/"), + ("keeper.xml", "/etc/clickhouse-server/config.d/"), + ("access_control.xml", "/etc/clickhouse-server/users.d/"), + ("system_logs.xml", "/etc/clickhouse-server/config.d/"), + ) + ] + mounts += [ + ("dev/clickhouse-ci/" + name, "/etc/clickhouse-server/users.d/" + name) + for name in ("log_queries_readonly.xml", "custom_settings_prefixes.xml") + ] + for source, target in mounts: + args += ["-v", str(self.workspace / source) + ":" + target + ":ro"] + self.ch_attempted = True + self.persist() + self.command(args + [self.ch_image]) + self.capture_ch() + self.phase = "clickhouse-readiness" + with httpx.Client(timeout=3, trust_env=False) as client: + self.record( + "clickhouse-readiness.json", + wait_fixture_ready(client, "http://127.0.0.1:18471/ping", 80), + ) + self.phase = "loom-migrations" + self.command([sys.executable, "-m", "server.core.schema.migrate"], timeout=300) + self.loom_pid = self.launch(["sh", "scripts/serve.sh"], logged=True).pid + self.phase = "loom-readiness" + with httpx.Client(timeout=5, trust_env=False) as client: + self.record( + "loom-readiness.json", + wait_fixture_ready(client, self.env["LOOM_BASE_URL"] + "/v1/health", 120), + ) + require( + client.get(self.env["LOOM_BASE_URL"] + "/v1/me/service_accounts").status_code + == 401, + "Owned fixture did not enforce authentication", + ) + # Same operator signing path and namespace registration as auth-on handoff CI. + os.environ.update( + {key: value for key, value in self.env.items() if key.startswith("LOOM_")} + ) + from loom import Agent + + from server.auth import keyring + from server.auth.jwt import mint_token + + keyring.sign_with_root_only() + self.org = "it-dify-app-" + secrets.token_hex(4) + self.namespace, self.agent = "dify-app", "dify-app" + admin = self.secret( + mint_token( + subject="dify-smoke-setup", + org=self.org, + namespaces=[self.namespace], + roles=["admin"], + ttl_seconds=600, + is_service_account=True, + ) + ) + self.token = self.secret( + mint_token( + subject="dify-smoke-plugin", + org=self.org, + namespaces=[self.namespace], + roles=["writer"], + ttl_seconds=600, + is_service_account=True, + ) + ) + from tests.integration._resources import ch_client_from_env, purge + + def purge_own_org(): + client = ch_client_from_env() + try: + require( + not purge(client, "loom", frozenset({self.org})), "Owned org purge left rows" + ) + self.record("loom-org-purge.json", {"org": self.org, "remaining_rows": {}}) + finally: + client.close() + + self.cleanup_callbacks.append(purge_own_org) + + async def register(): + async with Agent( + self.org, + self.namespace, + "dify-smoke-setup", + auto_run=False, + base_url=self.env["LOOM_BASE_URL"], + api_key=admin, + ) as agent: + await agent.namespaces.register(owner_email="dify-smoke@example.test") + + asyncio.run(register()) + + def capture_ch(self): + rows = self.docker_json("container", "inspect", self.project + "-ch") + (data,) = rows + require( + data["Id"] not in self.before["container"] + and data["Image"] == self.ch_image + and data["Created"] >= self.started + and (data["Config"].get("Labels") or {}).get("loom.dify-smoke") == self.project, + "ClickHouse ownership differs", + ) + self.ch_id = data["Id"] + self.ch_volumes = {} + for mount in data["Mounts"]: + if mount["Type"] == "volume": + (volume,) = self.docker_json("volume", "inspect", mount["Name"]) + require( + volume["Name"] not in self.before["volume"], "Pre-existing ClickHouse volume" + ) + self.ch_volumes[volume["Name"]] = volume["CreatedAt"] + + def artifact(self): + self.phase = "candidate-wheel" + wheels = list((self.workspace / "dify-sdk-wheel").glob("clickhouse_loom-*.whl")) + require(len(wheels) == 1, "Exactly one candidate wheel required") + (wheel,) = wheels + archive = self.workspace / "loom-0.1.0.difypkg" + self.plugin_dir.mkdir() + with zipfile.ZipFile(archive) as package: + validate_archive_members(package.infolist()) + for member in package.infolist(): + target = self.plugin_dir / member.filename + require( + target.resolve().is_relative_to(self.plugin_dir), "Unsafe plugin archive path" + ) + require( + (member.external_attr >> 16) & 0o170000 != 0o120000, "Plugin archive symlink" + ) + if not member.is_dir(): + source = self.workspace / "integrations/dify" / member.filename + require( + source.is_file() and package.read(member) == source.read_bytes(), + "Packaged source differs from candidate", + ) + package.extractall(self.plugin_dir) + self.command([sys.executable, "-m", "venv", str(self.root / "plugin-venv")]) + self.command( + [ + str(self.plugin_python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + str(wheel), + "-r", + str(self.plugin_dir / "requirements.txt"), + ], + timeout=180, + ) + provenance = self.command( + [ + str(self.plugin_python), + "-c", + "import importlib.metadata,json,loom; " + "print(json.dumps({'version':importlib.metadata.version('clickhouse-loom')," + "'module':loom.__file__}))", + ], + cwd=self.plugin_dir, + ).stdout + installed = json.loads(provenance) + require( + Path(installed["module"]).resolve().is_relative_to(self.root / "plugin-venv"), + "Plugin did not import isolated wheel installation", + ) + with zipfile.ZipFile(wheel) as package: + module_root = Path(installed["module"]).parent.parent + members = [ + name + for name in package.namelist() + if name.startswith("loom/") and not name.endswith("/") + ] + require(bool(members), "Candidate wheel has no SDK package") + require( + all( + (module_root / name).is_file() + and (module_root / name).read_bytes() == package.read(name) + for name in members + ), + "Installed SDK bytes differ from candidate wheel", + ) + self.record( + "artifact.json", + { + "sdk_files_matched": len(members), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "archive_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "installed_version": installed["version"], + "isolated_import": True, + "registry_publication_proven": False, + }, + ) + + def track_plugin(self, child): + identity = birth(child.pid) + require( + identity is not None and identity[1:] == (child.pid, child.pid), + "Plugin session differs", + ) + self.processes.append( + { + "child": child, + "birth": identity[0], + "logs": deque(), + "logged": False, + "members": {child.pid: (*identity, os.getuid())}, + } + ) + self.persist() + + def cleanup(self): + self.cleaning = True + errors = [] + try: + stop_monitor(self) + except Exception as error: # noqa: BLE001 — Cleanup continues after a failed monitor join. + errors.append(self.redact(error)) + # Close/purge while the fixture is alive; the plugin must stop before its log reader joins. + for entry in reversed(list(self.processes)): + if entry["child"].args[:1] == [str(self.plugin_python)]: + try: + self.stop_group(entry) + except Exception as error: # noqa: BLE001 — Preserve all cleanup failures. + errors.append(self.redact(error)) + for callback in self.cleanup_callbacks: + try: + callback() + except Exception as error: # noqa: BLE001 — Remaining owned resources still need cleanup. + errors.append(self.redact(error)) + for entry in reversed(list(self.processes)): + try: + self.stop_group(entry) + except Exception as error: # noqa: BLE001 — Remaining owned resources still need cleanup. + errors.append(self.redact(error)) + for thread in self.threads: + thread.join(timeout=2) + if thread.is_alive(): + errors.append("Owned log reader remains") + try: + if self.created: + self.discover() + for ident in self.containers: + (current,) = self.docker_json("container", "inspect", ident) + require( + current["Created"] == self.containers[ident]["created"], + "Container birth differs", + ) + self.command(["docker", "container", "stop", "--time", "10", ident], timeout=20) + self.command(["docker", "container", "rm", "-v", ident]) + for name, created in self.volumes.items(): + if name in self.ids("volume"): + (current,) = self.docker_json("volume", "inspect", name) + require(current["CreatedAt"] == created, "Volume birth differs") + self.command(["docker", "volume", "rm", name]) + for ident, created in self.networks.items(): + (current,) = self.docker_json("network", "inspect", ident) + require(current["Created"] == created, "Network birth differs") + self.command(["docker", "network", "rm", ident]) + require( + not self.ids( + "container", + "-a", + "--filter", + "label=com.docker.compose.project=" + self.project, + ) + and not self.ids( + "network", "--filter", "label=com.docker.compose.project=" + self.project + ) + and not self.ids( + "volume", "--filter", "label=com.docker.compose.project=" + self.project + ), + "Compose resources remain", + ) + if getattr(self, "ch_attempted", False): + self.capture_ch() + self.command( + ["docker", "container", "stop", "--time", "10", self.ch_id], timeout=20 + ) + self.command(["docker", "container", "rm", "-v", self.ch_id]) + require( + self.ch_id not in self.ids("container", "-a", "--no-trunc"), + "ClickHouse remains", + ) + require( + not (set(self.ch_volumes) & self.ids("volume")), "ClickHouse volumes remain" + ) + except Exception as error: # noqa: BLE001 — Unverified cleanup must invalidate the result. + errors.append(self.redact(error)) + self.record( + "cleanup.json", + { + "verified": not errors, + "errors": errors, + "runtime_files_removed": False, + "ephemeral_vm_disposal_is_not_cleanup_proof": True, + }, + ) + require(not errors, "Owned cleanup incomplete") + + +def cleanup_only(): + """Recover exact owned resources after an interrupted controller, then fail that run.""" + require( + os.environ.get("GITHUB_ACTIONS") == "true" + and os.environ.get("RUNNER_ENVIRONMENT") == "github-hosted" + and os.environ.get("RUNNER_OS") == "Linux", + "GitHub-hosted local Docker required", + ) + + def resolve_endpoint(): + result = subprocess.run( + ["docker", "context", "inspect", "--format", "{{json .Endpoints.docker.Host}}"], + capture_output=True, + text=True, + timeout=10, + ) + require(result.returncode == 0, "Cannot resolve effective Docker context") + return json.loads(result.stdout) + + endpoint = verify_docker_selection(os.environ, resolve_endpoint) + docker_env = dict(os.environ, DOCKER_HOST=endpoint) + evidence = Path(os.environ["GITHUB_WORKSPACE"]) / "dify-application-evidence" + completed = evidence / "cleanup.json" + if completed.is_file() and json.loads(completed.read_text()).get("verified") is True: + return + state_path = Path(os.environ["RUNNER_TEMP"]) / "dify-smoke-ownership.json" + require(state_path.is_file() and not state_path.is_symlink(), "Missing cleanup ownership state") + state = json.loads(state_path.read_text()) + require(state["docker_endpoint"] == endpoint, "Cleanup Docker endpoint changed") + require( + state["run_id"] == os.environ["GITHUB_RUN_ID"] + and state["attempt"] == os.environ["GITHUB_RUN_ATTEMPT"], + "Wrong job cleanup state", + ) + root = Path(state["root"]) + require( + root.is_relative_to(Path(os.environ["RUNNER_TEMP"]).resolve()) + and root.name.startswith("dify-smoke-") + and root.stat().st_uid == os.getuid(), + "Wrong cleanup root", + ) + errors = [] + + def command(*args): + result = subprocess.run( + docker_arguments(args, endpoint), + env=docker_env, + capture_output=True, + text=True, + timeout=30, + ) + require(result.returncode == 0, "Docker cleanup operation failed") + return result.stdout + + def inspect(kind, ident): + (row,) = json.loads(command(kind, "inspect", ident)) + return row + + try: + for entry in reversed(state["processes"]): + stop_owned_session( + entry, + observe=read_session, + current_birth=birth, + send=os.kill, + pause=time.sleep, + now=time.monotonic, + uid=os.getuid(), + ) + project = state["project"] + before = state["before"] + if state["compose_attempted"]: + identifiers = command( + "container", + "ls", + "-aq", + "--no-trunc", + "--filter", + "label=com.docker.compose.project=" + project, + ).split() + volumes = {} + for ident in identifiers: + row = inspect("container", ident) + labels = row["Config"].get("Labels") or {} + service = labels.get("com.docker.compose.service") + require( + ident not in before["container"] + and row["Created"] >= state["started"] + and labels.get("com.docker.compose.project") == project + and service in SERVICES + and row["Image"] == state["images"][service], + "Recovered container differs", + ) + for mount in row["Mounts"]: + if mount["Type"] == "bind": + require( + Path(mount["Source"]).resolve().is_relative_to(root / "dify"), + "Recovered bind differs", + ) + elif mount["Type"] == "volume": + volume = inspect("volume", mount["Name"]) + require( + mount["Name"] not in before["volume"] + and (volume.get("Labels") or {}).get("com.docker.compose.project") + == project, + "Recovered volume differs", + ) + volumes[mount["Name"]] = volume["CreatedAt"] + command("container", "stop", "--time", "10", ident) + command("container", "rm", "-v", ident) + for name in command( + "volume", "ls", "-q", "--filter", "label=com.docker.compose.project=" + project + ).split(): + row = inspect("volume", name) + require( + name not in before["volume"] + and (row.get("Labels") or {}).get("com.docker.compose.project") == project + and datetime.fromisoformat(row["CreatedAt"].replace("Z", "+00:00")) + >= datetime.fromisoformat(state["started"]).replace(microsecond=0), + "Recovered declared volume differs", + ) + volumes[name] = row["CreatedAt"] + remaining_volumes = command("volume", "ls", "-q").split() + for name, created in volumes.items(): + if name in remaining_volumes: + require( + inspect("volume", name)["CreatedAt"] == created, + "Recovered volume birth differs", + ) + command("volume", "rm", name) + for ident in command( + "network", + "ls", + "-q", + "--no-trunc", + "--filter", + "label=com.docker.compose.project=" + project, + ).split(): + row = inspect("network", ident) + require( + ident not in before["network"] + and row["Created"] >= state["started"] + and (row.get("Labels") or {}).get("com.docker.compose.project") == project, + "Recovered network differs", + ) + command("network", "rm", ident) + if state["ch_attempted"]: + matches = command( + "container", + "ls", + "-aq", + "--no-trunc", + "--filter", + "label=loom.dify-smoke=" + project, + ).split() + require(len(matches) <= 1, "Ambiguous recovered ClickHouse") + for ident in matches: + row = inspect("container", ident) + require( + ident not in before["container"] + and row["Created"] >= state["started"] + and row["Image"] == state["ch_image"] + and (row["Config"].get("Labels") or {}).get("loom.dify-smoke") == project, + "Recovered ClickHouse differs", + ) + command("container", "stop", "--time", "10", ident) + command("container", "rm", "-v", ident) + except Exception as error: # noqa: BLE001 — Preserve interrupted cleanup as a failure. + errors.append(type(error).__name__) + evidence.mkdir(exist_ok=True) + (evidence / "cleanup-fallback.json").write_text( + json.dumps( + { + "errors": errors, + "original_controller_cleanup_verified": False, + "status": "INCOMPLETE", + "vm_disposal_is_not_cleanup_proof": True, + } + ) + + "\n" + ) + # Recovered containment does not manufacture a successful controller or purge receipt. + raise SystemExit(1) + + +def main(): + runtime = Runtime() + + def interrupt(signum, frame): + if not runtime.cleaning: + raise RuntimeError("Interrupted signal " + str(signum)) + + signal.signal(signal.SIGTERM, interrupt) + signal.signal(signal.SIGINT, interrupt) + failure = None + try: + runtime.admit() + runtime.artifact() + runtime.platform() + runtime.fixture() + runtime.platform_ready() + # The scenario is ordinary tool use against this fixture, never the parent deployment. + from dify_application_scenario import run + + runtime.phase = "native-application" + run(runtime) + runtime.platform_ready() + require(runtime.monitor_error is None, "Resource monitor failed") + except Exception as error: # noqa: BLE001 — Record a bounded failure before owned cleanup. + failure = runtime.redact(error) + runtime.record( + "failure.json", + {"phase": runtime.phase, "error_type": type(error).__name__, "error": failure}, + ) + finally: + try: + runtime.cleanup() + except Exception as error: # noqa: BLE001 — Cleanup failure overrides an apparent success. + failure = failure or runtime.redact(error) + failure = final_failure(failure, runtime.monitor_error) + runtime.record( + "result.json", + { + "passed": failure is None, + "failure": failure, + "monitor_error": runtime.monitor_error, + "scope": "CE normal setup, workspace debug, three native workflow tools", + "cloud_marketplace_browser_protected_family_qualified": False, + }, + ) + if failure: + raise SystemExit(1) + + +if __name__ == "__main__": + if sys.argv[1:] == ["--cleanup-only"]: + cleanup_only() + else: + require(not sys.argv[1:], "Unexpected arguments") + main() diff --git a/server/operations/reach_manifest.py b/server/operations/reach_manifest.py index 796894952e..2880c5b073 100644 --- a/server/operations/reach_manifest.py +++ b/server/operations/reach_manifest.py @@ -350,6 +350,7 @@ "knowledge.update_metadata": ("dashboard",), "knowledge.upsert": ( "sdk_method", + "sdk_example", "plugin_hook", "dashboard", ), @@ -586,6 +587,7 @@ "namespace.mark_seen": ("dashboard",), "namespace.register": ( "sdk_method", + "sdk_example", "dashboard", ), "namespace.register_category": ( diff --git a/tests/unit/test_dify_application_guards.py b/tests/unit/test_dify_application_guards.py new file mode 100644 index 0000000000..0b57b504b0 --- /dev/null +++ b/tests/unit/test_dify_application_guards.py @@ -0,0 +1,241 @@ +"""Pure guard regressions; no runtime, Docker, network or real signal operation.""" + +import signal +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch +from zipfile import ZipInfo + +import httpx + +from scripts.gates import dify_application_scenario as scenario +from scripts.gates import dify_application_smoke as smoke +from tests.unit.clock_fake import ManualClock + + +class DifyApplicationGuards(unittest.TestCase): + def test_starting_fixture_can_reset_before_healthy_within_same_budget(self): + client = Mock() + client.get.side_effect = [ + httpx.ReadError("reset"), + httpx.Response(503), + httpx.Response(200), + ] + clock = ManualClock() + result = smoke.wait_fixture_ready( + client, "http://127.0.0.1/ping", 3, now=clock.monotonic, pause=clock.sleep + ) + self.assertEqual(result, [{"error_type": "ReadError"}, {"status": 503}, {"status": 200}]) + self.assertEqual(clock.t, 2) + self.assertEqual(client.get.call_count, 3) + + def test_persistent_fixture_reset_exhausts_original_readiness_budget(self): + client = Mock() + client.get.side_effect = httpx.ReadError("reset") + clock = ManualClock() + with self.assertRaisesRegex(RuntimeError, "readiness failed.*ReadError"): + smoke.wait_fixture_ready( + client, "http://127.0.0.1/ping", 2, now=clock.monotonic, pause=clock.sleep + ) + self.assertEqual(clock.t, 2) + self.assertEqual(client.get.call_count, 2) + + def test_fixture_permission_refusal_is_terminal_without_another_poll(self): + client = Mock() + client.get.return_value = httpx.Response(403) + clock = ManualClock() + with self.assertRaisesRegex(RuntimeError, "permission refused"): + smoke.wait_fixture_ready( + client, "http://127.0.0.1/ping", 2, now=clock.monotonic, pause=clock.sleep + ) + self.assertEqual(client.get.call_count, 1) + self.assertEqual(clock.t, 0) + + def test_credentials_stay_in_memory_and_are_redacted_without_stdout(self): + runtime = smoke.Runtime.__new__(smoke.Runtime) + runtime.secrets = [] + with patch("builtins.print") as output: + self.assertEqual(runtime.secret("test-credential"), "test-credential") + output.assert_not_called() + self.assertEqual(runtime.redact("value=test-credential"), "value=[REDACTED]") + + def test_uncovered_image_volume_gets_explicit_owned_compose_volume(self): + service = {"volumes": [{"target": "/var/lib/postgresql/data/pgdata"}]} + image = {"Config": {"Volumes": {"/var/lib/postgresql/data": {}}}} + mounts, volumes = smoke.image_volume_mounts("db_postgres", service, image) + self.assertEqual( + mounts, + [ + { + "type": "volume", + "source": "image-db_postgres-0", + "target": "/var/lib/postgresql/data", + } + ], + ) + self.assertEqual(volumes, {"image-db_postgres-0": {}}) + self.assertEqual(service["volumes"], [{"target": "/var/lib/postgresql/data/pgdata"}]) + + def test_covered_image_volume_needs_no_extra_mount(self): + service = {"volumes": [{"target": "/data"}]} + image = {"Config": {"Volumes": {"/data": {}}}} + self.assertEqual(smoke.image_volume_mounts("redis", service, image), ([], {})) + + def test_invalid_image_volume_target_refuses(self): + for target in ("/", "relative", "/data/../foreign"): + with ( + self.subTest(target=target), + self.assertRaisesRegex(RuntimeError, "target is invalid"), + ): + smoke.image_volume_mounts("service", {}, {"Config": {"Volumes": {target: {}}}}) + + def test_init_password_respects_pinned_schema(self): + for _ in range(8): + password = smoke.initial_password() + self.assertEqual(len(password), 27) + self.assertTrue(password.startswith("Aa7")) + + def test_remote_current_context_refuses_before_bound_operation(self): + resolve = Mock(return_value="ssh://remote-daemon") + with self.assertRaisesRegex(RuntimeError, "Effective Docker context"): + smoke.verify_docker_selection({}, resolve) + resolve.assert_called_once_with() + + def test_docker_selector_overrides_refuse_before_context_lookup(self): + for name in ("DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG"): + with self.subTest(name=name): + resolve = Mock() + with self.assertRaisesRegex(RuntimeError, "overrides"): + smoke.verify_docker_selection({name: "foreign"}, resolve) + resolve.assert_not_called() + + def test_verified_endpoint_is_bound_to_every_docker_operation(self): + endpoint = smoke.verify_docker_selection({}, lambda: smoke.LOCAL_DOCKER) + for command in (["container", "inspect", "owned"], ["container", "rm", "owned"]): + self.assertEqual( + smoke.docker_arguments(command, endpoint), + ["docker", "--host", smoke.LOCAL_DOCKER, *command], + ) + with self.assertRaisesRegex(RuntimeError, "unverified"): + smoke.docker_arguments(["container", "rm", "owned"], "ssh://remote") + + def test_same_archive_count_cannot_replace_license_with_excluded_test(self): + members = [ZipInfo(name) for name in smoke.EXPECTED_PLUGIN_FILES if name != "LICENSE"] + members.append(ZipInfo("tests/native_concurrency_case.py")) + self.assertEqual(len(members), 19) + with self.assertRaisesRegex(RuntimeError, "reviewed set"): + smoke.validate_archive_members(members) + + def test_archive_rejects_duplicate_and_directory_members(self): + valid = [ZipInfo(name) for name in smoke.EXPECTED_PLUGIN_FILES] + smoke.validate_archive_members(valid) + for extra in (ZipInfo("LICENSE"), ZipInfo("tools/")): + with self.subTest(extra=extra.filename), self.assertRaises(RuntimeError): + smoke.validate_archive_members([*valid, extra]) + + def test_native_graph_keeps_application_user_in_form_configuration(self): + graph = scenario.build_graph( + {"start": {}, "end": {}}, + {"id": "discovered-provider", "name": "loom"}, + "recall", + "application-user", + {"query": "aisle seats"}, + ) + tool = next(node["data"] for node in graph["nodes"] if node["id"] == "tool") + self.assertEqual(tool["provider_id"], "discovered-provider") + self.assertEqual( + tool["tool_configurations"]["user_id"], + {"type": "constant", "value": "application-user"}, + ) + self.assertNotIn("user_id", tool["tool_parameters"]) + self.assertEqual(tool["tool_parameters"]["query"]["value"], "aisle seats") + self.assertEqual(tool["tool_configurations"]["top_k"]["value"], 5) + + @staticmethod + def saved_session(): + return {"pid": 41, "start": 8, "members": {41: (8, 41, 41, 1000)}} + + def stop_mocked(self, saved, snapshot, send): + smoke.stop_owned_session( + saved, + observe=lambda _: dict(snapshot), + current_birth=lambda pid: snapshot.get(pid, ())[:3], + send=send, + pause=Mock(side_effect=AssertionError("Unexpected wait")), + now=lambda: 0, + uid=1000, + ) + + def test_reused_session_leader_is_never_signalled(self): + send = Mock() + with self.assertRaisesRegex(RuntimeError, "leader generation"): + self.stop_mocked(self.saved_session(), {41: (99, 41, 41, 1000)}, send) + send.assert_not_called() + + def test_unrecorded_orphan_is_never_adopted_or_signalled(self): + send = Mock() + with self.assertRaisesRegex(RuntimeError, "Orphan member generation"): + self.stop_mocked(self.saved_session(), {42: (9, 41, 41, 1000)}, send) + send.assert_not_called() + + def test_exact_live_leader_allows_owned_members_and_saves_their_generations(self): + saved = self.saved_session() + snapshot = {41: (8, 41, 41, 1000), 42: (9, 41, 41, 1000)} + send = Mock(side_effect=lambda pid, _: snapshot.pop(pid)) + self.stop_mocked(saved, snapshot, send) + self.assertEqual( + send.call_args_list, + [unittest.mock.call(41, signal.SIGTERM), unittest.mock.call(42, signal.SIGTERM)], + ) + self.assertEqual(saved["members"][42], (9, 41, 41, 1000)) + + def test_exact_saved_orphan_remains_owned_without_leader(self): + saved = self.saved_session() + saved["members"][42] = (9, 41, 41, 1000) + snapshot = {42: (9, 41, 41, 1000)} + send = Mock(side_effect=lambda pid, _: snapshot.pop(pid)) + self.stop_mocked(saved, snapshot, send) + send.assert_called_once_with(42, signal.SIGTERM) + + def test_completed_empty_command_retires_from_persisted_ownership(self): + runtime = smoke.Runtime.__new__(smoke.Runtime) + entry = {"child": SimpleNamespace(pid=41), "birth": 8, "members": {41: (8, 41, 41, 1000)}} + runtime.processes = [entry] + runtime.persist = Mock() + with patch.object(smoke, "read_session", return_value={}): + runtime.retire(entry) + self.assertEqual(runtime.processes, []) + runtime.persist.assert_called_once_with() + + def test_completed_command_cannot_retire_over_reused_leader(self): + runtime = smoke.Runtime.__new__(smoke.Runtime) + entry = {"child": SimpleNamespace(pid=41), "birth": 8, "members": {41: (8, 41, 41, 1000)}} + runtime.processes = [entry] + runtime.persist = Mock() + with patch.object(smoke, "read_session", return_value={41: (99, 41, 41, 1000)}): + with self.assertRaisesRegex(RuntimeError, "leader generation"): + runtime.retire(entry) + runtime.persist.assert_not_called() + + def test_monitor_failure_arriving_during_join_invalidates_final_success(self): + runtime = SimpleNamespace(finished=Mock(), monitor_error=None) + runtime.monitor_thread = Mock() + runtime.monitor_thread.is_alive.return_value = False + runtime.monitor_thread.join.side_effect = lambda **_: setattr( + runtime, "monitor_error", "late reserve failure" + ) + smoke.stop_monitor(runtime) + self.assertEqual(smoke.final_failure(None, runtime.monitor_error), "late reserve failure") + runtime.finished.set.assert_called_once_with() + runtime.monitor_thread.join.assert_called_once_with(timeout=5) + + def test_live_monitor_after_join_fails_closed(self): + runtime = SimpleNamespace(finished=Mock(), monitor_error=None, monitor_thread=Mock()) + runtime.monitor_thread.is_alive.return_value = True + with self.assertRaisesRegex(RuntimeError, "monitor did not stop"): + smoke.stop_monitor(runtime) + self.assertEqual(smoke.final_failure("cleanup failed", None), "cleanup failed") + + +if __name__ == "__main__": + unittest.main()