diff --git a/application/route.py b/application/route.py index b8d7a91..e1723c2 100644 --- a/application/route.py +++ b/application/route.py @@ -773,17 +773,37 @@ def save_bash_malicious_model(): try: from application.vulnerabilities.supply_chain import save_bash_malicious_model result = save_bash_malicious_model() - + if result['success']: return jsonify(result) else: return jsonify(result), 500 - + except Exception as e: return jsonify({ 'success': False, 'error': f'Error saving bash model: {str(e)}' }), 500 + + +@application.app.route('/load-saved-model', methods=['POST']) +def load_saved_model_route(): + """Load the saved bash model file and report whether its __reduce__ payload executed. + + The file is written by /save-bash-malicious-model. Body: {"hardened": bool}. + hardened=false uses an unsafe pickle.load (payload runs -> solve); hardened=true + uses a restricted loader that refuses the payload. + """ + try: + from application.vulnerabilities.supply_chain import load_saved_model + data = request.get_json(silent=True) or {} + result = load_saved_model(hardened=bool(data.get('hardened'))) + return jsonify(result) if result.get('success') else (jsonify(result), 400) + except Exception as e: + return jsonify({ + 'success': False, + 'error': f'Error loading saved model: {str(e)}' + }), 500 @application.app.route('/data-poisoning') def data_poisoning_main(): diff --git a/application/templates/supply_chain.html b/application/templates/supply_chain.html index 3c68e54..b4e6ff8 100644 --- a/application/templates/supply_chain.html +++ b/application/templates/supply_chain.html @@ -10,12 +10,28 @@ - + + {% include 'navbar.html' %}

Supply Chain Vulnerability

- +
Description @@ -25,7 +41,7 @@

The Hidden Risk: AI Models Supply Chain Attacks

LLMs come with a complex supply chain that introduces a lot of potential vulnerabilities. One big risk is relying on pre-trained models from public hubs, while convenient, these model binaries might be tampered with or contain hidden backdoors.

Even trusted platforms can miss malicious uploads, meaning developers could unknowingly integrate compromised models into their systems. Once a bad model is in place, it can quietly affect performance or leak data without obvious signs.

-

But that’s just one piece of the puzzle. The whole LLM pipeline, from collecting and cleaning data to using third-party libraries and deploying apps, offers entry points for attackers.

+

But that’s just one piece of the puzzle. The whole LLM pipeline, from collecting and cleaning data to using third-party libraries and deploying apps, offers entry points for attackers.

Poisoned datasets, buggy frameworks, or insecure optimization tools can all introduce problems that ripple downstream. To stay safe, developers need to treat every part of the chain with caution and assume that any component could be a target. Being proactive about verifying sources and testing for vulnerabilities is key to building secure LLM systems.

You can use tools like ModelScan to scan models for malicious code before deploying them.

This page demonstrates LLM03:2025 Supply Chain, as defined in the OWASP Top 10 for LLM Applications 2025.

@@ -33,172 +49,179 @@

The Hidden Risk: AI Models Supply Chain Attacks

- - + +
Demonstration
- +
Mitigation Strategies
- + + + - \ No newline at end of file + diff --git a/application/vulnerabilities/supply_chain.py b/application/vulnerabilities/supply_chain.py index 27496b1..b7d8f48 100644 --- a/application/vulnerabilities/supply_chain.py +++ b/application/vulnerabilities/supply_chain.py @@ -2,6 +2,8 @@ from sklearn.linear_model import LogisticRegression import pickle import os +import io +import uuid class SentimentModel_JS_malicious: def __init__(self): @@ -171,7 +173,9 @@ def __init__(self): self.model = None self.vocab = None self.executed_commands = [] - + # Token tying this instance to the marker its __reduce__ payload drops on load. + self.token = uuid.uuid4().hex + # Execute multiple dangerous commands to demonstrate the attack commands_to_execute = [ "cat /etc/passwd", @@ -309,6 +313,21 @@ def predict_proba(self, text): vector = self.vectorizer.transform([text]) return self.model.predict_proba(vector)[0] + def __reduce__(self): + """Load-time payload: unpickling this model runs attacker-controlled code. + + The commands in __init__ only fire when the SERVER constructs the object. + __reduce__ is the hook pickle.load / torch.load call while restoring a FILE, + so merely loading the saved .pkl re-executes code -- the real supply-chain + shape. The payload is deliberately benign: it calls _record_load_proof, which + drops a marker file instead of running shell commands, so the saved artifact + is safe to ship while still proving code ran on load. A real attacker would + return (os.system, ("...",)) here -- exactly the global ModelScan / picklescan + flag, which is why the fix is to scan artifacts and load with a restricted + (weights-only) loader. + """ + return (_record_load_proof, (self.token,)) + def save_js_malicious_model(): """ Create and save the JavaScript malicious model as a pickle file. @@ -346,23 +365,21 @@ def save_bash_malicious_model(): """ try: model = SentimentModel_bash_malicious() - - # Create downloads directory if it doesn't exist - downloads_dir = os.path.join(os.getcwd(), 'downloads') - os.makedirs(downloads_dir, exist_ok=True) - - # Save the model as pickle - model_path = os.path.join(downloads_dir, 'malicious_bash_sentiment_model.pkl') + + # Save the model as pickle. Because the class defines __reduce__, the saved + # file carries a load-time payload: loading it back re-executes code. + model_path = os.path.join(_downloads_dir(), BASH_MODEL_FILENAME) with open(model_path, 'wb') as f: pickle.dump(model, f) - + return { 'success': True, 'message': 'Malicious bash model saved successfully', 'file_path': model_path, - 'filename': 'malicious_bash_sentiment_model.pkl', + 'filename': BASH_MODEL_FILENAME, + 'token': model.token, 'commands_executed': model.executed_commands, - 'scan_note': 'You can now scan this file with tools like ModelScan to detect malicious code!' + 'scan_note': 'Scan this file with ModelScan / picklescan, or load it below to see the __reduce__ payload fire on load.' } except Exception as e: @@ -370,3 +387,128 @@ def save_bash_malicious_model(): 'success': False, 'error': f'Error saving malicious bash model: {str(e)}' } + + +# --------------------------------------------------------------------------- +# Load-time deserialization win condition (issue #32, item 2) +# +# SentimentModel_bash_malicious above carries a __reduce__ method, so unpickling +# its saved .pkl re-executes code -- the real supply-chain shape (the JS/bash +# payloads in __init__ only fire when the SERVER constructs the object; pickle +# restores fields without calling __init__). The __reduce__ payload is benign: +# it writes a marker file via _record_load_proof instead of running shell +# commands, so the artifact is safe to ship while still proving code ran on load. +# load_saved_model drives the win condition and the hardened-loader fix below. +# --------------------------------------------------------------------------- + +LOAD_PROOF_PREFIX = '.load_proof_' +BASH_MODEL_FILENAME = 'malicious_bash_sentiment_model.pkl' + + +def _downloads_dir(): + downloads_dir = os.path.join(os.getcwd(), 'downloads') + os.makedirs(downloads_dir, exist_ok=True) + return downloads_dir + + +def _proof_path(token): + return os.path.join(_downloads_dir(), f'{LOAD_PROOF_PREFIX}{token}') + + +def _record_load_proof(token): + """Benign side effect proving that unpickling executed attacker-controlled code. + + Referenced from SentimentModel_bash_malicious.__reduce__, so pickle.load calls + it while restoring the file. It only drops a marker file; nothing sensitive or + destructive runs. Returns the marker path, which becomes the "restored object". + """ + path = _proof_path(token) + with open(path, 'w') as f: + f.write(token) + return path + + +# Globals a weights-only / data-only load may legitimately need. Anything else +# (os.system, subprocess, our _record_load_proof, ...) is refused. +SAFE_ALLOWED_GLOBALS = { + ('builtins', 'list'), + ('builtins', 'dict'), + ('builtins', 'tuple'), + ('builtins', 'set'), + ('builtins', 'int'), + ('builtins', 'float'), + ('builtins', 'str'), + ('builtins', 'bool'), + ('builtins', 'complex'), + ('builtins', 'bytes'), +} + + +class RestrictedUnpickler(pickle.Unpickler): + """Minimal stand-in for torch.load(weights_only=True): allow only safe data types.""" + + def find_class(self, module, name): + if (module, name) in SAFE_ALLOWED_GLOBALS: + return super().find_class(module, name) + raise pickle.UnpicklingError( + f'blocked unsafe global {module}.{name} (hardened load allows data types only)' + ) + + +def _safe_load_bytes(raw): + return RestrictedUnpickler(io.BytesIO(raw)).load() + + +def evaluate_supply_chain_solve(payload_executed, hardened): + """A solve = loading an untrusted model file executed attacker code. + + Counts only when the payload actually ran AND the unsafe loader was used. + Hardened mode must never count as solved. + """ + solved = bool(payload_executed) and not hardened + return { + 'solved': solved, + 'payload_executed': bool(payload_executed), + 'hardened': bool(hardened), + } + + +def load_saved_model(hardened=False): + """Load the saved tampered model file and report whether its payload executed. + + Loads the bash model's saved .pkl (SentimentModel_bash_malicious.__reduce__). + hardened=False -> plain pickle.load (vulnerable): the __reduce__ payload runs. + hardened=True -> RestrictedUnpickler (fixed): the payload's global is refused, + so nothing executes. + """ + model_path = os.path.join(_downloads_dir(), BASH_MODEL_FILENAME) + if not os.path.exists(model_path): + return { + 'success': False, + 'error': 'No saved model file. Use "Save Model" on the OS command demo first.', + } + + with open(model_path, 'rb') as f: + raw = f.read() + + if hardened: + try: + _safe_load_bytes(raw) + blocked = False + detail = 'Hardened loader returned without executing any payload.' + except pickle.UnpicklingError as e: + blocked = True + detail = str(e) + payload_executed = False + else: + loaded = pickle.loads(raw) + payload_executed = isinstance(loaded, str) and os.path.exists(loaded) + blocked = False + detail = ( + f'pickle.load executed the __reduce__ payload; it wrote {os.path.basename(loaded)}.' + if payload_executed else 'Load completed but produced no payload marker.' + ) + + result = evaluate_supply_chain_solve(payload_executed=payload_executed, hardened=hardened) + result.update({'success': True, 'hardened': hardened, 'blocked': blocked, 'detail': detail}) + return result diff --git a/docs/labs.md b/docs/labs.md index f6023e9..a49e6db 100644 --- a/docs/labs.md +++ b/docs/labs.md @@ -7,7 +7,7 @@ | 1 | Learning to get weights from a model | [/model-theft](#model-theft-attack) | [LLM10:2025 \- Unbounded Consumption](https://genai.owasp.org/llmrisk/llm102025-unbounded-consumption/) | | 2 | Swapping negative and positive data | [/data-poisoning](#model-theft-attack) | [LLM04:2025 \- Data and Model Poisoning](https://genai.owasp.org/llmrisk/llm042025-data-and-model-poisoning/) | | 3 | Adding wrong data in vector database | [/catering-rag](#model-theft-attack) | [LLM08:2025 \- Vector and Embedding Weaknesses](https://genai.owasp.org/llmrisk/llm082025-vector-and-embedding-weaknesses/) | -| 4 | Libraries that can cause vulnerabilities | [/supply-chain](#model-theft-attack) | [LLM03:2025 \- Supply Chain Vulnerabilities](https://www.google.com/search?q=https://genai.owasp.org/llmrisk/llm032025-supply-chain-vulnerabilities/) | +| 4 | Libraries that can cause vulnerabilities | [/supply-chain](#supply-chain-vulnerability) | [LLM03:2025 \- Supply Chain Vulnerabilities](https://www.google.com/search?q=https://genai.owasp.org/llmrisk/llm032025-supply-chain-vulnerabilities/) | | 5 | Exploring SQL injection in LLM apps | [/insecure-plugin](#model-theft-attack) | [LLM05:2025 \- Improper Output Handling](https://genai.owasp.org/llmrisk/llm052025-improper-output-handling/) | | 6 | Prompt injection | [/direct-prompt-injection](#model-theft-attack) | [LLM01:2025 \- Prompt Injection](https://www.google.com/search?q=https://genai.owasp.org/llmrisk/llm012025-prompt-injection/) | | 7 | Prompt injection with guard rails | [/direct-prompt-injection/guardrail-ladder](#model-theft-attack) | [LLM01:2025 \- Prompt Injection](https://www.google.com/search?q=https://genai.owasp.org/llmrisk/llm012025-prompt-injection/) | @@ -114,13 +114,13 @@ Offers a **Hardened retrieval** run option that restricts the RAG system to trus Description: **Analyze how insecure third-party dependencies and libraries can introduce hidden security flaws into an AI application.** -**Objective:** Execute code via model file. +**Objective:** Load an untrusted model file so its hidden payload runs on deserialization, then confirm that hardened (weights-only) loading neutralises it. **Threat Model:** Elevation of Privilege: Achieving code execution via deserialization. **Difficulty Rating:** Advanced -**Hint:** Look at the malicious code snippets for XSS or Bash models; loading the whole object via torch.load() or pickle automatically triggers the underlying Python script. +**Hint:** The XSS payload lives in `__init__`, so it fires when the server *constructs* the model (view the page source of the demo page). The OS command (Bash) model additionally defines `__reduce__`, which pickle *does* call while restoring a file — so loading its saved `.pkl` runs code on load. In the **OS Command Execution** demo, click **Save Model**, then with **Hardened load** off click **Load saved file**: the unsafe `pickle.load` runs the payload and the lab is solved. Turn **Hardened load** on and load again — the restricted (weights-only style) loader refuses the payload, so it is not solved. The bundled payload only writes a marker file; a real attacker would return `(os.system, (...))` from `__reduce__`, which is what ModelScan / picklescan flag. ![Supply Chain Vulnerability](assets/images/labs/image6.png) diff --git a/tests/functional/test_vulnerability_workflows.py b/tests/functional/test_vulnerability_workflows.py index c51aab1..dec8e2c 100644 --- a/tests/functional/test_vulnerability_workflows.py +++ b/tests/functional/test_vulnerability_workflows.py @@ -287,6 +287,32 @@ def test_malicious_model_loading_workflow(self, client, test_app): assert 'warning' in data assert 'commands_executed' in data or 'message' in data + def test_reduce_model_load_solves_lab(self, client, test_app): + """Loading the saved bash model with the unsafe loader solves the lab.""" + with test_app.app_context(): + # Save the tampered model; its __reduce__ payload fires on load, not save. + response = client.post('/save-bash-malicious-model') + assert response.status_code == 200 + assert json.loads(response.data)['success'] is True + + # Unsafe load: payload fires -> solved. + response = client.post('/load-saved-model', json={'hardened': False}) + assert response.status_code == 200 + data = json.loads(response.data) + assert data['payload_executed'] is True + assert data['solved'] is True + + def test_reduce_model_hardened_load_blocks_payload(self, client, test_app): + """Hardened load refuses the payload and does not count as solved.""" + with test_app.app_context(): + client.post('/save-bash-malicious-model') + response = client.post('/load-saved-model', json={'hardened': True}) + assert response.status_code == 200 + data = json.loads(response.data) + assert data['payload_executed'] is False + assert data['blocked'] is True + assert data['solved'] is False + class TestDoSWorkflow: """Test Denial of Service demonstration workflows.""" diff --git a/tests/unit/test_vulnerabilities.py b/tests/unit/test_vulnerabilities.py index b4ccd4c..440b3a9 100644 --- a/tests/unit/test_vulnerabilities.py +++ b/tests/unit/test_vulnerabilities.py @@ -287,3 +287,39 @@ def test_data_poisoning_mixed_sentiments(self, test_app): model_data = data_poisoning.create_new_model_with_poisoned_data(mixed_comments) assert 'vocabulary_size' in model_data assert model_data['vocabulary_size'] > 0 + + +class TestSupplyChainDeserialization: + """Tests for the load-time __reduce__ payload and its win condition.""" + + def test_evaluate_solve_matrix(self): + from application.vulnerabilities.supply_chain import evaluate_supply_chain_solve + # Solve only when the payload ran AND the unsafe loader was used. + assert evaluate_supply_chain_solve(payload_executed=True, hardened=False)['solved'] is True + assert evaluate_supply_chain_solve(payload_executed=True, hardened=True)['solved'] is False + assert evaluate_supply_chain_solve(payload_executed=False, hardened=False)['solved'] is False + + def test_reduce_global_is_benign_marker(self): + """__reduce__ must carry the benign marker writer, never a shell gadget.""" + from application.vulnerabilities import supply_chain as sc + # Build an instance without running __init__ (which would shell out). + obj = sc.SentimentModel_bash_malicious.__new__(sc.SentimentModel_bash_malicious) + obj.token = 'unit-reduce-check' + func, args = obj.__reduce__() + assert func is sc._record_load_proof + assert args == ('unit-reduce-check',) + + def test_reduce_payload_fires_on_unsafe_load_only(self, test_app): + from application.vulnerabilities import supply_chain as sc + with test_app.app_context(): + saved = sc.save_bash_malicious_model() + assert saved['success'] is True + + unsafe = sc.load_saved_model(hardened=False) + assert unsafe['payload_executed'] is True + assert unsafe['solved'] is True + + hardened = sc.load_saved_model(hardened=True) + assert hardened['payload_executed'] is False + assert hardened['blocked'] is True + assert hardened['solved'] is False