Skip to content

Commit 018ba92

Browse files
BazookaMusicCopilot
andcommitted
Add additional Python prompt-injection sinks for uncovered SDK methods
Cover prompt-carrying public API methods that were missing from the framework models: - OpenAI: videos.create/create_and_poll/edit/remix/extend (Sora, user), beta.realtime.sessions.create instructions (system), and role-filtered beta.threads.messages.create content (Assistants API). - Anthropic: legacy completions.create prompt (user). - agents: Agent.as_tool tool_description (system). - Google GenAI: caches.create CreateCachedContentConfig system_instruction (system) and contents (user). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 8e5f214 commit 018ba92

13 files changed

Lines changed: 142 additions & 38 deletions

File tree

python/ql/lib/semmle/python/frameworks/OpenAI.qll

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,18 @@ module OpenAI {
103103
result = msg.getSubscript("content").getASubscript().getSubscript("text")
104104
}
105105

106+
/** Gets the `beta.threads.messages.create` call (Assistants API thread messages). */
107+
private API::Node threadMessageCreate() {
108+
result =
109+
classRef().getMember("beta").getMember("threads").getMember("messages").getMember("create")
110+
}
111+
112+
/** Holds if the `role` keyword of thread-message `call` is a privileged (assistant) role. */
113+
private predicate threadRoleIsAssistant(API::Node call) {
114+
call.getKeywordParameter("role").getAValueReachingSink().asExpr().(StringLiteral).getText() =
115+
"assistant"
116+
}
117+
106118
/**
107119
* Gets role-filtered system/developer/assistant message content sinks that
108120
* MaD cannot express.
@@ -111,6 +123,10 @@ module OpenAI {
111123
exists(API::Node msg | msg = [chatMessage(), responsesMessage()] and isSystemOrDevMessage(msg) |
112124
result = messageContent(msg)
113125
)
126+
or
127+
exists(API::Node call | call = threadMessageCreate() and threadRoleIsAssistant(call) |
128+
result = call.getKeywordParameter("content")
129+
)
114130
}
115131

116132
/**
@@ -124,6 +140,10 @@ module OpenAI {
124140
result = messageContent(msg)
125141
)
126142
or
143+
exists(API::Node call | call = threadMessageCreate() and not threadRoleIsAssistant(call) |
144+
result = call.getKeywordParameter("content")
145+
)
146+
or
127147
// realtime conversation items, role cannot be statically resolved in general
128148
result =
129149
classRef()

python/ql/lib/semmle/python/frameworks/agent.model.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ extensions:
66
# Agent instructions, handoff descriptions and tool descriptions are system-level prompts
77
- ['agents', 'Member[Agent].Argument[instructions:]', 'system-prompt-injection']
88
- ['agents', 'Member[Agent].Argument[handoff_description:]', 'system-prompt-injection']
9+
- ['agents', 'Member[Agent].ReturnValue.Member[as_tool].Argument[1,tool_description:]', 'system-prompt-injection']
910
- ['agents', 'Member[FunctionTool].Argument[description:]', 'system-prompt-injection']
1011
# The input passed to a run is user-level content
1112
- ['agents', 'Member[Runner].Member[run,run_sync,run_streamed].Argument[1]', 'user-prompt-injection']

python/ql/lib/semmle/python/frameworks/anthropic.model.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ extensions:
1010
- ['Anthropic', 'Member[beta].Member[messages].Member[create,stream].Argument[system:].ListElement.DictionaryElement[text]', 'system-prompt-injection']
1111
# The managed agents `system` field is a system-level prompt
1212
- ['Anthropic', 'Member[beta].Member[agents].Member[create,update].Argument[system:]', 'system-prompt-injection']
13+
# The legacy Text Completions API `prompt` is user-level content
14+
- ['Anthropic', 'Member[completions].Member[create].Argument[prompt:]', 'user-prompt-injection']
1315

1416
- addsTo:
1517
pack: codeql/python-all

python/ql/lib/semmle/python/frameworks/google-genai.model.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ extensions:
55
data:
66
# `system_instruction` on the generation config is a system-level prompt
77
- ['google.genai', 'Member[types].Member[GenerateContentConfig].Argument[system_instruction:]', 'system-prompt-injection']
8+
# Cached content carries a system instruction and user content
9+
- ['google.genai', 'Member[types].Member[CreateCachedContentConfig].Argument[system_instruction:]', 'system-prompt-injection']
10+
- ['google.genai', 'Member[types].Member[CreateCachedContentConfig].Argument[contents:]', 'user-prompt-injection']
811
# User-level content
912
- ['GoogleGenAI', 'Member[models].Member[generate_content,generate_content_stream].Argument[contents:]', 'user-prompt-injection']
1013
- ['GoogleGenAI', 'Member[models].Member[generate_images,generate_videos,edit_image].Argument[prompt:]', 'user-prompt-injection']

python/ql/lib/semmle/python/frameworks/openai.model.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,15 @@ extensions:
99
- ['OpenAI', 'Member[beta].Member[assistants].Member[update].Argument[instructions:]', 'system-prompt-injection']
1010
- ['OpenAI', 'Member[beta].Member[threads].Member[runs].Member[create].Argument[instructions:]', 'system-prompt-injection']
1111
- ['OpenAI', 'Member[beta].Member[threads].Member[runs].Member[create].Argument[additional_instructions:]', 'system-prompt-injection']
12+
# The default system instructions for a realtime session
13+
- ['OpenAI', 'Member[beta].Member[realtime].Member[sessions].Member[create].Argument[instructions:]', 'system-prompt-injection']
1214
# User-level prompts
1315
- ['OpenAI', 'Member[responses].Member[create].Argument[input:]', 'user-prompt-injection']
1416
- ['OpenAI', 'Member[completions].Member[create].Argument[prompt:]', 'user-prompt-injection']
1517
- ['OpenAI', 'Member[images].Member[generate,edit].Argument[prompt:]', 'user-prompt-injection']
1618
- ['OpenAI', 'Member[audio].Member[transcriptions,translations].Member[create].Argument[prompt:]', 'user-prompt-injection']
19+
# Sora video generation prompts are user-level content
20+
- ['OpenAI', 'Member[videos].Member[create,create_and_poll,edit,remix,extend].Argument[prompt:]', 'user-prompt-injection']
1721

1822
- addsTo:
1923
pack: codeql/python-all

0 commit comments

Comments
 (0)