Skip to content

Commit 041d873

Browse files
BazookaMusicCopilot
andcommitted
Python: route prompt-injection models through the AIPrompt concept
Expose system/user classification on AIPrompt and contribute role-sensitive framework sinks and data-extension sinks through that concept. Keep query customizations framework-independent without changing their results. Add a concept regression test covering all six SDK families, document the supported frameworks, and correct the release notes for the new queries and library API. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent c72c7d5 commit 041d873

15 files changed

Lines changed: 183 additions & 51 deletions

File tree

‎docs/codeql/reusables/supported-frameworks.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,12 @@ and the CodeQL library pack ``codeql/python-all`` (`changelog <https://github.co
281281
libxml2, XML processing library
282282
lxml, XML processing library
283283
xmltodict, XML processing library
284+
anthropic, AI model SDK
285+
google-genai, AI model SDK
286+
langchain, AI framework
287+
openai, AI model SDK
288+
openai-agents, AI agent framework
289+
openrouter, AI model SDK
284290

285291

286292
Ruby built-in support
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
---
2-
category: minorAnalysis
2+
category: feature
33
---
4-
* Added prompt-injection sink models (`system-prompt-injection` and `user-prompt-injection` kinds) for the `openai`, `agents`, `anthropic`, `google-genai`, `openrouter` and `langchain` frameworks.
4+
* Added the `AIPrompt` concept to model AI prompts and distinguish system-level prompts from user-level prompts. Added prompt-injection sink models (`system-prompt-injection` and `user-prompt-injection` kinds) for the `openai`, OpenAI Agents SDK (`agents`), `anthropic`, `google-genai`, `openrouter`, and `langchain` frameworks.

‎python/ql/lib/semmle/python/Concepts.qll‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1804,6 +1804,12 @@ module Cryptography {
18041804
class AIPrompt extends DataFlow::Node instanceof AIPrompt::Range {
18051805
/** Gets an input that is used as AI prompt. */
18061806
DataFlow::Node getAPrompt() { result = super.getAPrompt() }
1807+
1808+
/** Holds if this prompt supplies system-level instructions or assistant content. */
1809+
predicate isSystemPrompt() { super.isSystemPrompt() }
1810+
1811+
/** Holds if this prompt supplies user-level content. */
1812+
predicate isUserPrompt() { super.isUserPrompt() }
18071813
}
18081814

18091815
/** Provides a class for modeling new AI prompting mechanisms. */
@@ -1817,5 +1823,11 @@ module AIPrompt {
18171823
abstract class Range extends DataFlow::Node {
18181824
/** Gets an input that is used as AI prompt. */
18191825
abstract DataFlow::Node getAPrompt();
1826+
1827+
/** Holds if this prompt supplies system-level instructions or assistant content. */
1828+
predicate isSystemPrompt() { none() }
1829+
1830+
/** Holds if this prompt supplies user-level content. */
1831+
predicate isUserPrompt() { none() }
18201832
}
18211833
}

‎python/ql/lib/semmle/python/Frameworks.qll‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ private import semmle.python.frameworks.Aiomysql
1212
private import semmle.python.frameworks.Aiopg
1313
private import semmle.python.frameworks.Aiosqlite
1414
private import semmle.python.frameworks.Airspeed
15+
private import semmle.python.frameworks.Anthropic
1516
private import semmle.python.frameworks.Anyio
1617
private import semmle.python.frameworks.Asyncpg
1718
private import semmle.python.frameworks.Baize
@@ -34,6 +35,7 @@ private import semmle.python.frameworks.Flask
3435
private import semmle.python.frameworks.FlaskAdmin
3536
private import semmle.python.frameworks.FlaskSqlAlchemy
3637
private import semmle.python.frameworks.Genshi
38+
private import semmle.python.frameworks.GoogleGenAI
3739
private import semmle.python.frameworks.Gradio
3840
private import semmle.python.frameworks.Hdbcli
3941
private import semmle.python.frameworks.Httpx
@@ -54,6 +56,8 @@ private import semmle.python.frameworks.Multidict
5456
private import semmle.python.frameworks.Mysql
5557
private import semmle.python.frameworks.MySQLdb
5658
private import semmle.python.frameworks.Numpy
59+
private import semmle.python.frameworks.OpenAI
60+
private import semmle.python.frameworks.OpenRouter
5761
private import semmle.python.frameworks.Opml
5862
private import semmle.python.frameworks.Oracledb
5963
private import semmle.python.frameworks.Pandas

‎python/ql/lib/semmle/python/frameworks/Anthropic.qll‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
private import python
1313
private import semmle.python.ApiGraphs
14+
private import semmle.python.Concepts
1415

1516
/** Provides classes modeling prompt-injection sinks of the `anthropic` package. */
1617
module Anthropic {
@@ -55,4 +56,14 @@ module Anthropic {
5556
result = msg.getSubscript("content")
5657
)
5758
}
59+
60+
private class MessagePrompt extends AIPrompt::Range {
61+
MessagePrompt() { this = [getSystemOrAssistantPromptNode(), getUserPromptNode()].asSink() }
62+
63+
override DataFlow::Node getAPrompt() { result = this }
64+
65+
override predicate isSystemPrompt() { this = getSystemOrAssistantPromptNode().asSink() }
66+
67+
override predicate isUserPrompt() { this = getUserPromptNode().asSink() }
68+
}
5869
}

‎python/ql/lib/semmle/python/frameworks/GoogleGenAI.qll‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
private import python
1313
private import semmle.python.ApiGraphs
14+
private import semmle.python.Concepts
1415

1516
/** Provides classes modeling prompt-injection sinks of the `google-genai` package. */
1617
module GoogleGenAI {
@@ -55,4 +56,14 @@ module GoogleGenAI {
5556
result = msg.getSubscript("parts").getASubscript().getSubscript("text")
5657
)
5758
}
59+
60+
private class ContentPrompt extends AIPrompt::Range {
61+
ContentPrompt() { this = [getSystemOrAssistantPromptNode(), getUserPromptNode()].asSink() }
62+
63+
override DataFlow::Node getAPrompt() { result = this }
64+
65+
override predicate isSystemPrompt() { this = getSystemOrAssistantPromptNode().asSink() }
66+
67+
override predicate isUserPrompt() { this = getUserPromptNode().asSink() }
68+
}
5869
}

‎python/ql/lib/semmle/python/frameworks/OpenAI.qll‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
private import python
1616
private import semmle.python.ApiGraphs
17+
private import semmle.python.Concepts
1718

1819
/** Holds if `msg` is a message dictionary with a privileged (system/developer/assistant) role. */
1920
private predicate isSystemOrDevMessage(API::Node msg) {
@@ -65,6 +66,16 @@ module AgentSdk {
6566
result = msg.getSubscript("content")
6667
)
6768
}
69+
70+
private class RunnerPrompt extends AIPrompt::Range {
71+
RunnerPrompt() { this = [getSystemOrAssistantPromptNode(), getUserPromptNode()].asSink() }
72+
73+
override DataFlow::Node getAPrompt() { result = this }
74+
75+
override predicate isSystemPrompt() { this = getSystemOrAssistantPromptNode().asSink() }
76+
77+
override predicate isUserPrompt() { this = getUserPromptNode().asSink() }
78+
}
6879
}
6980

7081
/**
@@ -174,4 +185,14 @@ module OpenAI {
174185
result = realtimeItemContent(item)
175186
)
176187
}
188+
189+
private class ClientPrompt extends AIPrompt::Range {
190+
ClientPrompt() { this = [getSystemOrAssistantPromptNode(), getUserPromptNode()].asSink() }
191+
192+
override DataFlow::Node getAPrompt() { result = this }
193+
194+
override predicate isSystemPrompt() { this = getSystemOrAssistantPromptNode().asSink() }
195+
196+
override predicate isUserPrompt() { this = getUserPromptNode().asSink() }
197+
}
177198
}

‎python/ql/lib/semmle/python/frameworks/OpenRouter.qll‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
private import python
1010
private import semmle.python.ApiGraphs
11+
private import semmle.python.Concepts
1112

1213
/** Holds if `msg` is a message dictionary with a privileged (system/developer/assistant) role. */
1314
private predicate isSystemOrDevMessage(API::Node msg) {
@@ -57,4 +58,14 @@ module OpenRouter {
5758
result = messageContent(msg)
5859
)
5960
}
61+
62+
private class MessagePrompt extends AIPrompt::Range {
63+
MessagePrompt() { this = [getSystemOrAssistantPromptNode(), getUserPromptNode()].asSink() }
64+
65+
override DataFlow::Node getAPrompt() { result = this }
66+
67+
override predicate isSystemPrompt() { this = getSystemOrAssistantPromptNode().asSink() }
68+
69+
override predicate isUserPrompt() { this = getUserPromptNode().asSink() }
70+
}
6071
}

‎python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,22 @@ private class ThreatModelSourceFromDataExtension extends ThreatModelSource::Rang
3535
}
3636
}
3737

38+
private class AIPromptFromDataExtension extends AIPrompt::Range {
39+
AIPromptFromDataExtension() {
40+
this = ModelOutput::getASinkNode(["system-prompt-injection", "user-prompt-injection"]).asSink()
41+
}
42+
43+
override DataFlow::Node getAPrompt() { result = this }
44+
45+
override predicate isSystemPrompt() {
46+
this = ModelOutput::getASinkNode("system-prompt-injection").asSink()
47+
}
48+
49+
override predicate isUserPrompt() {
50+
this = ModelOutput::getASinkNode("user-prompt-injection").asSink()
51+
}
52+
}
53+
3854
private class SummarizedCallableFromModel extends SummarizedCallable::Range {
3955
string type;
4056
string path;

‎python/ql/lib/semmle/python/security/dataflow/SystemPromptInjectionCustomizations.qll‎

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,6 @@ private import semmle.python.Concepts
99
private import semmle.python.ApiGraphs
1010
private import semmle.python.dataflow.new.RemoteFlowSources
1111
private import semmle.python.dataflow.new.BarrierGuards
12-
private import semmle.python.frameworks.data.ModelsAsData
13-
private import semmle.python.frameworks.OpenAI
14-
private import semmle.python.frameworks.Anthropic
15-
private import semmle.python.frameworks.GoogleGenAI
16-
private import semmle.python.frameworks.OpenRouter
1712

1813
/**
1914
* Provides default sources, sinks and sanitizers for detecting
@@ -45,25 +40,7 @@ module SystemPromptInjection {
4540
* A prompt to an AI model, considered as a flow sink.
4641
*/
4742
class AIPromptAsSink extends Sink {
48-
AIPromptAsSink() { this = any(AIPrompt p).getAPrompt() }
49-
}
50-
51-
private class SinkFromModel extends Sink {
52-
SinkFromModel() { this = ModelOutput::getASinkNode("system-prompt-injection").asSink() }
53-
}
54-
55-
private class PromptContentSink extends Sink {
56-
PromptContentSink() {
57-
this = OpenAI::getSystemOrAssistantPromptNode().asSink()
58-
or
59-
this = AgentSdk::getSystemOrAssistantPromptNode().asSink()
60-
or
61-
this = Anthropic::getSystemOrAssistantPromptNode().asSink()
62-
or
63-
this = GoogleGenAI::getSystemOrAssistantPromptNode().asSink()
64-
or
65-
this = OpenRouter::getSystemOrAssistantPromptNode().asSink()
66-
}
43+
AIPromptAsSink() { this = any(AIPrompt p | p.isSystemPrompt()).getAPrompt() }
6744
}
6845

6946
/**

0 commit comments

Comments
 (0)