Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions crates/codegraph-extract/src/lang/c.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ impl LanguageSpec for CSpec {
fn variable_types(&self) -> &'static [&'static str] {
&["declaration"]
}
fn resolve_name(&self, node: Node<'_>, source: &str) -> Option<String> {
crate::lang::cpp::recover_single_arg_macro_defined_name(node, source)
}
fn name_field(&self) -> &'static str {
"declarator"
}
Expand Down
106 changes: 105 additions & 1 deletion crates/codegraph-extract/src/lang/cpp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ impl LanguageSpec for CppSpec {
"type"
}
fn resolve_name(&self, node: Node<'_>, source: &str) -> Option<String> {
if let Some(name) = recover_cpp_macro_defined_name(node, source) {
if let Some(name) = recover_single_arg_macro_defined_name(node, source)
.or_else(|| recover_cpp_macro_defined_name(node, source))
{
return Some(name);
}
let qid = declarator_qualified_id(child_by_field(node, "declarator")?)?;
Expand Down Expand Up @@ -921,6 +923,108 @@ fn mask_quoted_literal(bytes: &[u8], mask: &mut [bool], start: usize, quote: u8)
i
}

/// Recover the function a single-argument macro defines —
/// `#define NATIVE_FN(name) int name(void)` then `NATIVE_FN(get_version) { … }`
/// (upstream #1373). Without its definition the shape is ambiguous, so a
/// preceding local `#define` whose replacement uses its sole parameter as the
/// function declarator is required; registration, token-pasting and `typedef`
/// replacements, an `#undef` or redefinition in between, and any conditional
/// block leave the name alone. C parses the macro as the return type with a
/// `(name)` declarator; C++ as an implicit-return-type function.
pub(crate) fn recover_single_arg_macro_defined_name(
node: Node<'_>,
source: &str,
) -> Option<String> {
if node.kind() != "function_definition" {
return None;
}
let declarator = child_by_field(node, "declarator")?;
let (macro_node, argument) = match declarator.kind() {
"parenthesized_declarator" if declarator.named_child_count() == 1 => {
let macro_node = child_by_field(node, "type")?;
let argument = declarator.named_child(0)?;
if macro_node.kind() != "type_identifier" || argument.kind() != "identifier" {
return None;
}
(macro_node, argument)
}
"function_declarator" if child_by_field(node, "type").is_none() => {
let macro_node = child_by_field(declarator, "declarator")?;
let params = child_by_field(declarator, "parameters")?;
let param = params.named_child(0)?;
if macro_node.kind() != "identifier"
|| params.named_child_count() != 1
|| param.kind() != "parameter_declaration"
|| param.named_child_count() != 1
{
return None;
}
let argument = param.named_child(0)?;
if argument.kind() != "type_identifier" {
return None;
}
(macro_node, argument)
}
_ => return None,
};
let macro_name = node_text(macro_node, source);
static REPLACEMENT: OnceLock<Regex> = OnceLock::new();
let replacement_pattern = REPLACEMENT.get_or_init(|| {
Regex::new(
r"^(?:[A-Za-z_][A-Za-z0-9_:]*\s+)+[*&\s]*([A-Za-z_][A-Za-z0-9_]*)\s*\([^(){};#]*\)\s*$",
)
.expect("single-argument macro replacement pattern")
});
static TYPEDEF: OnceLock<Regex> = OnceLock::new();
let typedef = TYPEDEF.get_or_init(|| Regex::new(r"(?-u:\b)typedef(?-u:\b)").expect("typedef"));
let mut scope = Some(node);
while let Some(current) = scope {
if current.kind() == "preproc_else" || current.kind().starts_with("preproc_elif") {
return None;
}
let mut previous = current.prev_named_sibling();
while let Some(prev) = previous {
previous = prev.prev_named_sibling();
if prev.kind().starts_with("preproc_if") {
return None;
}
if prev.kind() == "preproc_call"
&& child_by_field(prev, "directive")
.is_some_and(|directive| node_text(directive, source) == "#undef")
&& child_by_field(prev, "argument")
.is_some_and(|argument| node_text(argument, source).trim() == macro_name)
{
return None;
}
if !matches!(prev.kind(), "preproc_function_def" | "preproc_def")
|| child_by_field(prev, "name").map(|name| node_text(name, source))
!= Some(macro_name.clone())
{
continue;
}
let params = child_by_field(prev, "parameters")?;
let param = params.named_child(0)?;
let value = child_by_field(prev, "value")?;
if params.named_child_count() != 1 || param.kind() != "identifier" {
return None;
}
let replacement = node_text(value, source)
.replace("\\\r\n", " ")
.replace("\\\n", " ");
let replacement = replacement.trim();
let declared = replacement_pattern
.captures(replacement)
.and_then(|captures| captures.get(1))?;
if typedef.is_match(replacement) || declared.as_str() != node_text(param, source) {
return None;
}
return Some(node_text(argument, source));
}
scope = current.parent();
}
None
}

/// Recover the real function name from the macro-definition idiom
/// `MACRO_NAME(real_name, typed args…) { body }` (flash-attention's
/// `DEFINE_FLASH_FORWARD_KERNEL(flash_fwd_kernel, bool Is_dropout, …) { … }`):
Expand Down
10 changes: 10 additions & 0 deletions crates/codegraph-extract/src/lang/javascript.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,16 @@ impl LanguageSpec for JavaScriptSpec {
crate::lang::typescript::class_field_is_callable(node, "field_definition")
}

fn resolve_name(&self, node: Node<'_>, source: &str) -> Option<String> {
// JS `field_definition` names its key the `property` field (TS's
// `public_field_definition` uses `name`); without this every JS class
// field, handler fields included, is `<anonymous>` (upstream #808).
if node.kind() == "field_definition" {
return child_by_field(node, "property").map(|property| node_text(property, source));
}
None
}

fn interface_types(&self) -> &'static [&'static str] {
&[]
}
Expand Down
6 changes: 6 additions & 0 deletions crates/codegraph-extract/src/lang/jsx.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ impl LanguageSpec for JsxSpec {
fn method_types(&self) -> &'static [&'static str] {
JAVASCRIPT_SPEC.method_types()
}
fn class_member_is_method(&self, node: Node<'_>, source: &str) -> bool {
JAVASCRIPT_SPEC.class_member_is_method(node, source)
}
fn interface_types(&self) -> &'static [&'static str] {
&[]
}
Expand Down Expand Up @@ -60,6 +63,9 @@ impl LanguageSpec for JsxSpec {
fn name_field(&self) -> &'static str {
JAVASCRIPT_SPEC.name_field()
}
fn resolve_name(&self, node: Node<'_>, source: &str) -> Option<String> {
JAVASCRIPT_SPEC.resolve_name(node, source)
}
fn body_field(&self) -> &'static str {
JAVASCRIPT_SPEC.body_field()
}
Expand Down
136 changes: 136 additions & 0 deletions crates/codegraph-extract/src/lang/python.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ impl LanguageSpec for PythonSpec {
Some(signature)
}

fn body_docstring(&self, node: Node<'_>, source: &str) -> Option<String> {
python_body_docstring(node, source)
}

fn is_async(&self, node: Node<'_>) -> bool {
node.prev_sibling()
.is_some_and(|prev| prev.kind() == "async")
Expand All @@ -96,3 +100,135 @@ impl LanguageSpec for PythonSpec {
})
}
}

/// Python states intent in a docstring — a bare string literal as the first
/// statement of a module, class or function body (upstream #1905). Reads the
/// grammar's `string_content`, so `r`/`u` prefixes and both triple-quote forms
/// work; bytes and f-strings are not docstrings, and a tuple is not a string.
fn python_body_docstring(node: Node<'_>, source: &str) -> Option<String> {
let body = if node.kind() == "module" {
node
} else {
child_by_field(node, "body")?
};
let first = body
.named_children(&mut body.walk())
.find(|child| child.kind() != "comment")?;
if first.kind() != "expression_statement"
|| first.named_child_count() != 1
|| first
.children(&mut first.walk())
.any(|child| child.kind() == ",")
{
return None;
}
let mut literal = first.named_child(0)?;
while literal.kind() == "parenthesized_expression" {
literal = literal
.named_children(&mut literal.walk())
.find(|child| child.kind() != "comment")?;
}
let strings = if literal.kind() == "concatenated_string" {
literal
.named_children(&mut literal.walk())
.filter(|child| child.kind() != "comment")
.collect::<Vec<_>>()
} else {
vec![literal]
};
let mut raw = String::new();
for string in strings {
if string.kind() != "string" {
return None;
}
let parts = string
.named_children(&mut string.walk())
.collect::<Vec<_>>();
let start = parts.iter().find(|part| part.kind() == "string_start")?;
if node_text(*start, source)
.chars()
.any(|ch| matches!(ch, 'b' | 'B' | 'f' | 'F'))
|| !parts.iter().any(|part| part.kind() == "string_end")
{
return None;
}
if let Some(content) = parts.iter().find(|part| part.kind() == "string_content") {
raw.push_str(&node_text(*content, source));
}
}
let docstring = dedent_docstring(&raw);
(!docstring.is_empty()).then_some(docstring)
}

/// PEP 257 cleaning: tabs expand to 8-column stops, every line after the first
/// loses the common indentation of the non-blank ones (the first line starts
/// right after the quotes), and blank edges are dropped.
fn dedent_docstring(raw: &str) -> String {
let lines = raw
.replace("\r\n", "\n")
.replace('\r', "\n")
.split('\n')
.map(|line| {
let mut column = 0usize;
let mut expanded = String::with_capacity(line.len());
for ch in line.chars() {
if ch == '\t' {
let width = 8 - column % 8;
expanded.extend(std::iter::repeat_n(' ', width));
column += width;
} else {
expanded.push(ch);
column += 1;
}
}
expanded
})
.collect::<Vec<_>>();
let leading = |line: &str| line.chars().take_while(|ch| ch.is_whitespace()).count();
let indent = lines
.iter()
.skip(1)
.filter(|line| !line.trim().is_empty())
.map(|line| leading(line))
.min()
.unwrap_or(0);
let mut out = Vec::with_capacity(lines.len());
out.push(
lines
.first()
.map(|line| line.trim().to_string())
.unwrap_or_default(),
);
for line in lines.iter().skip(1) {
out.push(
line.chars()
.skip(indent)
.collect::<String>()
.trim_end()
.to_string(),
);
}
while out.first().is_some_and(|line| line.trim().is_empty()) {
out.remove(0);
}
while out.last().is_some_and(|line| line.trim().is_empty()) {
out.pop();
}
out.join("\n")
}

#[cfg(test)]
mod docstring_tests {
use super::dedent_docstring;

#[test]
fn dedent_follows_pep_257() {
assert_eq!(
dedent_docstring("Summary line.\n\n Details here.\n indented\n "),
"Summary line.\n\nDetails here.\n indented"
);
assert_eq!(dedent_docstring("\n Only body.\n "), "Only body.");
assert_eq!(dedent_docstring("a\r\n\tb"), "a\nb");
assert_eq!(dedent_docstring(" "), "");
}
}
6 changes: 6 additions & 0 deletions crates/codegraph-extract/src/lang/tsx.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ impl LanguageSpec for TsxSpec {
fn method_types(&self) -> &'static [&'static str] {
TYPESCRIPT_SPEC.method_types()
}
fn property_types(&self) -> &'static [&'static str] {
TYPESCRIPT_SPEC.property_types()
}
fn class_member_is_method(&self, node: Node<'_>, source: &str) -> bool {
TYPESCRIPT_SPEC.class_member_is_method(node, source)
}
fn interface_types(&self) -> &'static [&'static str] {
TYPESCRIPT_SPEC.interface_types()
}
Expand Down
7 changes: 7 additions & 0 deletions crates/codegraph-extract/src/spec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,13 @@ pub trait LanguageSpec: Sync {
None
}

/// Prose that lives INSIDE the node's body rather than in a preceding
/// comment — Python's bare first-statement string (upstream
/// `getBodyDocstring`, #1905). Cleaned text, or `None`.
fn body_docstring(&self, _node: Node<'_>, _source: &str) -> Option<String> {
None
}

fn get_return_type(&self, _node: Node<'_>, _source: &str) -> Option<String> {
None
}
Expand Down
Loading
Loading