Add endpoint to fetch fluent reference variants

pontoon/base/fluent_utils.py

@@ -0,0 +1,112 @@
+from collections.abc import Iterator
+from typing import TypedDict
+
+from moz.l10n.formats.fluent import fluent_parse_entry
+from moz.l10n.model import (
+    CatchallKey,
+    Entry,
+    Expression,
+    Message,
+    Pattern,
+    PatternMessage,
+    SelectMessage,
+)
+
+
+def _parse_fluent_entry(source: str) -> Entry[Message] | None:
+    """Parse a Fluent entry; returns None if the source is invalid FTL."""
+    try:
+        return fluent_parse_entry(source, with_linepos=False)
+    except ValueError:
+        return None
+
+
+def _entry_messages(entry: Entry[Message]) -> Iterator[Message]:
+    """Iterate over the entry's value message and its property messages."""
+    if isinstance(entry.value, (PatternMessage, SelectMessage)):
+        yield entry.value
+    yield from entry.properties.values()
+
+
+def get_references(entry_or_str: Entry[Message] | str) -> set[str]:
+    """Collect the keys of the messages/terms referenced by a Fluent entry,
+    e.g. `-brand-name` in `msg = This is { -brand-name }`.
+
+    Accepts the entry itself or a Fluent source string;
+    a string that cannot be parsed yields an empty set.
+    """
+    entry = (
+        _parse_fluent_entry(entry_or_str)
+        if isinstance(entry_or_str, str)
+        else entry_or_str
+    )
+    if entry is None:
+        return set()
+
+    names: set[str] = set()
+
+    def from_expression(expr: Expression) -> None:
+        # Remember message/term references
+        if expr.function == "message" and isinstance(expr.arg, str):
+            names.add(expr.arg)
+
+    def from_pattern(pattern: Pattern) -> None:
+        for part in pattern:
+            if isinstance(part, Expression):
+                from_expression(part)
+
+    def from_message(msg: Message) -> None:
+        for expr in msg.declarations.values():
+            from_expression(expr)
+        if isinstance(msg, PatternMessage):
+            from_pattern(msg.pattern)
+        elif isinstance(msg, SelectMessage):
+            for pattern in msg.variants.values():
+                from_pattern(pattern)
+
+    for msg in _entry_messages(entry):
+        from_message(msg)
+
+    return names
+
+
+class SelectorField(TypedDict):
+    """A selector from a Fluent entry, together with its variant keys."""
+
+    name: str
+    values: list[str]
+
+
+def get_selector_variants(entry_or_str: Entry[Message] | str) -> list[SelectorField]:
+    """Extract the selector fields and their variants from a Fluent entry's
+    value and attributes.
+
+    Accepts the entry itself or a Fluent source string;
+    a string that cannot be parsed yields an empty list.
+
+    Note default variants always come last, which is the same way as provided by
+    moz.l10n.fluent.
+    """
+    entry = (
+        _parse_fluent_entry(entry_or_str)
+        if isinstance(entry_or_str, str)
+        else entry_or_str
+    )
+    if entry is None:
+        return []
+
+    fields: dict[str, list[str]] = {}
+    for msg in _entry_messages(entry):
+        # Only selectors contain variants, so we skip any other message type.
+        if not isinstance(msg, SelectMessage):
+            continue
+
+        for idx, selector in enumerate(msg.selectors):
+            values = fields.setdefault(selector.name, [])
+            for keys in msg.variants:
+                key = keys[idx]
+                value = key.value if isinstance(key, CatchallKey) else key
+                if value and value not in values:
+                    values.append(value)
+
+    return [{"name": name, "values": values} for name, values in fields.items()]

pontoon/base/tests/test_fluent_utils.py

@@ -0,0 +1,154 @@
+from textwrap import dedent
+
+from moz.l10n.formats.fluent import fluent_parse_entry
+
+from pontoon.base.fluent_utils import (
+    get_references,
+    get_selector_variants,
+)
+
+
+def test_get_references_from_source_string():
+    """Test that a Fluent source string is parsed and its references collected."""
+    source = "message = Welcome to { -brand-name }, { $user }!"
+
+    assert get_references(source) == {"-brand-name"}
+
+
+def test_get_references_invalid_source_string():
+    """Test unparseable Fluent source strings produce an empty set."""
+    source = "no valid fluent here ]["
+
+    assert get_references(source) == set()
+
+
+def test_get_references_from_entry():
+    """Test that references are collected from the parsed entry itself."""
+    source = "message = Welcome to { -brand-name }, { $user }!"
+    entry = fluent_parse_entry(source, with_linepos=False)
+
+    assert get_references(entry) == {"-brand-name"}
+
+
+def test_get_references_no_placeholders():
+    """Test a string with no terms collects no references."""
+    source = "message = Simple string"
+
+    assert get_references(source) == set()
+
+
+def test_get_references_from_attributes():
+    """Test terms can be extracted from attributes."""
+    source = dedent(
+        """\
+        message =
+            .gender = { -brand-name(case: "genitive") }
+            .tooltip = { $count }
+        """
+    )
+
+    assert get_references(source) == {"-brand-name"}
+
+
+def test_get_references_from_declarations_and_select_variants():
+    """Test references from select variants are picked up."""
+    source = dedent(
+        """\
+        message =
+            { $count ->
+                [one] { -brand-short-name } blocked one tracker
+                *[other] { -brand-short-name } blocked many trackers
+            }
+        """
+    )
+
+    assert get_references(source) == {"-brand-short-name"}
+
+
+def test_get_selector_variants_no_selectors():
+    """Test extracting variants from a term with no selector."""
+    source = "message = Welcome to { -brand-name }!"
+
+    assert get_selector_variants(source) == []
+
+
+def test_get_selector_variants_from_value():
+    """Test extracting variants from a selector."""
+    source = dedent(
+        """\
+        message =
+            { $count ->
+                [one] One tracker blocked
+               *[other] Trackers blocked
+            }
+        """
+    )
+
+    assert get_selector_variants(source) == [
+        {"name": "count", "values": ["one", "other"]}
+    ]
+
+
+def test_get_selector_variants_multiple_selectors():
+    """Test extracting variants from nested selectors."""
+    source = dedent(
+        """\
+        message =
+            { $gender ->
+                [masculine] { $count ->
+                    [one] One
+                    *[other] Many
+                }
+               *[feminine] Feminine
+            }
+        """
+    )
+
+    assert get_selector_variants(source) == [
+        {"name": "gender", "values": ["masculine", "feminine"]},
+        {"name": "count", "values": ["one", "other"]},
+    ]
+
+
+def test_get_selector_variants_from_attributes():
+    """Test extracting variants from attribute selectors."""
+    source = dedent(
+        """\
+        message =
+            .accesskey = value with no selector
+            .tooltip =
+                { $case ->
+                    [nominative] Nominative
+                    [genitive] Genitive
+                   *[other] Other
+                }
+        """
+    )
+
+    assert get_selector_variants(source) == [
+        {"name": "case", "values": ["nominative", "genitive", "other"]}
+    ]
+
+
+def test_get_selector_variants_sorting():
+    """Test extracting variants from a selector keeps the default at the end."""
+    source = dedent(
+        """\
+        message =
+            { $count ->
+                *[other] Trackers blocked
+                [one] One tracker blocked
+            }
+        """
+    )
+
+    assert get_selector_variants(source) == [
+        {"name": "count", "values": ["one", "other"]}
+    ]
+
+
+def test_get_selector_variants_invalid_source_string():
+    """Test unparseable Fluent source strings produce an empty list."""
+    source = "no valid fluent here ]["
+
+    assert get_selector_variants(source) == []