diff --git a/pontoon/base/fluent_utils.py b/pontoon/base/fluent_utils.py new file mode 100644 index 0000000000..df26e5cbcc --- /dev/null +++ b/pontoon/base/fluent_utils.py @@ -0,0 +1,112 @@ +from collections.abc import Iterator +from typing import TypedDict + +from moz.l10n.formats.fluent import fluent_parse_entry +from moz.l10n.model import ( + CatchallKey, + Entry, + Expression, + Message, + Pattern, + PatternMessage, + SelectMessage, +) + + +def _parse_fluent_entry(source: str) -> Entry[Message] | None: + """Parse a Fluent entry; returns None if the source is invalid FTL.""" + try: + return fluent_parse_entry(source, with_linepos=False) + except ValueError: + return None + + +def _entry_messages(entry: Entry[Message]) -> Iterator[Message]: + """Iterate over the entry's value message and its property messages.""" + if isinstance(entry.value, (PatternMessage, SelectMessage)): + yield entry.value + yield from entry.properties.values() + + +def get_references(entry_or_str: Entry[Message] | str) -> set[str]: + """Collect the keys of the messages/terms referenced by a Fluent entry, + e.g. `-brand-name` in `msg = This is { -brand-name }`. + + Accepts the entry itself or a Fluent source string; + a string that cannot be parsed yields an empty set. + """ + entry = ( + _parse_fluent_entry(entry_or_str) + if isinstance(entry_or_str, str) + else entry_or_str + ) + if entry is None: + return set() + + names: set[str] = set() + + def from_expression(expr: Expression) -> None: + # Remember message/term references + if expr.function == "message" and isinstance(expr.arg, str): + names.add(expr.arg) + + def from_pattern(pattern: Pattern) -> None: + for part in pattern: + if isinstance(part, Expression): + from_expression(part) + + def from_message(msg: Message) -> None: + for expr in msg.declarations.values(): + from_expression(expr) + if isinstance(msg, PatternMessage): + from_pattern(msg.pattern) + elif isinstance(msg, SelectMessage): + for pattern in msg.variants.values(): + from_pattern(pattern) + + for msg in _entry_messages(entry): + from_message(msg) + + return names + + +class SelectorField(TypedDict): + """A selector from a Fluent entry, together with its variant keys.""" + + name: str + values: list[str] + + +def get_selector_variants(entry_or_str: Entry[Message] | str) -> list[SelectorField]: + """Extract the selector fields and their variants from a Fluent entry's + value and attributes. + + Accepts the entry itself or a Fluent source string; + a string that cannot be parsed yields an empty list. + + Note default variants always come last, which is the same way as provided by + moz.l10n.fluent. + """ + entry = ( + _parse_fluent_entry(entry_or_str) + if isinstance(entry_or_str, str) + else entry_or_str + ) + if entry is None: + return [] + + fields: dict[str, list[str]] = {} + for msg in _entry_messages(entry): + # Only selectors contain variants, so we skip any other message type. + if not isinstance(msg, SelectMessage): + continue + + for idx, selector in enumerate(msg.selectors): + values = fields.setdefault(selector.name, []) + for keys in msg.variants: + key = keys[idx] + value = key.value if isinstance(key, CatchallKey) else key + if value and value not in values: + values.append(value) + + return [{"name": name, "values": values} for name, values in fields.items()] diff --git a/pontoon/base/tests/test_fluent_utils.py b/pontoon/base/tests/test_fluent_utils.py new file mode 100644 index 0000000000..d5a1bdd523 --- /dev/null +++ b/pontoon/base/tests/test_fluent_utils.py @@ -0,0 +1,154 @@ +from textwrap import dedent + +from moz.l10n.formats.fluent import fluent_parse_entry + +from pontoon.base.fluent_utils import ( + get_references, + get_selector_variants, +) + + +def test_get_references_from_source_string(): + """Test that a Fluent source string is parsed and its references collected.""" + source = "message = Welcome to { -brand-name }, { $user }!" + + assert get_references(source) == {"-brand-name"} + + +def test_get_references_invalid_source_string(): + """Test unparseable Fluent source strings produce an empty set.""" + source = "no valid fluent here ][" + + assert get_references(source) == set() + + +def test_get_references_from_entry(): + """Test that references are collected from the parsed entry itself.""" + source = "message = Welcome to { -brand-name }, { $user }!" + entry = fluent_parse_entry(source, with_linepos=False) + + assert get_references(entry) == {"-brand-name"} + + +def test_get_references_no_placeholders(): + """Test a string with no terms collects no references.""" + source = "message = Simple string" + + assert get_references(source) == set() + + +def test_get_references_from_attributes(): + """Test terms can be extracted from attributes.""" + source = dedent( + """\ + message = + .gender = { -brand-name(case: "genitive") } + .tooltip = { $count } + """ + ) + + assert get_references(source) == {"-brand-name"} + + +def test_get_references_from_declarations_and_select_variants(): + """Test references from select variants are picked up.""" + source = dedent( + """\ + message = + { $count -> + [one] { -brand-short-name } blocked one tracker + *[other] { -brand-short-name } blocked many trackers + } + """ + ) + + assert get_references(source) == {"-brand-short-name"} + + +def test_get_selector_variants_no_selectors(): + """Test extracting variants from a term with no selector.""" + source = "message = Welcome to { -brand-name }!" + + assert get_selector_variants(source) == [] + + +def test_get_selector_variants_from_value(): + """Test extracting variants from a selector.""" + source = dedent( + """\ + message = + { $count -> + [one] One tracker blocked + *[other] Trackers blocked + } + """ + ) + + assert get_selector_variants(source) == [ + {"name": "count", "values": ["one", "other"]} + ] + + +def test_get_selector_variants_multiple_selectors(): + """Test extracting variants from nested selectors.""" + source = dedent( + """\ + message = + { $gender -> + [masculine] { $count -> + [one] One + *[other] Many + } + *[feminine] Feminine + } + """ + ) + + assert get_selector_variants(source) == [ + {"name": "gender", "values": ["masculine", "feminine"]}, + {"name": "count", "values": ["one", "other"]}, + ] + + +def test_get_selector_variants_from_attributes(): + """Test extracting variants from attribute selectors.""" + source = dedent( + """\ + message = + .accesskey = value with no selector + .tooltip = + { $case -> + [nominative] Nominative + [genitive] Genitive + *[other] Other + } + """ + ) + + assert get_selector_variants(source) == [ + {"name": "case", "values": ["nominative", "genitive", "other"]} + ] + + +def test_get_selector_variants_sorting(): + """Test extracting variants from a selector keeps the default at the end.""" + source = dedent( + """\ + message = + { $count -> + *[other] Trackers blocked + [one] One tracker blocked + } + """ + ) + + assert get_selector_variants(source) == [ + {"name": "count", "values": ["one", "other"]} + ] + + +def test_get_selector_variants_invalid_source_string(): + """Test unparseable Fluent source strings produce an empty list.""" + source = "no valid fluent here ][" + + assert get_selector_variants(source) == [] diff --git a/pontoon/base/tests/views/test_ajax.py b/pontoon/base/tests/views/test_ajax.py index 2e6c1f20fd..dd42f19b0d 100644 --- a/pontoon/base/tests/views/test_ajax.py +++ b/pontoon/base/tests/views/test_ajax.py @@ -6,9 +6,11 @@ from django.http import Http404 +from pontoon.base.models import Resource from pontoon.base.views import ( AjaxFormPostView, AjaxFormView, + get_fluent_reference_variants, get_sibling_entities, get_team_comments, get_translation_history, @@ -21,6 +23,7 @@ ProjectFactory, ResourceFactory, TranslatedResourceFactory, + TranslationFactory, UserFactory, ) @@ -250,6 +253,545 @@ def test_get_sibling_entities_authentication(rf, user_a, admin): assert response.status_code == 200 +@pytest.mark.django_db +def test_get_fluent_reference_variants_private_project_access( + rf, locale_a, user_a, admin +): + """Test retrieval of entities from private projects.""" + project_a = ProjectFactory(name="Project A", visibility="private") + resource_a = ResourceFactory(project=project_a) + entity_a = EntityFactory(string="Entity A", resource=resource_a) + + request_a = rf.get( + f"/get-fluent-reference-variants/?entity={entity_a.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request_a.user = user_a + + with pytest.raises(Http404): + get_fluent_reference_variants(request_a) + + request_b = rf.get( + f"/get-fluent-reference-variants/?entity={entity_a.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request_b.user = admin + + response = get_fluent_reference_variants(request_b) + + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_invalid_locale(rf, user_a, project_a): + """Test a 404 response for a non-existent locale.""" + resource_a = ResourceFactory(project=project_a) + entity_a = EntityFactory(string="Entity A", resource=resource_a) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_a.id}&locale=invalid-locale", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + with pytest.raises(Http404): + get_fluent_reference_variants(request) + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_nonexistent_entity( + rf, user_a, locale_a, project_a +): + """Test a 404 response for a non-existent entity.""" + request = rf.get( + "/get-fluent-reference-variants/?entity=1234&locale=ab", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + with pytest.raises(Http404): + get_fluent_reference_variants(request) + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "query", + [ + # Missing entity parameter + "locale=ab", + # Missing locale parameter + "entity=1", + # Non-integer entity parameter + "entity=abc&locale=ab", + ], +) +def test_get_fluent_reference_variants_bad_request(rf, user_a, query): + """Test a 400 response for missing or invalid parameters.""" + request = rf.get( + f"/get-fluent-reference-variants/?{query}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 400 + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_non_fluent_resource( + rf, user_a, locale_a, project_a +): + """Test no variants are returned for a non-Fluent resource.""" + resource_a = ResourceFactory( + project=project_a, + path="a.dtd", + format=Resource.Format.DTD, + ) + entity_term = EntityFactory( + resource=resource_a, + string="-brand-term = about Brand", + ) + TranslationFactory( + entity=entity_term, + locale=locale_a, + active=True, + approved=True, + string=( + "-brand-term = { $case ->\n" + " *[nominative] Brand-nom\n" + " [accusative] Brand-acc\n" + "}" + ), + ) + + entity_current = EntityFactory( + resource=resource_a, + string="message = { -brand-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_happy_path(rf, user_a, locale_a, project_a): + """Test fetching term variants for a locale.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + entity_term = EntityFactory( + resource=resource_a, + string="-brand-term = about Brand", + ) + TranslationFactory( + entity=entity_term, + locale=locale_a, + active=True, + approved=True, + string=( + "-brand-term = { $case ->\n" + " *[nominative] Brand-nom\n" + " [accusative] Brand-acc\n" + "}" + ), + ) + + entity_current = EntityFactory( + resource=resource_a, + string="message = This uses { -brand-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == { + "-brand-term": [{"name": "case", "values": ["accusative", "nominative"]}], + } + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_no_selector(rf, user_a, locale_a, project_a): + """Test retrieving variants from a translation that has no selector.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + entity_term = EntityFactory( + resource=resource_a, + string=( + "-brand-term = { $case ->\n" + " *[nominative] Brand-nom\n" + " [genitive] Brand-gen\n" + "}" + ), + ) + TranslationFactory( + entity=entity_term, + locale=locale_a, + active=True, + approved=True, + string=("-brand-term = no selectors in translation"), + ) + + entity_current = EntityFactory( + resource=resource_a, + string="message = { -brand-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_dangling_term_reference( + rf, user_a, locale_a, project_a +): + """Test no variant fields are returned for a term without a matching entity.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + entity_current = EntityFactory( + resource=resource_a, + string="message = { -brand-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_obsolete_term(rf, user_a, locale_a, project_a): + """Test no variant fields are returned for an obsolete term entity.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + entity_term = EntityFactory( + resource=resource_a, + string="-brand-term = Brand", + obsolete=True, + ) + TranslationFactory( + entity=entity_term, + locale=locale_a, + active=True, + approved=True, + string=( + "-brand-term = { $case ->\n" + " *[nominative] Brand-nom\n" + " [accusative] Brand-acc\n" + "}" + ), + ) + + entity_current = EntityFactory( + resource=resource_a, + string="message = { -brand-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_no_translation(rf, user_a, locale_a, project_a): + """Test no variant fields are returned without a translation.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + _entity_term = EntityFactory( + resource=resource_a, + string=( + "-brand-term = { $case ->\n" + " *[nominative] Brand-nom\n" + " [genitive] Brand-gen\n" + "}" + ), + ) + + entity_current = EntityFactory( + resource=resource_a, + string="message = { -brand-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_inactive_translation( + rf, user_a, locale_a, project_a +): + """Test variants come only from the active translation.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + entity_term = EntityFactory( + resource=resource_a, + string="-brand-term = Brand", + ) + TranslationFactory( + entity=entity_term, + locale=locale_a, + active=True, + approved=True, + string="-brand-term = Brand", + ) + TranslationFactory( + entity=entity_term, + locale=locale_a, + active=False, + approved=False, + string=( + "-brand-term = { $case ->\n" + " *[nominative] Brand-nom\n" + " [accusative] Brand-acc\n" + "}" + ), + ) + + entity_current = EntityFactory( + resource=resource_a, + string="message = { -brand-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == {} + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_multiple_terms(rf, user_a, locale_a, project_a): + """Test variant retrieval for an entity referencing multiple terms.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + _entity_term_a = EntityFactory( + resource=resource_a, + string=("-brand-term = Brand"), + ) + entity_term_b = EntityFactory( + resource=resource_a, + string=("-brand-short-term = Brand-Short"), + ) + TranslationFactory( + entity=entity_term_b, + locale=locale_a, + active=True, + approved=True, + string=( + "-brand-short-term = { $case ->\n" + " *[nominative] Brand-Short-nom\n" + " [accusative] Brand-Short-acc\n" + "}" + ), + ) + + entity_current = EntityFactory( + resource=resource_a, + string="message = { -brand-term } or { -brand-short-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == { + "-brand-short-term": [{"name": "case", "values": ["accusative", "nominative"]}] + } + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_multiple_resources( + rf, user_a, locale_a, project_a +): + """Test prioritizing the current entity's resource over another resource in the project.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + entity_term_a = EntityFactory( + resource=resource_a, + string="-brand-term = Brand From A", + ) + TranslationFactory( + entity=entity_term_a, + locale=locale_a, + active=True, + approved=True, + string=( + "-brand-term = { $case-a ->\n" + " *[nominative-a] Brand-A-nom\n" + " [accusative-a] Brand-A-acc\n" + "}" + ), + ) + + resource_b = ResourceFactory( + project=project_a, + path="b.ftl", + format=Resource.Format.FLUENT, + ) + entity_term_b = EntityFactory( + resource=resource_b, + string="-brand-term = Brand From B", + ) + TranslationFactory( + entity=entity_term_b, + locale=locale_a, + active=True, + approved=True, + string=( + "-brand-term = { $case-b ->\n" + " *[nominative-b] Brand-B-nom\n" + " [accusative-b] Brand-B-acc\n" + "}" + ), + ) + + entity_current = EntityFactory( + resource=resource_a, + string="message= { -brand-term }", + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == { + "-brand-term": [{"name": "case-a", "values": ["accusative-a", "nominative-a"]}], + } + + +@pytest.mark.django_db +def test_get_fluent_reference_variants_term_in_other_resource_fallback( + rf, user_a, locale_a, project_a +): + """Test falling back to the term defined in another resource of the project.""" + resource_a = ResourceFactory( + project=project_a, + path="a.ftl", + format=Resource.Format.FLUENT, + ) + entity_current = EntityFactory( + resource=resource_a, + string="message = { -brand-term }", + ) + + resource_b = ResourceFactory( + project=project_a, + path="b.ftl", + format=Resource.Format.FLUENT, + ) + entity_term_b = EntityFactory( + resource=resource_b, + string="-brand-term = Brand", + ) + TranslationFactory( + entity=entity_term_b, + locale=locale_a, + active=True, + approved=True, + string=( + "-brand-term = { $case ->\n" + " *[nominative] Brand-nom\n" + " [accusative] Brand-acc\n" + "}" + ), + ) + + request = rf.get( + f"/get-fluent-reference-variants/?entity={entity_current.id}&locale={locale_a.code}", + HTTP_X_REQUESTED_WITH="XMLHttpRequest", + ) + request.user = user_a + + response = get_fluent_reference_variants(request) + + assert response.status_code == 200 + assert json.loads(response.content) == { + "-brand-term": [{"name": "case", "values": ["accusative", "nominative"]}], + } + + @pytest.mark.django_db def test_get_translation_history(rf, user_a, admin): project_a = ProjectFactory(name="Project A", visibility="private") diff --git a/pontoon/base/urls.py b/pontoon/base/urls.py index 178fed25a3..2b691de2c6 100644 --- a/pontoon/base/urls.py +++ b/pontoon/base/urls.py @@ -106,4 +106,9 @@ path("upload/", views.upload, name="pontoon.upload"), path("user-data/", views.user_data, name="pontoon.user_data"), path("get-sibling-entities/", views.get_sibling_entities), + path( + "get-fluent-reference-variants/", + views.get_fluent_reference_variants, + name="pontoon.fluent.reference_variants", + ), ] diff --git a/pontoon/base/views.py b/pontoon/base/views.py index bf42d77248..409b2f4ef4 100755 --- a/pontoon/base/views.py +++ b/pontoon/base/views.py @@ -35,6 +35,11 @@ from pontoon.actionlog.utils import log_action from pontoon.base import forms, utils from pontoon.base.badge_utils import badges_review_level, badges_translation_level +from pontoon.base.fluent_utils import ( + SelectorField, + get_references, + get_selector_variants, +) from pontoon.base.get_entities import ( get_entities_for_project_locale, get_mismatched_filters, @@ -521,6 +526,58 @@ def get_sibling_entities(request): ) +@utils.require_AJAX +def get_fluent_reference_variants(request: HttpRequest) -> JsonResponse: + """Get the variant fields and their possible values for each message or term + referenced by a Fluent entity. + """ + try: + entity_pk = int(request.GET["entity"]) + locale_code = request.GET["locale"] + except (MultiValueDictKeyError, ValueError) as e: + return JsonResponse( + {"status": False, "message": f"Bad Request: {e}"}, + status=400, + ) + + visible_projects = Project.objects.available().visible_for(request.user) + entities = Entity.objects.filter(resource__project__in=visible_projects) + + entity = get_object_or_404(entities, pk=entity_pk) + locale = get_object_or_404(Locale, code=locale_code) + + payload: dict[str, list[SelectorField]] = {} + if entity.resource.format != Resource.Format.FLUENT: + return JsonResponse(payload) + + for key in get_references(entity.string): + ref_entities = Entity.objects.filter( + resource__project=entity.resource.project, + obsolete=False, + key=[key], + ) + # Prioritize using the reference's entity defined in the same resource, + # and fall back to the first project occurrence. + ref_entity = ( + ref_entities.filter(resource=entity.resource).first() + or ref_entities.first() + ) + if ref_entity is None: + continue + + translation = ref_entity.translation_set.filter( + locale=locale, active=True + ).first() + if translation is None: + continue + + selector_variants = get_selector_variants(translation.string) + if selector_variants: + payload[key] = selector_variants + + return JsonResponse(payload) + + @utils.require_AJAX def get_translation_history(request): """Get history of translations of given entity to given locale."""