diff --git a/documentation/docs/dev/deployment.md b/documentation/docs/dev/deployment.md index c541aaa006..b55ab36c1b 100644 --- a/documentation/docs/dev/deployment.md +++ b/documentation/docs/dev/deployment.md @@ -17,6 +17,18 @@ Optional. Email address for the `ADMINS` setting. `ADMIN_NAME` Optional. Name for the `ADMINS` setting. +`API_TERMINOLOGY_THROTTLE_BURST` +Optional. Short-window rate limit for the terminology matching API endpoint, +applied per user, or per IP address for anonymous requests. Uses the Django REST +Framework format `/`, where period is one of `second`, `minute`, +`hour` or `day` (default: `60/minute`). + +`API_TERMINOLOGY_THROTTLE_SUSTAINED` +Optional. Long-window rate limit for the terminology matching API endpoint, +applied per user, or per IP address for anonymous requests. Uses the Django REST +Framework format `/`, where period is one of `second`, `minute`, +`hour` or `day` (default: `600/hour`). + `API_UPLOAD_THROTTLE_BURST` Optional. Short-window rate limit for the upload API endpoints, applied per authenticated user. Uses the Django REST Framework format diff --git a/pontoon/api/README.md b/pontoon/api/README.md index 2605655e5e..515c631bf9 100644 --- a/pontoon/api/README.md +++ b/pontoon/api/README.md @@ -73,6 +73,72 @@ An example may look like this: $ curl --globoff "https://example.com/api/v2/locales/?page_size=50" ``` +## Terminology Matching + +### `GET /api/v2/terminology/matches/` + +Find the terms appearing in a text, with their translation in a given locale. + +Unlike [`/api/v2/search/terminology/`](#/search/search_terminology_list), which looks up terms by name, this +endpoint matches every known term against the text, at word boundaries: a term matches +the start of a word, so `open` matches `Opened`, but not `Reopened`. Terms without a +definition, and terms marked as forbidden, are never returned. + +| Parameter | Description | +| --------- | --------------------------- | +| `locale` | Locale code | +| `text` | Text to match terms against | + +```bash +$ curl --globoff \ + --data-urlencode "locale=it" \ + --data-urlencode "text=Open a new tab" \ + --get "https://example.com/api/v2/terminology/matches/" +``` + +```json +{ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "definition": "Allow access", + "part_of_speech": "verb", + "text": "open", + "translation_text": "apri", + "usage": "Open the door.", + "notes": "" + }, + { + "definition": "A page in the browser", + "part_of_speech": "noun", + "text": "tab", + "translation_text": "scheda", + "usage": "Open a new tab.", + "notes": "" + } + ] +} +``` + +`translation_text` is `null` for terms not yet translated in the locale, and the term +itself for terms marked as "do not translate", such as product names. + +No authentication is required. Texts over the maximum length are rejected with `400`: +the limit is configurable via `TERMINOLOGY_API_MAX_CHARS` (default 2048 characters). +An unknown locale returns `404`. + +The endpoint is rate limited per user, or per IP address for anonymous requests, with a +burst limit of 60 calls per minute and a sustained limit of 600 calls per hour by default +(configurable via `API_TERMINOLOGY_THROTTLE_BURST` and +`API_TERMINOLOGY_THROTTLE_SUSTAINED`). Calls over the limit are rejected with `429`. +The two limits are not independent: calls rejected by the burst limit still count against +the sustained limit, so a client that keeps calling after a `429` spends its hourly quota +on rejected calls. For example, 60 accepted calls followed by 540 rejected ones exhaust +the hourly quota, locking the client out for one hour. This quota is separate from the one +used by the write endpoints. + ## Write Endpoints The following endpoints can write data and always require authentication with a Personal diff --git a/pontoon/api/serializers.py b/pontoon/api/serializers.py index e67d0874c2..19b5dd1c26 100644 --- a/pontoon/api/serializers.py +++ b/pontoon/api/serializers.py @@ -257,6 +257,9 @@ class Meta: ] def get_translation_text(self, obj): + if obj.do_not_translate: + return obj.text + if hasattr(obj, "filtered_translations") and (ft := obj.filtered_translations): return ft[0].text diff --git a/pontoon/api/tests/test_views.py b/pontoon/api/tests/test_views.py index efdd969ba1..5312d1f74a 100644 --- a/pontoon/api/tests/test_views.py +++ b/pontoon/api/tests/test_views.py @@ -25,6 +25,7 @@ from pontoon.base.models.translated_resource import TranslatedResource from pontoon.base.models.translation import Translation from pontoon.base.models.translation_memory import TranslationMemoryEntry +from pontoon.settings.base import TERMINOLOGY_API_MAX_CHARS from pontoon.terminology.models import Term, TermTranslation from pontoon.test.factories import ( EntityFactory, @@ -1181,6 +1182,211 @@ def test_terminology_search(django_assert_num_queries): } +@pytest.fixture +def terminology_matches_setup(): + locale = LocaleFactory(code="kg", name="Klingon") + other_locale = LocaleFactory(code="gs", name="Geonosian") + + term_open = Term.objects.create( + text="open", + part_of_speech="verb", + definition="Allow access", + usage="Open the door.", + ) + term_tab = Term.objects.create( + text="tab", + part_of_speech="noun", + definition="A page in the browser", + usage="Open a new tab.", + ) + term_click = Term.objects.create( + text="click", + part_of_speech="verb", + definition="Press", + usage="Click the button.", + ) + Term.objects.create( + text="Firefox", + part_of_speech="noun", + definition="A web browser", + do_not_translate=True, + ) + # Terms without a definition, or forbidden, are never matched + Term.objects.create(text="window", part_of_speech="noun", definition="") + Term.objects.create( + text="bookmark", + part_of_speech="noun", + definition="A saved page", + forbidden=True, + ) + + TermTranslation.objects.create(term=term_open, locale=locale, text="odpri") + TermTranslation.objects.create(term=term_tab, locale=locale, text="zavihek") + TermTranslation.objects.create(term=term_click, locale=other_locale, text="klikni") + + return SimpleNamespace(locale=locale, other_locale=other_locale) + + +@pytest.mark.django_db +def test_terminology_matches(terminology_matches_setup, django_assert_num_queries): + with django_assert_num_queries(3): + response = APIClient().get( + "/api/v2/terminology/matches/", + {"locale": "kg", "text": "Open a new tab in this window."}, + ) + + assert response.status_code == 200 + assert response.data == { + "count": 2, + "next": None, + "previous": None, + "results": [ + { + "definition": "Allow access", + "part_of_speech": "verb", + "text": "open", + "translation_text": "odpri", + "usage": "Open the door.", + "notes": "", + }, + { + "definition": "A page in the browser", + "part_of_speech": "noun", + "text": "tab", + "translation_text": "zavihek", + "usage": "Open a new tab.", + "notes": "", + }, + ], + } + + +@pytest.mark.django_db +def test_terminology_matches_word_start(terminology_matches_setup): + """Terms are matched at the start of a word, to also catch inflected forms.""" + response = APIClient().get( + "/api/v2/terminology/matches/", + {"locale": "kg", "text": "Reopened the crab."}, + ) + + assert response.status_code == 200 + assert response.data["results"] == [] + + response = APIClient().get( + "/api/v2/terminology/matches/", + {"locale": "kg", "text": "Opened the tabs."}, + ) + + assert response.status_code == 200 + assert [t["text"] for t in response.data["results"]] == ["open", "tab"] + + +@pytest.mark.django_db +def test_terminology_matches_missing_translation( + terminology_matches_setup, +): + response = APIClient().get( + "/api/v2/terminology/matches/", + {"locale": "kg", "text": "Click here."}, + ) + + assert response.status_code == 200 + assert [(t["text"], t["translation_text"]) for t in response.data["results"]] == [ + ("click", None) + ] + + +@pytest.mark.django_db +def test_terminology_matches_do_not_translate(terminology_matches_setup): + """Terms that must not be translated are reported as-is, in every locale.""" + response = APIClient().get( + "/api/v2/terminology/matches/", + {"locale": "kg", "text": "Open Firefox."}, + ) + + assert response.status_code == 200 + assert [(t["text"], t["translation_text"]) for t in response.data["results"]] == [ + ("Firefox", "Firefox"), + ("open", "odpri"), + ] + + +@pytest.mark.django_db +def test_terminology_matches_fields(terminology_matches_setup): + response = APIClient().get( + "/api/v2/terminology/matches/", + {"locale": "kg", "text": "Open a new tab.", "fields": "text"}, + ) + + assert response.status_code == 200 + assert response.data["results"] == [{"text": "open"}, {"text": "tab"}] + + +@pytest.mark.django_db +def test_terminology_matches_errors(terminology_matches_setup): + client = APIClient() + + response = client.get("/api/v2/terminology/matches/", {"text": "Open"}) + assert response.status_code == 400 + assert response.data == {"locale": ["This field is required."]} + + response = client.get("/api/v2/terminology/matches/", {"locale": "kg"}) + assert response.status_code == 400 + assert response.data == {"text": ["This field is required."]} + + response = client.get( + "/api/v2/terminology/matches/", {"locale": "missing", "text": "Open"} + ) + assert response.status_code == 404 + + response = client.get( + "/api/v2/terminology/matches/", + {"locale": "kg", "text": "Open a new tab. " * TERMINOLOGY_API_MAX_CHARS}, + ) + assert response.status_code == 400 + assert response.data == { + "text": [ + f"Text exceeds maximum length of {TERMINOLOGY_API_MAX_CHARS} characters." + ] + } + + # Whitespace-only text that is also too long reports the length error + response = client.get( + "/api/v2/terminology/matches/", + {"locale": "kg", "text": " " * (TERMINOLOGY_API_MAX_CHARS + 1)}, + ) + assert response.status_code == 400 + assert response.data == { + "text": [ + f"Text exceeds maximum length of {TERMINOLOGY_API_MAX_CHARS} characters." + ] + } + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "rates", + [ + {"terminology_burst": "2/minute", "terminology_sustained": "1000/hour"}, + {"terminology_burst": "60/minute", "terminology_sustained": "2/hour"}, + ], +) +def test_terminology_matches_throttled(monkeypatch, terminology_matches_setup, rates): + # DRF copies the rates into a class attribute at import time, so overriding the + # REST_FRAMEWORK setting has no effect here. + monkeypatch.setattr(SimpleRateThrottle, "THROTTLE_RATES", rates) + cache.clear() + + client = APIClient() + for expected_status in (200, 200, 429): + response = client.get( + "/api/v2/terminology/matches/", {"locale": "kg", "text": "Open a new tab."} + ) + assert response.status_code == expected_status + + cache.clear() + + @pytest.mark.django_db def test_tm_search(django_assert_num_queries): locale_a = LocaleFactory( diff --git a/pontoon/api/throttling.py b/pontoon/api/throttling.py index b656f871bc..0ad6b6103d 100644 --- a/pontoon/api/throttling.py +++ b/pontoon/api/throttling.py @@ -35,7 +35,7 @@ class SustainedRateThrottle(_SuffixedScopedRateThrottle): suffix = "sustained" -# Throttles for endpoints that write translations from uploaded files. -# Views using these should set `throttle_scope = "upload"` -# so that they share a single quota per user. -UPLOAD_THROTTLE_CLASSES = [BurstRateThrottle, SustainedRateThrottle] +# Throttles for expensive endpoints. Views using these should set a `throttle_scope`, +# and endpoints sharing a scope share a single quota per user (or per IP address, for +# anonymous requests). +SCOPED_THROTTLE_CLASSES = [BurstRateThrottle, SustainedRateThrottle] diff --git a/pontoon/api/urls.py b/pontoon/api/urls.py index 8f0fedc8c9..ba1d1379ca 100644 --- a/pontoon/api/urls.py +++ b/pontoon/api/urls.py @@ -61,6 +61,11 @@ views.UploadPretranslationsView.as_view(), name="upload-pretranslations", ), + path( + "terminology/matches/", + views.TermMatchListView.as_view(), + name="term-matches", + ), path( # Terminology Search "search/terminology/", diff --git a/pontoon/api/views.py b/pontoon/api/views.py index c29501a065..ee1ed5250b 100644 --- a/pontoon/api/views.py +++ b/pontoon/api/views.py @@ -2,7 +2,7 @@ from types import SimpleNamespace from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import OpenApiResponse, extend_schema +from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema from rest_framework import generics, status from rest_framework.exceptions import APIException, PermissionDenied, ValidationError from rest_framework.permissions import IsAuthenticated @@ -23,7 +23,7 @@ PersonalAccessTokenAuthentication, ) from pontoon.api.filters import TermFilter, TranslationMemoryFilter -from pontoon.api.throttling import UPLOAD_THROTTLE_CLASSES +from pontoon.api.throttling import SCOPED_THROTTLE_CLASSES from pontoon.base import forms from pontoon.base.badge_utils import badges_review_level, badges_translation_level from pontoon.base.get_entities import get_entities_for_project_locale @@ -43,11 +43,15 @@ from pontoon.base.user_utils import can_translate from pontoon.messaging.notifications import send_badge_notification from pontoon.pretranslation.pretranslate import get_pretranslation -from pontoon.settings.base import PRETRANSLATION_API_MAX_CHARS +from pontoon.settings.base import ( + PRETRANSLATION_API_MAX_CHARS, + TERMINOLOGY_API_MAX_CHARS, +) from pontoon.terminology.models import ( Term, TermTranslation, ) +from pontoon.terminology.utils import get_terms_for_text from pontoon.translations.utils import parse_source_string_to_json from .serializers import ( @@ -475,6 +479,62 @@ def get_queryset(self): return qs +class TermMatchListView(generics.ListAPIView): + """Terms matching a text.""" + + serializer_class = TermSerializer + queryset = Term.objects.none() + throttle_classes = SCOPED_THROTTLE_CLASSES + throttle_scope = "terminology" + + @extend_schema( + parameters=[ + OpenApiParameter("locale", str, required=True, description="Locale code."), + OpenApiParameter( + "text", + str, + required=True, + description="Text to match terms against " + f"(max {TERMINOLOGY_API_MAX_CHARS} characters).", + ), + ], + responses={ + 200: TermSerializer(many=True), + 400: OpenApiResponse( + description="Missing parameter, or a text that is too long." + ), + 404: OpenApiResponse(description="Unknown locale."), + 429: OpenApiResponse(description="Rate limit exceeded."), + }, + description=( + "Find all known terms appearing in a text, with their " + "translation in the given locale." + ), + ) + def get(self, request, *args, **kwargs): + return super().get(request, *args, **kwargs) + + def get_queryset(self): + locale_code = self.request.query_params.get("locale") + text = self.request.query_params.get("text", "") + + errors = {} + if not locale_code: + errors["locale"] = ["This field is required."] + if not text.strip(): + errors["text"] = ["This field is required."] + if len(text) > TERMINOLOGY_API_MAX_CHARS: + errors["text"] = [ + f"Text exceeds maximum length of {TERMINOLOGY_API_MAX_CHARS} characters." + ] + if errors: + raise ValidationError(errors) + + locale = get_object_or_404(Locale, code=locale_code) + + return get_terms_for_text(locale, text) + + class TranslationMemorySearchListView(generics.ListAPIView): serializer_class = TranslationMemorySerializer filter_backends = [DjangoFilterBackend] @@ -641,7 +701,7 @@ class UploadView(APIView): authentication_classes = [PersonalAccessTokenAuthentication] permission_classes = [IsAuthenticated] - throttle_classes = UPLOAD_THROTTLE_CLASSES + throttle_classes = SCOPED_THROTTLE_CLASSES # Endpoints share a single upload quota per user. throttle_scope = "upload" diff --git a/pontoon/settings/base.py b/pontoon/settings/base.py index 56f38900c6..d7db67e717 100644 --- a/pontoon/settings/base.py +++ b/pontoon/settings/base.py @@ -1341,6 +1341,12 @@ def account_username(user): "DEFAULT_THROTTLE_RATES": { "upload_burst": os.environ.get("API_UPLOAD_THROTTLE_BURST", "30/minute"), "upload_sustained": os.environ.get("API_UPLOAD_THROTTLE_SUSTAINED", "180/hour"), + "terminology_burst": os.environ.get( + "API_TERMINOLOGY_THROTTLE_BURST", "60/minute" + ), + "terminology_sustained": os.environ.get( + "API_TERMINOLOGY_THROTTLE_SUSTAINED", "600/hour" + ), }, } @@ -1355,3 +1361,6 @@ def account_username(user): # Maximum length of input text allowed for pretranslation PRETRANSLATION_API_MAX_CHARS = int(os.environ.get("PRETRANSLATION_API_MAX_CHARS", 2048)) + +# Maximum length of input text allowed for terminology matching +TERMINOLOGY_API_MAX_CHARS = int(os.environ.get("TERMINOLOGY_API_MAX_CHARS", 2048)) diff --git a/pontoon/terminology/tests/test_utils.py b/pontoon/terminology/tests/test_utils.py index 827cfc2fdf..d82076d865 100644 --- a/pontoon/terminology/tests/test_utils.py +++ b/pontoon/terminology/tests/test_utils.py @@ -1,8 +1,12 @@ from textwrap import dedent +import pytest + from moz.l10n.formats.fluent import fluent_parse_entry -from pontoon.terminology.utils import get_all_message_text +from pontoon.terminology.models import Term, TermTranslation +from pontoon.terminology.utils import get_all_message_text, get_terms_for_text +from pontoon.test.factories import LocaleFactory def test_all_message_text(): @@ -39,3 +43,33 @@ def test_all_message_text_excludes_placeholders(): # Placeholders are left out, and the surrounding text is not joined into a # single line, so that terms are not matched across a placeholder. assert get_all_message_text([entry.value]) == "Welcome to \n, \n!" + + +@pytest.mark.django_db +def test_get_terms_for_text(): + locale = LocaleFactory(code="kg", name="Klingon") + term_open = Term.objects.create( + text="open", part_of_speech="verb", definition="Allow access" + ) + Term.objects.create(text="close", part_of_speech="verb", definition="Block access") + + TermTranslation.objects.create(term=term_open, locale=locale, text="odpri") + + terms = get_terms_for_text(locale, "Open a new tab.") + + assert [term.text for term in terms] == ["open"] + assert [t.text for t in terms[0].filtered_translations] == ["odpri"] + + +@pytest.mark.django_db +def test_get_terms_for_text_translation_in_other_locale(): + locale = LocaleFactory(code="kg", name="Klingon") + other_locale = LocaleFactory(code="gs", name="Geonosian") + term = Term.objects.create( + text="open", part_of_speech="verb", definition="Allow access" + ) + TermTranslation.objects.create(term=term, locale=other_locale, text="opena") + + terms = get_terms_for_text(locale, "Open a new tab.") + + assert terms[0].filtered_translations == [] diff --git a/pontoon/terminology/utils.py b/pontoon/terminology/utils.py index 7c6009ea32..74b0f2f143 100644 --- a/pontoon/terminology/utils.py +++ b/pontoon/terminology/utils.py @@ -4,6 +4,10 @@ from moz.l10n.model import Message, Pattern, PatternMessage from django.conf import settings +from django.db.models import Prefetch, prefetch_related_objects + +from pontoon.base.models import Locale +from pontoon.terminology.models import Term, TermTranslation def get_message_patterns(msg: Message) -> Iterator[Pattern]: @@ -30,6 +34,26 @@ def get_all_message_text(messages: list[Message]) -> str: return "\n".join(text_parts) +def get_terms_for_text(locale: Locale, text: str) -> list[Term]: + """ + Get terms matching a text, with their translation in the locale prefetched. + + Translations are prefetched as `filtered_translations`, the attribute read + by the API `TermSerializer`. They are fetched only for the matched terms, + rather than for every candidate. + """ + terms = Term.objects.order_by("text", "id").for_string(text) + prefetch_related_objects( + terms, + Prefetch( + "translations", + queryset=TermTranslation.objects.filter(locale=locale), + to_attr="filtered_translations", + ), + ) + return terms + + def build_tbx_v2_file(term_translations, locale): """ Generates contents of the TBX 2008 (v2) file (TBX-Default dialect):