Add API to extract terms from provided text

documentation/docs/dev/deployment.md

@@ -17,6 +17,18 @@ Optional. Email address for the `ADMINS` setting.
 `ADMIN_NAME`  
 Optional. Name for the `ADMINS` setting.
 
+`API_TERMINOLOGY_THROTTLE_BURST`
+Optional. Short-window rate limit for the terminology matching API endpoint,
+applied per user, or per IP address for anonymous requests. Uses the Django REST
+Framework format `<number>/<period>`, where period is one of `second`, `minute`,
+`hour` or `day` (default: `60/minute`).
+
+`API_TERMINOLOGY_THROTTLE_SUSTAINED`
+Optional. Long-window rate limit for the terminology matching API endpoint,
+applied per user, or per IP address for anonymous requests. Uses the Django REST
+Framework format `<number>/<period>`, where period is one of `second`, `minute`,
+`hour` or `day` (default: `600/hour`).
+
 `API_UPLOAD_THROTTLE_BURST`  
 Optional. Short-window rate limit for the upload API endpoints,
 applied per authenticated user. Uses the Django REST Framework format

pontoon/api/README.md

@@ -73,6 +73,72 @@ An example may look like this:
 $ curl --globoff "https://example.com/api/v2/locales/?page_size=50"
 ```
 
+## Terminology Matching
+
+### `GET /api/v2/terminology/matches/`
+
+Find the terms appearing in a text, with their translation in a given locale.
+
+Unlike [`/api/v2/search/terminology/`](#/search/search_terminology_list), which looks up terms by name, this
+endpoint matches every known term against the text, at word boundaries: a term matches
+the start of a word, so `open` matches `Opened`, but not `Reopened`. Terms without a
+definition, and terms marked as forbidden, are never returned.
+
+| Parameter | Description                 |
+| --------- | --------------------------- |
+| `locale`  | Locale code                 |
+| `text`    | Text to match terms against |
+
+```bash
+$ curl --globoff \
+  --data-urlencode "locale=it" \
+  --data-urlencode "text=Open a new tab" \
+  --get "https://example.com/api/v2/terminology/matches/"
+```
+
+```json
+{
+  "count": 2,
+  "next": null,
+  "previous": null,
+  "results": [
+    {
+      "definition": "Allow access",
+      "part_of_speech": "verb",
+      "text": "open",
+      "translation_text": "apri",
+      "usage": "Open the door.",
+      "notes": ""
+    },
+    {
+      "definition": "A page in the browser",
+      "part_of_speech": "noun",
+      "text": "tab",
+      "translation_text": "scheda",
+      "usage": "Open a new tab.",
+      "notes": ""
+    }
+  ]
+}
+```
+
+`translation_text` is `null` for terms not yet translated in the locale, and the term
+itself for terms marked as "do not translate", such as product names.
+
+No authentication is required. Texts over the maximum length are rejected with `400`:
+the limit is configurable via `TERMINOLOGY_API_MAX_CHARS` (default 2048 characters).
+An unknown locale returns `404`.
+
+The endpoint is rate limited per user, or per IP address for anonymous requests, with a
+burst limit of 60 calls per minute and a sustained limit of 600 calls per hour by default
+(configurable via `API_TERMINOLOGY_THROTTLE_BURST` and
+`API_TERMINOLOGY_THROTTLE_SUSTAINED`). Calls over the limit are rejected with `429`.
+The two limits are not independent: calls rejected by the burst limit still count against
+the sustained limit, so a client that keeps calling after a `429` spends its hourly quota
+on rejected calls. For example, 60 accepted calls followed by 540 rejected ones exhaust
+the hourly quota, locking the client out for one hour. This quota is separate from the one
+used by the write endpoints.
+
 ## Write Endpoints
 
 The following endpoints can write data and always require authentication with a Personal

pontoon/api/serializers.py

@@ -257,6 +257,9 @@ class Meta:
         ]
 
     def get_translation_text(self, obj):
+        if obj.do_not_translate:
+            return obj.text
+
         if hasattr(obj, "filtered_translations") and (ft := obj.filtered_translations):
             return ft[0].text
 

pontoon/api/tests/test_views.py

@@ -25,6 +25,7 @@
 from pontoon.base.models.translated_resource import TranslatedResource
 from pontoon.base.models.translation import Translation
 from pontoon.base.models.translation_memory import TranslationMemoryEntry
+from pontoon.settings.base import TERMINOLOGY_API_MAX_CHARS
 from pontoon.terminology.models import Term, TermTranslation
 from pontoon.test.factories import (
     EntityFactory,
@@ -1181,6 +1182,211 @@ def test_terminology_search(django_assert_num_queries):
     }
 
 
+@pytest.fixture
+def terminology_matches_setup():
+    locale = LocaleFactory(code="kg", name="Klingon")
+    other_locale = LocaleFactory(code="gs", name="Geonosian")
+
+    term_open = Term.objects.create(
+        text="open",
+        part_of_speech="verb",
+        definition="Allow access",
+        usage="Open the door.",
+    )
+    term_tab = Term.objects.create(
+        text="tab",
+        part_of_speech="noun",
+        definition="A page in the browser",
+        usage="Open a new tab.",
+    )
+    term_click = Term.objects.create(
+        text="click",
+        part_of_speech="verb",
+        definition="Press",
+        usage="Click the button.",
+    )
+    Term.objects.create(
+        text="Firefox",
+        part_of_speech="noun",
+        definition="A web browser",
+        do_not_translate=True,
+    )
+    # Terms without a definition, or forbidden, are never matched
+    Term.objects.create(text="window", part_of_speech="noun", definition="")
+    Term.objects.create(
+        text="bookmark",
+        part_of_speech="noun",
+        definition="A saved page",
+        forbidden=True,
+    )
+
+    TermTranslation.objects.create(term=term_open, locale=locale, text="odpri")
+    TermTranslation.objects.create(term=term_tab, locale=locale, text="zavihek")
+    TermTranslation.objects.create(term=term_click, locale=other_locale, text="klikni")
+
+    return SimpleNamespace(locale=locale, other_locale=other_locale)
+
+
+@pytest.mark.django_db
+def test_terminology_matches(terminology_matches_setup, django_assert_num_queries):
+    with django_assert_num_queries(3):
+        response = APIClient().get(
+            "/api/v2/terminology/matches/",
+            {"locale": "kg", "text": "Open a new tab in this window."},
+        )
+
+    assert response.status_code == 200
+    assert response.data == {
+        "count": 2,
+        "next": None,
+        "previous": None,
+        "results": [
+            {
+                "definition": "Allow access",
+                "part_of_speech": "verb",
+                "text": "open",
+                "translation_text": "odpri",
+                "usage": "Open the door.",
+                "notes": "",
+            },
+            {
+                "definition": "A page in the browser",
+                "part_of_speech": "noun",
+                "text": "tab",
+                "translation_text": "zavihek",
+                "usage": "Open a new tab.",
+                "notes": "",
+            },
+        ],
+    }
+
+
+@pytest.mark.django_db
+def test_terminology_matches_word_start(terminology_matches_setup):
+    """Terms are matched at the start of a word, to also catch inflected forms."""
+    response = APIClient().get(
+        "/api/v2/terminology/matches/",
+        {"locale": "kg", "text": "Reopened the crab."},
+    )
+
+    assert response.status_code == 200
+    assert response.data["results"] == []
+
+    response = APIClient().get(
+        "/api/v2/terminology/matches/",
+        {"locale": "kg", "text": "Opened the tabs."},
+    )
+
+    assert response.status_code == 200
+    assert [t["text"] for t in response.data["results"]] == ["open", "tab"]
+
+
+@pytest.mark.django_db
+def test_terminology_matches_missing_translation(
+    terminology_matches_setup,
+):
+    response = APIClient().get(
+        "/api/v2/terminology/matches/",
+        {"locale": "kg", "text": "Click here."},
+    )
+
+    assert response.status_code == 200
+    assert [(t["text"], t["translation_text"]) for t in response.data["results"]] == [
+        ("click", None)
+    ]
+
+
+@pytest.mark.django_db
+def test_terminology_matches_do_not_translate(terminology_matches_setup):
+    """Terms that must not be translated are reported as-is, in every locale."""
+    response = APIClient().get(
+        "/api/v2/terminology/matches/",
+        {"locale": "kg", "text": "Open Firefox."},
+    )
+
+    assert response.status_code == 200
+    assert [(t["text"], t["translation_text"]) for t in response.data["results"]] == [
+        ("Firefox", "Firefox"),
+        ("open", "odpri"),
+    ]
+
+
+@pytest.mark.django_db
+def test_terminology_matches_fields(terminology_matches_setup):
+    response = APIClient().get(
+        "/api/v2/terminology/matches/",
+        {"locale": "kg", "text": "Open a new tab.", "fields": "text"},
+    )
+
+    assert response.status_code == 200
+    assert response.data["results"] == [{"text": "open"}, {"text": "tab"}]
+
+
+@pytest.mark.django_db
+def test_terminology_matches_errors(terminology_matches_setup):
+    client = APIClient()
+
+    response = client.get("/api/v2/terminology/matches/", {"text": "Open"})
+    assert response.status_code == 400
+    assert response.data == {"locale": ["This field is required."]}
+
+    response = client.get("/api/v2/terminology/matches/", {"locale": "kg"})
+    assert response.status_code == 400
+    assert response.data == {"text": ["This field is required."]}
+
+    response = client.get(
+        "/api/v2/terminology/matches/", {"locale": "missing", "text": "Open"}
+    )
+    assert response.status_code == 404
+
+    response = client.get(
+        "/api/v2/terminology/matches/",
+        {"locale": "kg", "text": "Open a new tab. " * TERMINOLOGY_API_MAX_CHARS},
+    )
+    assert response.status_code == 400
+    assert response.data == {
+        "text": [
+            f"Text exceeds maximum length of {TERMINOLOGY_API_MAX_CHARS} characters."
+        ]
+    }
+
+    # Whitespace-only text that is also too long reports the length error
+    response = client.get(
+        "/api/v2/terminology/matches/",
+        {"locale": "kg", "text": " " * (TERMINOLOGY_API_MAX_CHARS + 1)},
+    )
+    assert response.status_code == 400
+    assert response.data == {
+        "text": [
+            f"Text exceeds maximum length of {TERMINOLOGY_API_MAX_CHARS} characters."
+        ]
+    }
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "rates",
+    [
+        {"terminology_burst": "2/minute", "terminology_sustained": "1000/hour"},
+        {"terminology_burst": "60/minute", "terminology_sustained": "2/hour"},
+    ],
+)
+def test_terminology_matches_throttled(monkeypatch, terminology_matches_setup, rates):
+    # DRF copies the rates into a class attribute at import time, so overriding the
+    # REST_FRAMEWORK setting has no effect here.
+    monkeypatch.setattr(SimpleRateThrottle, "THROTTLE_RATES", rates)
+    cache.clear()
+
+    client = APIClient()
+    for expected_status in (200, 200, 429):
+        response = client.get(
+            "/api/v2/terminology/matches/", {"locale": "kg", "text": "Open a new tab."}
+        )
+        assert response.status_code == expected_status
+
+    cache.clear()
+
+
 @pytest.mark.django_db
 def test_tm_search(django_assert_num_queries):
     locale_a = LocaleFactory(

pontoon/api/throttling.py

@@ -35,7 +35,7 @@ class SustainedRateThrottle(_SuffixedScopedRateThrottle):
     suffix = "sustained"
 
 
-# Throttles for endpoints that write translations from uploaded files.
-# Views using these should set `throttle_scope = "upload"`
-# so that they share a single quota per user.
-UPLOAD_THROTTLE_CLASSES = [BurstRateThrottle, SustainedRateThrottle]
+# Throttles for expensive endpoints. Views using these should set a `throttle_scope`,
+# and endpoints sharing a scope share a single quota per user (or per IP address, for
+# anonymous requests).
+SCOPED_THROTTLE_CLASSES = [BurstRateThrottle, SustainedRateThrottle]

pontoon/api/urls.py

@@ -61,6 +61,11 @@
         views.UploadPretranslationsView.as_view(),
         name="upload-pretranslations",
     ),
+    path(
+        "terminology/matches/",
+        views.TermMatchListView.as_view(),
+        name="term-matches",
+    ),
     path(
         # Terminology Search
         "search/terminology/",