MPT-26918 makes different backends behave consistently

.github/workflows/pr-build-merge.yaml

@@ -15,6 +15,21 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
 
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: requela
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d requela"
+          --health-interval 2s
+          --health-timeout 5s
+          --health-retries 15
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -41,6 +56,8 @@ jobs:
 
       - name: Run tests
         run: uv run pytest
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/requela
 
       - name: Compute added/removed lines for notification
         if: ${{ github.event_name == 'pull_request' }}

docker-compose.yaml

@@ -1,9 +1,29 @@
-version: '3.8'
-
 services:
+  postgres:
+    image: postgres:17
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: requela
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
+    # Throwaway test database: keep the data in memory
+    tmpfs:
+      - /var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d requela"]
+      interval: 2s
+      timeout: 5s
+      retries: 15
+
   tests:
     build: .
-    command: pytest -v 
+    command: pytest -v
+    environment:
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/requela
+    depends_on:
+      postgres:
+        condition: service_healthy
 
 
   bash:
@@ -12,3 +32,8 @@ services:
     command: bash
     stdin_open: true
     tty: true
+    environment:
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/requela
+    depends_on:
+      postgres:
+        condition: service_healthy

pyproject.toml

@@ -23,7 +23,6 @@ classifiers = [
 
 dependencies = [
     "lark>=1.2.2",
-    "pytest-randomly>=3.16.0",
 ]
 
 [project.urls]
@@ -48,6 +47,7 @@ dev = [
     "ruff>=0.8.0,<1.0",
     "django>=5.1.6",
     "pre-commit>=4.1.0",
+    "pytest-randomly>=3.16.0",
 ]
 
 

src/requela/builders/django.py

@@ -1,22 +1,32 @@
-from collections.abc import Sequence
-from dataclasses import dataclass
+import re
+from collections.abc import Callable, Sequence
 from datetime import date, datetime
+from typing import Any
 
 from django.core.exceptions import FieldDoesNotExist
-from django.db.models import Q
+from django.db.models import Exists, OuterRef, Q
 from django.db.models.query import QuerySet
 
 from requela.builders.base import QueryBuilder
 from requela.dataclasses import FilterExpression, OrderByExpression
 
 
-@dataclass
-class PrefetchExpression:
-    relationship_name: str
-    condition: Q
-
-
 class DjangoQueryBuilder(QueryBuilder):
+    def __init__(
+        self,
+        model_class: Any,
+        resolve_alias_callback: Callable | None = None,
+        validate_operator_and_field_callback: Callable | None = None,
+        validate_ordering_callback: Callable | None = None,
+    ):
+        super().__init__(
+            model_class,
+            resolve_alias_callback=resolve_alias_callback,
+            validate_operator_and_field_callback=validate_operator_and_field_callback,
+            validate_ordering_callback=validate_ordering_callback,
+        )
+        self.prefetches: list[str] = []
+
     def get_initial_query(self):
         return self.model_class.objects.all()
 
@@ -86,41 +96,73 @@ def apply_out(self, prop: str, value: Sequence[str] | Sequence[float] | Sequence
         return ~Q(**{f"{self.resolve_property(prop)}__in": value})
 
     def apply_like(self, prop: str, value: str) -> Q:
-        if value.startswith("*") and value.endswith("*"):
-            return Q(**{f"{self.resolve_property(prop)}__contains": value[1:-1]})
-        elif value.startswith("*") and not value.endswith("*"):
-            return Q(**{f"{self.resolve_property(prop)}__endswith": value[1:]})
-        elif not value.startswith("*") and value.endswith("*"):
-            return Q(**{f"{self.resolve_property(prop)}__startswith": value[:-1]})
-        else:
-            return Q(**{f"{self.resolve_property(prop)}__contains": value})
+        return self.apply_pattern(prop, value, case_insensitive=False)
 
     def apply_ilike(self, prop: str, value: str) -> Q:
-        if value.startswith("*") and value.endswith("*"):
-            return Q(**{f"{self.resolve_property(prop)}__icontains": value[1:-1]})
-        elif value.startswith("*") and not value.endswith("*"):
-            return Q(**{f"{self.resolve_property(prop)}__iendswith": value[1:]})
-        elif not value.startswith("*") and value.endswith("*"):
-            return Q(**{f"{self.resolve_property(prop)}__istartswith": value[:-1]})
-        else:
-            return Q(**{f"{self.resolve_property(prop)}__icontains": value})
+        return self.apply_pattern(prop, value, case_insensitive=True)
+
+    def apply_pattern(self, prop: str, value: str, case_insensitive: bool) -> Q:
+        prefix = "i" if case_insensitive else ""
+        field = self.resolve_property(prop)
+        leading = value.startswith("*")
+        trailing = len(value) > 1 and value.endswith("*")
+        text = value[1 if leading else None : -1 if trailing else None]
+
+        if "*" in text:
+            # Wildcards in the middle: no lookup fits, use an anchored regex
+            regex = "^" + ".*".join(re.escape(part) for part in value.split("*")) + "$"
+            return Q(**{f"{field}__{prefix}regex": regex})
+        if leading and trailing:
+            return Q(**{f"{field}__{prefix}contains": text})
+        if leading:
+            return Q(**{f"{field}__{prefix}endswith": text})
+        if trailing:
+            return Q(**{f"{field}__{prefix}startswith": text})
+        return Q(**{f"{field}__{prefix}exact": text})
 
     def resolve_property(self, prop_path: str) -> str:
         prop = self.resolve_alias(prop_path)
         return prop.replace(".", "__")
 
-    def apply_any(self, relationship_name, condition: Q) -> PrefetchExpression:
-        return PrefetchExpression(relationship_name, condition)
+    def apply_any(self, relationship_name, condition: Q) -> Q:
+        relationship_name = self.resolve_alias(relationship_name)
+        relationship = self.model_class._meta.get_field(relationship_name)
+        related_rows = relationship.related_model.objects.filter(
+            **{relationship.remote_field.name: OuterRef("pk")}
+        ).filter(self._relative_condition(condition, f"{relationship_name}__"))
+        if relationship_name not in self.prefetches:
+            self.prefetches.append(relationship_name)
+        return Q(Exists(related_rows))
+
+    def _relative_condition(self, condition: Q, prefix: str) -> Q:
+        children = []
+        for child in condition.children:
+            if isinstance(child, Q):
+                children.append(self._relative_condition(child, prefix))
+            elif isinstance(child, tuple) and child[0].startswith(prefix):
+                children.append((child[0].removeprefix(prefix), child[1]))
+            elif isinstance(child, tuple):
+                relationship = prefix.removesuffix("__")
+                raise ValueError(
+                    f"Field '{child[0].replace('__', '.')}' is not part of "
+                    f"relationship '{relationship}'."
+                )
+            else:
+                raise ValueError("Nested any() is not supported.")
+        return Q(*children, _connector=condition.connector, _negated=condition.negated)
 
     def apply_filter(self, query: QuerySet, filter_expression: FilterExpression) -> QuerySet:
-        if isinstance(filter_expression.condition, PrefetchExpression):
-            query = query.prefetch_related(filter_expression.condition.relationship_name)
-            return query.filter(filter_expression.condition.condition)
         return query.filter(filter_expression.condition)
 
     def apply_joins(self, query: QuerySet):
+        if self.prefetches:
+            query = query.prefetch_related(*self.prefetches)
         return query
 
+    def build_query(self, rql_query: str, initial_query: Any = None) -> Any:
+        self.prefetches = []
+        return super().build_query(rql_query=rql_query, initial_query=initial_query)
+
     def apply_order_by(self, query: QuerySet, order_by_expression: OrderByExpression) -> QuerySet:
         fields = []
         for order_field in order_by_expression.fields:

src/requela/builders/sqlalchemy.py

@@ -12,10 +12,12 @@
     UnaryExpression,
     and_,
     exists,
+    inspect,
     or_,
     select,
 )
 from sqlalchemy.orm import DeclarativeBase, Query, RelationshipProperty, aliased
+from sqlalchemy.sql.util import find_tables
 
 from requela.builders.base import QueryBuilder
 from requela.dataclasses import FilterExpression, JoinExpression, OrderByExpression
@@ -36,6 +38,9 @@ def __init__(
             validate_ordering_callback=validate_ordering_callback,
         )
         self.joins: list[JoinExpression] = []
+        # Aliases of to-many relationships that are not joined, mapped to the property path
+        # that created them. They are only valid inside `any()`, which replaces them.
+        self.to_many_aliases: dict[Any, str] = {}
 
     def get_initial_query(self):
         return select(self.model_class)
@@ -140,6 +145,7 @@ def apply_ilike(self, prop: str, value: str) -> ColumnExpressionArgument:
         return self.resolve_property(prop).ilike(sql_pattern)
 
     def resolve_property(self, prop_path: str) -> UnaryExpression:
+        field_path = prop_path
         prop_path = self.resolve_alias(prop_path)
         model = self.model_class
         parts = prop_path.split(".")
@@ -152,7 +158,9 @@ def resolve_property(self, prop_path: str) -> UnaryExpression:
                 target_model = relationship.property.mapper.class_
                 alias = aliased(target_model)
 
-                if not relationship.property.uselist:
+                if relationship.property.uselist:
+                    self.to_many_aliases[inspect(alias).selectable] = field_path
+                else:
                     is_nullable = any(
                         column.nullable for column in relationship.property.local_columns
                     )
@@ -168,7 +176,7 @@ def resolve_property(self, prop_path: str) -> UnaryExpression:
     def apply_any(
         self, relationship_name, condition: ColumnElement | ColumnExpressionArgument
     ) -> Exists:
-        relationship = getattr(self.model_class, relationship_name)
+        relationship = getattr(self.model_class, self.resolve_alias(relationship_name))
         related_model = relationship.property.mapper.class_
         alias = aliased(related_model)
 
@@ -211,7 +219,18 @@ def _adapt_condition(self, condition, alias):
             )
         return condition.operator(getattr(alias, condition.left.key), condition.right.value)
 
+    def check_no_to_many_traversal(self, clause: Any) -> None:
+        for table in find_tables(clause, check_columns=True, include_aliases=True):
+            if table in self.to_many_aliases:
+                field_path = self.to_many_aliases[table]
+                relationship_path = field_path.rsplit(".", 1)[0]
+                raise ValueError(
+                    f"Field '{field_path}' goes through the to-many relationship "
+                    f"'{relationship_path}': use any() to filter on it."
+                )
+
     def apply_filter(self, query: Query, filter_expression: FilterExpression) -> Query:
+        self.check_no_to_many_traversal(filter_expression.condition)
         return query.filter(filter_expression.condition)
 
     def apply_joins(self, query: Query):
@@ -223,11 +242,13 @@ def apply_order_by(self, query: Query, order_by_expression: OrderByExpression) -
         fields = []
         for order_field in order_by_expression.fields:
             field = self.resolve_property(order_field.field_path)
+            self.check_no_to_many_traversal(field.expression)
             if order_field.direction == "-":
                 field = field.desc()
             fields.append(field)
         return query.order_by(*fields)
 
     def build_query(self, rql_query: str, initial_query: Any = None) -> Any:
         self.joins = []
+        self.to_many_aliases = {}
         return super().build_query(rql_query=rql_query, initial_query=initial_query)

src/requela/rules.py

@@ -198,7 +198,7 @@ def _get_relation_by_alias(cls, alias: str) -> tuple[str, RelationshipRule]:
         relations = {}
         for relation_name, relation_def in cls._relations.items():
             relation_field_name = relation_def.alias or relation_name
-            if alias.startswith(relation_field_name):
+            if alias == relation_field_name or alias.startswith(f"{relation_field_name}."):
                 relations[relation_name] = relation_def
         if not relations:
             raise ValueError(f"Relation with alias '{alias}' not found")

tests/conftest.py

@@ -1,16 +1,26 @@
-def pytest_sessionstart(session):
-    import django
-    from django.conf import settings
+from urllib.parse import urlsplit
 
-    settings.configure(
-        INSTALLED_APPS=[
-            "tests.django",
-        ],
-        DATABASES={
-            "default": {
-                "ENGINE": "django.db.backends.sqlite3",
-                "NAME": ":memory:",
-            }
-        },
-    )
-    django.setup()
+import django
+from django.conf import settings
+
+from tests.settings import DATABASE_URL
+
+# Configured at import time, not in a pytest hook: conftest files below this one import the
+# Django models and are loaded before session hooks run when their directory is targeted.
+url = urlsplit(DATABASE_URL)
+settings.configure(
+    INSTALLED_APPS=[
+        "tests.django",
+    ],
+    DATABASES={
+        "default": {
+            "ENGINE": "django.db.backends.postgresql",
+            "NAME": url.path.lstrip("/"),
+            "USER": url.username,
+            "PASSWORD": url.password,
+            "HOST": url.hostname,
+            "PORT": url.port,
+        }
+    },
+)
+django.setup()

tests/django/rules.py

@@ -29,3 +29,17 @@ class UserRules(ModelRQLRules):
         alias="account",
         rules=AccountRules(),
     )
+
+
+class UserNameRules(ModelRQLRules):
+    __model__ = User
+
+    name = FieldRule()
+    age = FieldRule()
+
+
+class AccountWithAliasedUsersRules(ModelRQLRules):
+    __model__ = Account
+
+    name = FieldRule()
+    users = RelationshipRule(alias="members", rules=UserNameRules())