From 498a669b4eeb5d835f87f6157a6a8056c749456b Mon Sep 17 00:00:00 2001 From: lujiakun Date: Tue, 13 Oct 2026 16:08:40 +0800 Subject: [PATCH 1/5] feat: add distributed expiry scheduler --- Dockerfile | 4 +- MANIFEST.in | 1 + openviking/scheduling/__init__.py | 3 + openviking/scheduling/expiry/__init__.py | 31 + openviking/scheduling/expiry/api.py | 85 ++ openviking/scheduling/expiry/config.py | 86 ++ openviking/scheduling/expiry/consumer.py | 369 +++++++++ openviking/scheduling/expiry/keys.py | 122 +++ openviking/scheduling/expiry/leases.py | 101 +++ openviking/scheduling/expiry/lua/_common.lua | 87 ++ .../scheduling/expiry/lua/cas_zset_member.lua | 16 + .../expiry/lua/cleanup_source_page.lua | 8 + .../scheduling/expiry/lua/compare_owner.lua | 13 + .../expiry/lua/defer_registration.lua | 3 + .../scheduling/expiry/lua/ensure_metadata.lua | 19 + .../scheduling/expiry/lua/publish_minute.lua | 15 + .../scheduling/expiry/lua/read_zset_page.lua | 13 + .../scheduling/expiry/lua/upsert_resource.lua | 7 + openviking/scheduling/expiry/metrics.py | 75 ++ openviking/scheduling/expiry/models.py | 236 ++++++ openviking/scheduling/expiry/publisher.py | 223 ++++++ openviking/scheduling/expiry/redis_store.py | 752 ++++++++++++++++++ openviking/scheduling/expiry/registry.py | 48 ++ openviking/scheduling/expiry/runtime.py | 398 +++++++++ openviking/scheduling/expiry/service.py | 97 +++ openviking/scheduling/expiry/shard_runner.py | 353 ++++++++ openviking/service/core.py | 53 +- openviking/service/session_service.py | 145 +++- openviking/service/session_timer.py | 152 ++++ openviking/session/session.py | 65 +- openviking_cli/expiry_bootstrap.py | 76 ++ openviking_cli/utils/config/__init__.py | 3 + openviking_cli/utils/config/expiry_config.py | 5 + openviking_cli/utils/config/memory_config.py | 7 + .../utils/config/open_viking_config.py | 24 + pyproject.toml | 5 + setup.py | 1 + tests/scheduling/expiry/test_config.py | 101 +++ tests/scheduling/expiry/test_consumer.py | 668 ++++++++++++++++ .../expiry/test_facade_and_registry.py | 130 +++ tests/scheduling/expiry/test_leases.py | 538 +++++++++++++ .../scheduling/expiry/test_models_and_keys.py | 102 +++ tests/scheduling/expiry/test_publisher.py | 306 +++++++ tests/scheduling/expiry/test_redis_loop.py | 338 ++++++++ tests/scheduling/expiry/test_redis_store.py | 411 ++++++++++ tests/scheduling/expiry/test_runtime.py | 338 ++++++++ .../expiry/test_session_timeout_service.py | 637 +++++++++++++++ tests/scheduling/expiry/test_session_timer.py | 223 ++++++ tests/session/test_session_commit.py | 92 +++ uv.lock | 19 +- 50 files changed, 7596 insertions(+), 8 deletions(-) create mode 100644 openviking/scheduling/__init__.py create mode 100644 openviking/scheduling/expiry/__init__.py create mode 100644 openviking/scheduling/expiry/api.py create mode 100644 openviking/scheduling/expiry/config.py create mode 100644 openviking/scheduling/expiry/consumer.py create mode 100644 openviking/scheduling/expiry/keys.py create mode 100644 openviking/scheduling/expiry/leases.py create mode 100644 openviking/scheduling/expiry/lua/_common.lua create mode 100644 openviking/scheduling/expiry/lua/cas_zset_member.lua create mode 100644 openviking/scheduling/expiry/lua/cleanup_source_page.lua create mode 100644 openviking/scheduling/expiry/lua/compare_owner.lua create mode 100644 openviking/scheduling/expiry/lua/defer_registration.lua create mode 100644 openviking/scheduling/expiry/lua/ensure_metadata.lua create mode 100644 openviking/scheduling/expiry/lua/publish_minute.lua create mode 100644 openviking/scheduling/expiry/lua/read_zset_page.lua create mode 100644 openviking/scheduling/expiry/lua/upsert_resource.lua create mode 100644 openviking/scheduling/expiry/metrics.py create mode 100644 openviking/scheduling/expiry/models.py create mode 100644 openviking/scheduling/expiry/publisher.py create mode 100644 openviking/scheduling/expiry/redis_store.py create mode 100644 openviking/scheduling/expiry/registry.py create mode 100644 openviking/scheduling/expiry/runtime.py create mode 100644 openviking/scheduling/expiry/service.py create mode 100644 openviking/scheduling/expiry/shard_runner.py create mode 100644 openviking/service/session_timer.py create mode 100644 openviking_cli/expiry_bootstrap.py create mode 100644 openviking_cli/utils/config/expiry_config.py create mode 100644 tests/scheduling/expiry/test_config.py create mode 100644 tests/scheduling/expiry/test_consumer.py create mode 100644 tests/scheduling/expiry/test_facade_and_registry.py create mode 100644 tests/scheduling/expiry/test_leases.py create mode 100644 tests/scheduling/expiry/test_models_and_keys.py create mode 100644 tests/scheduling/expiry/test_publisher.py create mode 100644 tests/scheduling/expiry/test_redis_loop.py create mode 100644 tests/scheduling/expiry/test_redis_store.py create mode 100644 tests/scheduling/expiry/test_runtime.py create mode 100644 tests/scheduling/expiry/test_session_timeout_service.py create mode 100644 tests/scheduling/expiry/test_session_timer.py diff --git a/Dockerfile b/Dockerfile index a13872431d..18ac814235 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,13 +83,13 @@ RUN --mount=type=cache,target=/root/.cache/uv,id=uv-${TARGETPLATFORM} \ fi; \ case "${UV_LOCK_STRATEGY}" in \ locked) \ - uv sync --locked --no-editable --reinstall-package openviking --extra bot --extra gemini --extra opengauss --extra context-gateway \ + uv sync --locked --no-editable --reinstall-package openviking --extra bot --extra gemini --extra opengauss --extra context-gateway --extra expiry \ ;; \ auto) \ if ! uv lock --check; then \ uv lock; \ fi; \ - uv sync --locked --no-editable --reinstall-package openviking --extra bot --extra gemini --extra opengauss --extra context-gateway \ + uv sync --locked --no-editable --reinstall-package openviking --extra bot --extra gemini --extra opengauss --extra context-gateway --extra expiry \ ;; \ *) \ echo "Unsupported UV_LOCK_STRATEGY: ${UV_LOCK_STRATEGY}" >&2; \ diff --git a/MANIFEST.in b/MANIFEST.in index a0e67c64fd..a3f13b40e6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,6 +12,7 @@ include Cargo.lock graft crates/ragfs graft crates/ragfs-python recursive-include openviking *.yaml +recursive-include openviking/scheduling/expiry *.lua recursive-include context_gateway *.json # sdist should be source-only: never ship runtime binaries from working tree diff --git a/openviking/scheduling/__init__.py b/openviking/scheduling/__init__.py new file mode 100644 index 0000000000..e908d0001e --- /dev/null +++ b/openviking/scheduling/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Scheduling implementations.""" diff --git a/openviking/scheduling/expiry/__init__.py b/openviking/scheduling/expiry/__init__.py new file mode 100644 index 0000000000..fcb0117a66 --- /dev/null +++ b/openviking/scheduling/expiry/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +from .api import ( + SESSION_ITEM_TYPE, + CallbackRequest, + CallbackResult, + CallbackStatus, + ExpiryError, + ExpiryHandler, + ExpiryMode, + ExpiryRegistration, + ExpiryRegistrationResult, + ExpiryScheduler, + HandlerOptions, + ItemIdentity, +) + +__all__ = [ + "ExpiryScheduler", + "SESSION_ITEM_TYPE", + "ItemIdentity", + "ExpiryRegistration", + "ExpiryRegistrationResult", + "ExpiryMode", + "ExpiryError", + "HandlerOptions", + "ExpiryHandler", + "CallbackRequest", + "CallbackResult", + "CallbackStatus", +] diff --git a/openviking/scheduling/expiry/api.py b/openviking/scheduling/expiry/api.py new file mode 100644 index 0000000000..13ffed0fbe --- /dev/null +++ b/openviking/scheduling/expiry/api.py @@ -0,0 +1,85 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Public contracts for distributed expiry scheduling.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Protocol + +SESSION_ITEM_TYPE = "session" +SUPPORTED_ITEM_TYPES: frozenset[str] = frozenset({SESSION_ITEM_TYPE}) + + +class ExpiryMode(str, Enum): + IDLE_ONESHOT = "idle_oneshot" + FIXED = "fixed" + + +class CallbackStatus(str, Enum): + COMPLETED = "completed" + RETRY = "retry" + + +class ExpiryError(RuntimeError): + def __init__(self, code: str, message: str, *, retryable: bool = False) -> None: + super().__init__(message) + self.code = code + self.retryable = retryable + + +@dataclass(frozen=True) +class ItemIdentity: + item_type: str + account_id: str + user_id: str + local_id: str + + +@dataclass(frozen=True) +class ExpiryRegistration: + identity: ItemIdentity + due_at_ms: int + timeout_seconds: int + mode: ExpiryMode = ExpiryMode.IDLE_ONESHOT + + +@dataclass(frozen=True) +class ExpiryRegistrationResult: + accepted_due_at_ms: int + + +@dataclass(frozen=True) +class CallbackRequest: + identity: ItemIdentity + + +@dataclass(frozen=True) +class CallbackResult: + status: CallbackStatus + reason: str + + +@dataclass(frozen=True) +class HandlerOptions: + concurrency: int = 4 + max_attempts: int = 3 + + +class ExpiryHandler(Protocol): + async def handle(self, request: CallbackRequest) -> CallbackResult: ... + + +class ExpiryScheduler(Protocol): + async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: ... + + async def unregister(self, identity: ItemIdentity) -> None: ... + + def register_handler( + self, + item_type: str, + handler: ExpiryHandler, + *, + options: HandlerOptions | None = None, + ) -> None: ... diff --git a/openviking/scheduling/expiry/config.py b/openviking/scheduling/expiry/config.py new file mode 100644 index 0000000000..7ee2e084ea --- /dev/null +++ b/openviking/scheduling/expiry/config.py @@ -0,0 +1,86 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Configuration independent from the aggregate OpenViking config package.""" + +from __future__ import annotations + +from typing import Literal +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class ExpiryRedisConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + mode: Literal["standalone", "cluster"] = "standalone" + endpoints: list[str] = Field(default_factory=list) + username: str | None = None + password: str | None = None + password_env: str | None = None + tls: bool = False + connect_timeout_seconds: float = Field(default=1.0, gt=0) + socket_timeout_seconds: float = Field(default=2.0, gt=0) + request_budget_seconds: float = Field(default=5.0, gt=0) + max_connections: int = Field(default=64, ge=1) + max_inflight_requests: int = Field(default=128, ge=1) + + @model_validator(mode="after") + def validate_connection(self) -> "ExpiryRedisConfig": + if self.password is not None and self.password_env is not None: + raise ValueError("Redis password and password_env are mutually exclusive") + if self.mode == "standalone" and len(self.endpoints) > 1: + raise ValueError("standalone Redis accepts at most one endpoint") + schemes: set[str] = set() + for endpoint in self.endpoints: + normalized = ( + endpoint + if "://" in endpoint + else (("rediss" if self.tls else "redis") + f"://{endpoint}") + ) + parsed = urlsplit(normalized) + if parsed.scheme not in {"redis", "rediss"} or not parsed.hostname or not parsed.port: + raise ValueError(f"invalid Redis endpoint: {endpoint}") + if parsed.username or parsed.password or parsed.path not in {"", "/"}: + raise ValueError("Redis endpoints must not contain credentials or paths") + schemes.add(parsed.scheme) + if len(schemes) > 1: + raise ValueError("Redis cluster endpoints must use one transport scheme") + return self + + +class ExpiryConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + service_name: str = "" + reporting_enabled: bool = False + scheduler_enabled: bool = False + callback_enabled: bool = False + shard_count: Literal[512] = 512 + precision_seconds: Literal[60] = 60 + owner_ttl_seconds: Literal[120] = 120 + owner_renew_interval_seconds: Literal[20] = 20 + scan_batch_size: int = Field(default=128, ge=1, le=4096) + cleanup_batch_size: int = Field(default=128, ge=1, le=4096) + registry_page_size: int = Field(default=128, ge=1, le=4096) + processing_timeout_seconds: int = Field(default=120, ge=1) + callback_poll_seconds: float = Field(default=0.5, gt=0) + shutdown_timeout_seconds: float = Field(default=30.0, gt=0) + redis: ExpiryRedisConfig = Field(default_factory=ExpiryRedisConfig) + + @property + def enabled(self) -> bool: + return self.reporting_enabled or self.scheduler_enabled or self.callback_enabled + + @model_validator(mode="after") + def validate_roles(self) -> "ExpiryConfig": + if self.enabled: + if not self.service_name.strip(): + raise ValueError("expiry.service_name is required when enabled") + if not self.redis.endpoints: + raise ValueError("expiry.redis.endpoints is required when enabled") + if self.redis.request_budget_seconds >= self.owner_renew_interval_seconds: + raise ValueError("Redis request budget must be shorter than owner renew interval") + if self.callback_enabled and not self.reporting_enabled: + raise ValueError("callback_enabled requires reporting_enabled for deferred reports") + return self diff --git a/openviking/scheduling/expiry/consumer.py b/openviking/scheduling/expiry/consumer.py new file mode 100644 index 0000000000..3a630acd33 --- /dev/null +++ b/openviking/scheduling/expiry/consumer.py @@ -0,0 +1,369 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Published resource callback consumption.""" + +from __future__ import annotations + +import asyncio +import secrets +from dataclasses import dataclass, field +from typing import Awaitable, Callable + +from .api import CallbackRequest, CallbackStatus, ExpiryError +from .config import ExpiryConfig +from .keys import decode_item_id +from .metrics import ExpiryMetrics +from .models import ( + BucketItem, + BucketRef, + Claim, + OwnerRole, + StoreOperationError, + decode_claim_score, +) +from .redis_store import RedisExpiryStore +from .registry import HandlerBinding +from .shard_runner import ShardExecution, ShardRunner + + +@dataclass +class _ConsumerState: + reserved: int = 0 + tasks: set[asyncio.Task[None]] = field(default_factory=set) + + +@dataclass(frozen=True) +class _BucketScan: + next_poll_at_ms: int | None = None + conflicted: bool = False + + +class ExpiryConsumer: + def __init__( + self, + store: RedisExpiryStore, + runner: ShardRunner, + item_type: str, + binding: HandlerBinding, + config: ExpiryConfig, + metrics: ExpiryMetrics, + *, + start_shard: int | None = None, + ) -> None: + self._store = store + self._runner = runner + self._item_type = item_type + self._binding = binding + self._config = config + self._metrics = metrics + self._state = _ConsumerState() + self._next_shard = ( + secrets.randbelow(config.shard_count) if start_shard is None else start_shard + ) + + async def run(self, stop: asyncio.Event) -> None: + """Traverse every shard from a per-consumer offset.""" + while not stop.is_set(): + start_shard = self._next_shard + for offset in range(self._config.shard_count): + if stop.is_set(): + break + shard_id = (start_shard + offset) % self._config.shard_count + try: + await self._runner.run_if_acquired( + OwnerRole.CALLBACK, + shard_id, + self.process_shard, + stop, + item_type=self._item_type, + ) + except StoreOperationError as exc: + if exc.code == "shard_not_stopped": + raise + self._metrics.count( + "timer_shard_round", + item_type=self._item_type, + outcome="failed", + ) + except Exception: + self._metrics.count( + "timer_shard_round", + item_type=self._item_type, + outcome="failed", + ) + self._next_shard = (start_shard + 1) % self._config.shard_count + if not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), self._config.callback_poll_seconds) + except TimeoutError: + pass + + async def process_shard(self, lease: ShardExecution, stop: asyncio.Event) -> None: + self._reset_state() + try: + bucket = await self._discover_bucket(lease) + if bucket is None: + return + await self._process_bucket(lease, stop, bucket) + finally: + # The runner retains children which have not actually stopped. + if not any(not task.done() for task in self._state.tasks): + self._reset_state() + + async def _discover_bucket(self, lease: ShardExecution) -> BucketRef | None: + cursor = None + while True: + lease.ensure_active() + page = await self._store.read_registry_page( + lease.owner.shard_id, + after=cursor, + limit=self._config.registry_page_size, + ) + for bucket in page.buckets: + if bucket.item_type == self._item_type: + return bucket + cursor = page.next_cursor + if cursor is None: + return None + + async def _process_bucket( + self, + lease: ShardExecution, + stop: asyncio.Event, + bucket: BucketRef, + ) -> None: + while not stop.is_set(): + lease.ensure_active() + available = self._binding.options.concurrency - self._state.reserved + scan = _BucketScan() + redis_now_ms = 0 + if available > 0: + redis_now_ms = await self._store.read_time_ms() + lease.ensure_active() + scan = await self._claim_ready_items( + lease, + bucket, + redis_now_ms=redis_now_ms, + available=available, + ) + + active = {task for task in self._state.tasks if not task.done()} + if active: + await lease.wait(self._config.callback_poll_seconds, stop) + continue + if scan.conflicted: + continue + + lease.ensure_active() + empty = await self._store.remove_registration_if_empty(bucket) + lease.ensure_active() + if empty: + return + if scan.next_poll_at_ms is None or scan.next_poll_at_ms <= redis_now_ms: + continue + await self._store.defer_registration( + lease.owner, + bucket, + next_poll_at_ms=scan.next_poll_at_ms, + ) + return + + tasks = {task for task in self._state.tasks if not task.done()} + if tasks: + await asyncio.wait(tasks) + lease.ensure_active() + + async def _claim_ready_items( + self, + lease: ShardExecution, + bucket: BucketRef, + *, + redis_now_ms: int, + available: int, + ) -> _BucketScan: + next_poll_at_ms: int | None = None + started_at_seconds = redis_now_ms // 1000 + for attempt in range(self._binding.options.max_attempts): + while available > 0: + limit = min(self._config.scan_batch_size, available) + page = await self._store.read_bucket_page( + bucket, + attempt=attempt, + limit=limit, + ) + lease.ensure_active() + if not page.items: + break + + claimed_from_full_page = len(page.items) == limit + for item in page.items: + ready_at_ms = self._ready_at_ms(item, attempt) + if ready_at_ms is not None and ready_at_ms > redis_now_ms: + next_poll_at_ms = ( + ready_at_ms + if next_poll_at_ms is None + else min(next_poll_at_ms, ready_at_ms) + ) + claimed_from_full_page = False + break + + claim = await self._store.claim( + bucket, + item, + attempt=attempt, + started_at_seconds=started_at_seconds, + ) + lease.ensure_active() + if claim is None: + return _BucketScan(next_poll_at_ms, conflicted=True) + self._start_callback(lease, claim) + available -= 1 + if available == 0: + break + + if available == 0 or not claimed_from_full_page: + break + return _BucketScan(next_poll_at_ms) + + def _ready_at_ms(self, item: BucketItem, expected_attempt: int) -> int | None: + if item.score == 0: + if expected_attempt != 0: + raise StoreOperationError( + "namespace_corrupt", "Unclaimed expiry item is outside attempt zero" + ) + return None + attempt, started_at_seconds = decode_claim_score(item.score) + if attempt != expected_attempt: + raise StoreOperationError( + "namespace_corrupt", "Expiry item is outside its expected attempt layer" + ) + return (started_at_seconds + self._config.processing_timeout_seconds) * 1000 + + def _start_callback(self, lease: ShardExecution, claim: Claim) -> None: + task = lease.spawn(self._callback_operation(lease, claim)) + self._state.reserved += 1 + self._state.tasks.add(task) + task.add_done_callback(self._state.tasks.discard) + + def _reset_state(self) -> None: + if any(not task.done() for task in self._state.tasks): + raise StoreOperationError("shard_not_stopped", "Previous callbacks are still running") + self._state.tasks.clear() + self._state.reserved = 0 + + def _callback_operation( + self, lease: ShardExecution, claim: Claim + ) -> Callable[[], Awaitable[None]]: + async def operation() -> None: + await self._execute(lease, claim) + + return operation + + async def _execute(self, lease: ShardExecution, claim: Claim) -> None: + try: + try: + lease.ensure_active() + request = await self._resolve_request(claim) + lease.ensure_active() + result = await asyncio.wait_for( + self._binding.handler.handle(request), + timeout=self._config.processing_timeout_seconds, + ) + except asyncio.CancelledError: + raise + except TimeoutError as exc: + self._record_callback_error(claim, exc) + lease.ensure_active() + await self._finish_failed_attempt(lease, claim, "callback_timeout") + return + except StoreOperationError as exc: + if exc.code == "owner_lost" or exc.outcome_unknown: + raise + self._record_callback_error(claim, exc) + lease.ensure_active() + await self._finish_failed_attempt(lease, claim, "callback_exception") + return + except Exception as exc: + self._record_callback_error(claim, exc) + lease.ensure_active() + await self._finish_failed_attempt(lease, claim, "callback_exception") + return + + lease.ensure_active() + if result.status is CallbackStatus.COMPLETED: + await self._store.ack(claim) + lease.ensure_active() + else: + await self._finish_failed_attempt(lease, claim, "callback_retry") + finally: + self._state.reserved = max(0, self._state.reserved - 1) + + async def _finish_failed_attempt( + self, lease: ShardExecution, claim: Claim, reason: str + ) -> None: + attempt, _ = decode_claim_score(claim.claim_score) + if attempt + 1 >= self._binding.options.max_attempts: + await self._log_exhausted_and_ack(lease, claim, reason) + return + lease.ensure_active() + await self._store.advance_retry(claim) + lease.ensure_active() + + async def _log_exhausted_and_ack( + self, lease: ShardExecution, claim: Claim, reason: str + ) -> None: + lease.ensure_active() + self._metrics.count( + "timer_callback_exhausted", + item_type=claim.bucket.item_type, + outcome=reason, + ) + try: + self._metrics.log_exhausted( + claim, + max_attempts=self._binding.options.max_attempts, + reason=reason, + ) + except Exception as exc: + self._record_callback_error(claim, exc) + + try: + lease.ensure_active() + await self._store.ack(claim) + except StoreOperationError as exc: + if exc.code == "owner_lost" or exc.outcome_unknown: + raise + self._record_callback_error(claim, exc) + except Exception as exc: + self._record_callback_error(claim, exc) + + async def _resolve_request(self, claim: Claim) -> CallbackRequest: + identity = decode_item_id(claim.item_id) + if identity.item_type != claim.bucket.item_type: + raise ExpiryError("invalid_identity", "Bucket type differs from item ID") + metadata = await self._store.read_metadata(claim.item_id) + if metadata is not None and metadata.identity != identity: + raise ExpiryError("invalid_identity", "Metadata differs from item ID") + self._metrics.count( + "timer_metadata_lookup", + item_type=identity.item_type, + outcome="miss" if metadata is None else "hit", + ) + return CallbackRequest(identity) + + async def drain(self, timeout_seconds: float) -> None: + tasks = {task for task in self._state.tasks if not task.done()} + if not tasks: + return + _, pending = await asyncio.wait(tasks, timeout=max(0.0, timeout_seconds)) + for task in pending: + task.cancel() + if pending: + raise StoreOperationError("shard_not_stopped", "Callbacks did not drain within budget") + + def _record_callback_error(self, claim: Claim, exc: Exception) -> None: + self._metrics.count( + "timer_callback_error", + item_type=claim.bucket.item_type, + outcome=type(exc).__name__, + ) diff --git a/openviking/scheduling/expiry/keys.py b/openviking/scheduling/expiry/keys.py new file mode 100644 index 0000000000..12157ec59c --- /dev/null +++ b/openviking/scheduling/expiry/keys.py @@ -0,0 +1,122 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Redis key construction and reversible timer identities.""" + +from __future__ import annotations + +import base64 +import binascii +from dataclasses import dataclass + +import xxhash + +from .api import ExpiryError, ItemIdentity +from .models import MINUTE_MS, BucketRef, OwnerRole, validate_identity, validate_item_type + +HASH_SEED = 0 +HASH_NAME = "xxh64" +IDENTITY_ENCODING = "urlsafe_base64" + + +def encode_component(value: str) -> str: + if not isinstance(value, str) or not value: + raise ExpiryError("invalid_identity", "Identity components must be non-empty strings") + return base64.urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=") + + +def decode_component(value: str) -> str: + if not isinstance(value, str) or not value or "=" in value: + raise ExpiryError("invalid_identity", "Invalid encoded identity component") + try: + padded = value + "=" * (-len(value) % 4) + decoded = base64.b64decode(padded, altchars=b"-_", validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError) as exc: + raise ExpiryError("invalid_identity", "Invalid encoded identity component") from exc + if not decoded or encode_component(decoded) != value: + raise ExpiryError("invalid_identity", "Non-canonical encoded identity component") + return decoded + + +def encode_item_id(identity: ItemIdentity) -> str: + validate_identity(identity) + return ":".join( + ( + identity.item_type, + encode_component(identity.account_id), + encode_component(identity.user_id), + encode_component(identity.local_id), + ) + ) + + +def decode_item_id(item_id: str) -> ItemIdentity: + if not isinstance(item_id, str): + raise ExpiryError("invalid_identity", "Invalid expiry item ID") + parts = item_id.split(":") + if len(parts) != 4: + raise ExpiryError("invalid_identity", "Invalid expiry item ID") + validate_item_type(parts[0]) + identity = ItemIdentity(parts[0], *(decode_component(part) for part in parts[1:])) + validate_identity(identity) + return identity + + +def shard_for(item_id: str, shard_count: int) -> int: + if type(shard_count) is not int or shard_count <= 0: + raise ValueError("shard_count must be positive") + return xxhash.xxh64(item_id.encode("utf-8"), seed=HASH_SEED).intdigest() % shard_count + + +@dataclass(frozen=True) +class ExpiryKeys: + service_key: str + + def namespace(self) -> str: + return f"resource_timer:namespace:{self.service_key}" + + def metadata(self, item_id: str) -> str: + return f"resource_meta:item:{self.service_key}:{item_id}" + + def source(self, shard_id: int) -> str: + return f"resource_sche:item_list:{self.service_key}:{shard_id}" + + def control_tag(self, shard_id: int) -> str: + return f"{{{self.service_key}:{shard_id}}}" + + def progress(self, shard_id: int) -> str: + return f"resource_sche:process:consume_progress:{self.control_tag(shard_id)}" + + def registry(self, shard_id: int) -> str: + return f"resource_consume:bucket_register:{self.control_tag(shard_id)}" + + def owner(self, role: OwnerRole, shard_id: int, *, item_type: str | None = None) -> str: + if role is OwnerRole.SCHEDULER: + if item_type is not None: + raise ValueError("Scheduler owner must not include an item type") + return f"resource_timer:owner:{role.value}:{self.control_tag(shard_id)}" + if item_type is None: + raise ValueError("Callback owner requires an item type") + validate_item_type(item_type) + return f"resource_timer:owner:{role.value}:{item_type}:{self.control_tag(shard_id)}" + + def bucket(self, ref: BucketRef) -> str: + return ( + f"resource_consume:item_list:{self.service_key}:" + f"{ref.item_type}:{ref.shard_id}:{ref.minute_ms}" + ) + + def registry_member(self, ref: BucketRef) -> str: + return f"{ref.item_type}:{ref.shard_id}:{ref.minute_ms}" + + def parse_registry_member(self, value: str, *, expected_shard: int) -> BucketRef: + parts = value.split(":") + if len(parts) != 3: + raise ExpiryError("invalid_identity", "Invalid expiry registry member") + validate_item_type(parts[0]) + try: + shard_id, minute_ms = int(parts[1]), int(parts[2]) + except ValueError as exc: + raise ExpiryError("invalid_identity", "Invalid expiry registry member") from exc + if shard_id != expected_shard or minute_ms < 0 or minute_ms % MINUTE_MS: + raise ExpiryError("invalid_identity", "Invalid expiry registry member") + return BucketRef(parts[0], shard_id, minute_ms) diff --git a/openviking/scheduling/expiry/leases.py b/openviking/scheduling/expiry/leases.py new file mode 100644 index 0000000000..e812cc7aea --- /dev/null +++ b/openviking/scheduling/expiry/leases.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Redis token operations for temporary per-shard ownership.""" + +from __future__ import annotations + +import asyncio +import time +import uuid + +from .config import ExpiryConfig +from .metrics import ExpiryMetrics +from .models import OwnerRole, ShardLease, StoreOperationError, task_is_cancelling +from .redis_store import RedisExpiryStore + + +class ShardLeaseCoordinator: + def __init__( + self, + store: RedisExpiryStore, + config: ExpiryConfig, + metrics: ExpiryMetrics, + ) -> None: + self._store = store + self._config = config + self._metrics = metrics + + async def try_acquire( + self, role: OwnerRole, shard_id: int, *, item_type: str | None = None + ) -> ShardLease | None: + if role is OwnerRole.CALLBACK and item_type is None: + raise ValueError("Callback lease requires an item type") + if role is OwnerRole.SCHEDULER and item_type is not None: + raise ValueError("Scheduler lease must not include an item type") + started = time.monotonic() + token = uuid.uuid4().hex + owner = await self._store.try_acquire_owner( + role, + shard_id, + token, + self._config.owner_ttl_seconds, + item_type=item_type, + ) + if owner is None: + return None + lease = ShardLease(owner, asyncio.Event(), started + self._config.owner_ttl_seconds) + lease.ensure_active() + return lease + + async def renew(self, lease: ShardLease) -> bool: + """Never revive lost ownership, including a successful but late response.""" + try: + lease.ensure_active() + except StoreOperationError: + return False + started = time.monotonic() + previous_deadline = lease.valid_until_monotonic + try: + renewed = await self._store.renew_owner(lease.owner, self._config.owner_ttl_seconds) + except asyncio.CancelledError: + # Cancelling a request does not prove Redis did not execute it. + lease.lost.set() + raise + except Exception: + lease.lost.set() + return False + task = asyncio.current_task() + if ( + renewed is None + or lease.lost.is_set() + or task_is_cancelling(task) + or time.monotonic() >= previous_deadline + ): + lease.lost.set() + return False + if lease.revoked.is_set(): + return False + lease.owner = renewed + lease.valid_until_monotonic = started + self._config.owner_ttl_seconds + return True + + async def release(self, lease: ShardLease) -> None: + """Compare the token only after local work/renewal have terminated. + + Local revocation is expected during cleanup and is not ownership loss. + Failed or unknown release leaves the owner to expire; it never deletes a + successor's token. + """ + if time.monotonic() >= lease.valid_until_monotonic: + lease.lost.set() + if lease.lost.is_set(): + return + try: + await self._store.release_owner(lease.owner) + except asyncio.CancelledError: + lease.lost.set() + raise + except Exception as exc: + if isinstance(exc, StoreOperationError) and exc.outcome_unknown: + lease.lost.set() + self._metrics.count("timer_owner_release", outcome="failed") diff --git a/openviking/scheduling/expiry/lua/_common.lua b/openviking/scheduling/expiry/lua/_common.lua new file mode 100644 index 0000000000..69712a358a --- /dev/null +++ b/openviking/scheduling/expiry/lua/_common.lua @@ -0,0 +1,87 @@ +local BASE = 1099511627776 +local MAX_INT = 9007199254740991 + +local function decimal(n) + return string.format('%.0f', n) +end + +local function now_ms() + local t = redis.call('TIME') + return tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000) +end + +local function integer_arg(value, min_value, max_value) + if type(value) ~= 'string' or not string.match(value, '^%-?%d+$') then + error('invalid_integer') + end + local n = tonumber(value) + if not n or n ~= math.floor(n) or n < min_value or n > max_value or decimal(n) ~= value then + error('invalid_integer') + end + return n +end + +local function expect_type(key, allowed) + local actual = redis.call('TYPE', key)['ok'] + if actual ~= 'none' and actual ~= allowed then + error('wrong_key_type') + end +end + +local function require_owner(key, token) + if redis.call('GET', key) ~= token then + error('owner_lost') + end +end + +local function tuple_after(score, member, lower, last_score, last_member) + if score <= lower then return false end + if last_score == nil then return true end + return score > last_score or (score == last_score and member > last_member) +end + +local function page_after(key, lower, upper, last_score, last_member, limit) + local lo = nil + if last_score ~= nil then + local current = redis.call('ZSCORE', key, last_member) + if current and tonumber(current) == last_score and last_score > lower then + local rank = redis.call('ZRANK', key, last_member) + if rank then lo = rank + 1 end + end + end + if lo == nil then + local size = redis.call('ZCARD', key) + lo = 0 + local hi = size + while lo < hi do + local mid = math.floor((lo + hi) / 2) + local entry = redis.call('ZRANGE', key, mid, mid, 'WITHSCORES') + if #entry == 0 then + hi = mid + else + local score = tonumber(entry[2]) + if tuple_after(score, entry[1], lower, last_score, last_member) then + hi = mid + else + lo = mid + 1 + end + end + end + end + local raw = redis.call('ZRANGE', key, lo, lo + limit, 'WITHSCORES') + local items = {} + local count = 0 + for i = 1, #raw, 2 do + local score = tonumber(raw[i + 1]) + if score > upper then break end + count = count + 1 + items[count] = {raw[i], decimal(score)} + end + local next_value = false + if count > limit then + items[count] = nil + count = count - 1 + next_value = {items[count][2], items[count][1]} + end + return items, next_value +end diff --git a/openviking/scheduling/expiry/lua/cas_zset_member.lua b/openviking/scheduling/expiry/lua/cas_zset_member.lua new file mode 100644 index 0000000000..b08318743a --- /dev/null +++ b/openviking/scheduling/expiry/lua/cas_zset_member.lua @@ -0,0 +1,16 @@ +local action = ARGV[1] +local expected = integer_arg(ARGV[3], 0, 256 * BASE - 1) +local replacement = nil +if action == 'set' then + replacement = integer_arg(ARGV[4], 1, 256 * BASE - 1) +elseif action ~= 'delete' or ARGV[4] ~= '' then + return redis.error_reply('invalid_cas_action') +end + +local current = redis.call('ZSCORE', KEYS[1], ARGV[2]) +if not current or tonumber(current) ~= expected then return 0 end +if action == 'set' then + redis.call('ZADD', KEYS[1], decimal(replacement), ARGV[2]) + return 1 +end +return redis.call('ZREM', KEYS[1], ARGV[2]) diff --git a/openviking/scheduling/expiry/lua/cleanup_source_page.lua b/openviking/scheduling/expiry/lua/cleanup_source_page.lua new file mode 100644 index 0000000000..babba0ee09 --- /dev/null +++ b/openviking/scheduling/expiry/lua/cleanup_source_page.lua @@ -0,0 +1,8 @@ +local through = integer_arg(ARGV[1], 0, MAX_INT) +local limit = integer_arg(ARGV[2], 1, 4096) +local due = redis.call('ZCOUNT', KEYS[1], '-inf', decimal(through)) +local removed = math.min(due, limit) +if removed > 0 then + redis.call('ZREMRANGEBYRANK', KEYS[1], 0, removed - 1) +end +return removed diff --git a/openviking/scheduling/expiry/lua/compare_owner.lua b/openviking/scheduling/expiry/lua/compare_owner.lua new file mode 100644 index 0000000000..5164312a52 --- /dev/null +++ b/openviking/scheduling/expiry/lua/compare_owner.lua @@ -0,0 +1,13 @@ +local action = ARGV[1] +local ttl = nil +if action == 'renew' then + ttl = integer_arg(ARGV[3], 1, MAX_INT) +elseif action ~= 'release' or ARGV[3] ~= '' then + return redis.error_reply('invalid_owner_action') +end + +if redis.call('GET', KEYS[1]) ~= ARGV[2] then return 0 end +if action == 'renew' then + return redis.call('PEXPIRE', KEYS[1], ttl) +end +return redis.call('DEL', KEYS[1]) diff --git a/openviking/scheduling/expiry/lua/defer_registration.lua b/openviking/scheduling/expiry/lua/defer_registration.lua new file mode 100644 index 0000000000..c09fdb6430 --- /dev/null +++ b/openviking/scheduling/expiry/lua/defer_registration.lua @@ -0,0 +1,3 @@ +require_owner(KEYS[2], ARGV[1]) +local next_poll_at_ms = integer_arg(ARGV[3], 0, MAX_INT) +return redis.call('ZADD', KEYS[1], 'XX', decimal(next_poll_at_ms), ARGV[2]) diff --git a/openviking/scheduling/expiry/lua/ensure_metadata.lua b/openviking/scheduling/expiry/lua/ensure_metadata.lua new file mode 100644 index 0000000000..d770d14761 --- /dev/null +++ b/openviking/scheduling/expiry/lua/ensure_metadata.lua @@ -0,0 +1,19 @@ +expect_type(KEYS[1], 'hash') +if #ARGV[1] == 0 or #ARGV[2] == 0 or #ARGV[3] == 0 or #ARGV[4] == 0 then + return redis.error_reply('invalid_identity') +end +local ttl = integer_arg(ARGV[5], 1, MAX_INT) +local fields = {'item_type', 'local_id', 'account_id', 'user_id'} +local exists = redis.call('EXISTS', KEYS[1]) == 1 +if exists then + for i = 1, 4 do + local current = redis.call('HGET', KEYS[1], fields[i]) + if current ~= ARGV[i] then return redis.error_reply('metadata_conflict') end + end +else + redis.call('HSET', KEYS[1], + fields[1], ARGV[1], fields[2], ARGV[2], + fields[3], ARGV[3], fields[4], ARGV[4]) +end +redis.call('PEXPIRE', KEYS[1], ttl) +return exists and 'refreshed' or 'created' diff --git a/openviking/scheduling/expiry/lua/publish_minute.lua b/openviking/scheduling/expiry/lua/publish_minute.lua new file mode 100644 index 0000000000..360246a232 --- /dev/null +++ b/openviking/scheduling/expiry/lua/publish_minute.lua @@ -0,0 +1,15 @@ +require_owner(KEYS[3], ARGV[1]) +local expected = integer_arg(ARGV[2], 0, MAX_INT) +local minute = integer_arg(ARGV[3], 0, MAX_INT) +local count = integer_arg(ARGV[4], 1, 1024) +if minute % 60000 ~= 0 or #ARGV ~= 4 + count then return redis.error_reply('invalid_publish') end +local raw_current = redis.call('GET', KEYS[2]) +if not raw_current then return redis.error_reply('namespace_corrupt') end +local current = tonumber(raw_current) +if current >= minute then return {'already_published', decimal(current)} end +if current ~= expected or minute <= current then return redis.error_reply('progress_conflict') end +for i = 1, count do + redis.call('ZADD', KEYS[1], decimal(minute), ARGV[4 + i]) +end +redis.call('SET', KEYS[2], decimal(minute)) +return {'published', decimal(minute)} diff --git a/openviking/scheduling/expiry/lua/read_zset_page.lua b/openviking/scheduling/expiry/lua/read_zset_page.lua new file mode 100644 index 0000000000..8bc105695b --- /dev/null +++ b/openviking/scheduling/expiry/lua/read_zset_page.lua @@ -0,0 +1,13 @@ +local lower = integer_arg(ARGV[1], -1, MAX_INT) +local upper = nil +if ARGV[2] == '' then + upper = now_ms() +else + upper = integer_arg(ARGV[2], 0, MAX_INT) +end +local after_score = integer_arg(ARGV[3], 0, MAX_INT) +local limit = integer_arg(ARGV[5], 1, 4096) +local items, next_value = page_after( + KEYS[1], lower, upper, after_score, ARGV[4], limit +) +return cjson.encode({items=items, next=next_value}) diff --git a/openviking/scheduling/expiry/lua/upsert_resource.lua b/openviking/scheduling/expiry/lua/upsert_resource.lua new file mode 100644 index 0000000000..1bdcac33ea --- /dev/null +++ b/openviking/scheduling/expiry/lua/upsert_resource.lua @@ -0,0 +1,7 @@ +if #ARGV[1] == 0 then return redis.error_reply('invalid_identity') end +local requested = integer_arg(ARGV[2], 0, MAX_INT) +local accepted = math.max(requested, now_ms() + 1) +local max_supported_due = math.floor(MAX_INT / 60000) * 60000 +if accepted > max_supported_due then return redis.error_reply('invalid_report') end +redis.call('ZADD', KEYS[1], decimal(accepted), ARGV[1]) +return decimal(accepted) diff --git a/openviking/scheduling/expiry/metrics.py b/openviking/scheduling/expiry/metrics.py new file mode 100644 index 0000000000..cbbe16df5f --- /dev/null +++ b/openviking/scheduling/expiry/metrics.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Low-cardinality metrics and structured timer logging.""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Mapping + +from .keys import decode_item_id +from .models import Claim + +logger = logging.getLogger(__name__) + + +class ExpiryMetrics: + def __init__(self, sink: Callable[[str, float, Mapping[str, str]], None] | None = None) -> None: + self._sink = sink + + def _emit(self, name: str, value: float, labels: Mapping[str, str]) -> None: + if self._sink is None: + return + try: + self._sink(name, value, labels) + except Exception: + logger.debug("expiry metric sink failed", exc_info=True) + + def count(self, name: str, value: int = 1, *, item_type: str = "", outcome: str = "") -> None: + self._emit(name, float(value), {"item_type": item_type, "outcome": outcome}) + + def observe(self, name: str, value: float, *, item_type: str = "", outcome: str = "") -> None: + self._emit(name, value, {"item_type": item_type, "outcome": outcome}) + + def gauge(self, name: str, value: float, *, item_type: str = "") -> None: + self._emit(name, value, {"item_type": item_type}) + + def log_invalid_source_items( + self, + *, + shard_id: int, + minute_ms: int, + reason: str, + count: int, + ) -> None: + logger.error( + "expiry discarded invalid source items shard_id=%d minute_ms=%d " + "reason=%s count=%d", + shard_id, + minute_ms, + reason, + count, + ) + + def log_exhausted( + self, + claim: Claim, + *, + max_attempts: int, + reason: str, + ) -> None: + identity = decode_item_id(claim.item_id) + logger.error( + "expiry callback exhausted item_type=%s account_id=%s user_id=%s " + "local_id=%s shard_id=%d minute_ms=%d claim_score=%d max_attempts=%d " + "reason=%s", + identity.item_type, + identity.account_id, + identity.user_id, + identity.local_id, + claim.bucket.shard_id, + claim.bucket.minute_ms, + claim.claim_score, + max_attempts, + reason, + ) diff --git a/openviking/scheduling/expiry/models.py b/openviking/scheduling/expiry/models.py new file mode 100644 index 0000000000..c81e56c344 --- /dev/null +++ b/openviking/scheduling/expiry/models.py @@ -0,0 +1,236 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Internal models and pure validation helpers for expiry scheduling.""" + +from __future__ import annotations + +import asyncio +import re +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Awaitable, Callable, Literal, Protocol, TypeVar + +from .api import ExpiryError, ExpiryMode, ExpiryRegistration, ItemIdentity + +T = TypeVar("T") +MINUTE_MS = 60_000 +MIN_METADATA_TTL_MS = 86_400_000 +SCORE_TIME_BASE = 1 << 40 +MAX_ATTEMPT = 255 +MAX_EXACT_INTEGER = (1 << 53) - 1 +_ITEM_TYPE_RE = re.compile(r"[a-z][a-z0-9_]{0,63}\Z") + + +class StoreOperationError(ExpiryError): + def __init__( + self, + code: str, + message: str, + *, + retryable: bool = False, + outcome_unknown: bool = False, + ) -> None: + super().__init__(code, message, retryable=retryable) + self.outcome_unknown = outcome_unknown + + +class OwnerRole(str, Enum): + SCHEDULER = "scheduler" + CALLBACK = "callback" + + +@dataclass(frozen=True) +class ItemMetadata: + identity: ItemIdentity + + +@dataclass(frozen=True) +class NamespaceSpec: + shard_count: int + hash_name: str + hash_seed: int + identity_encoding: str + + +@dataclass(frozen=True) +class OwnerToken: + role: OwnerRole + shard_id: int + token: str + expires_at_ms: int + item_type: str | None = None + + +@dataclass +class ShardLease: + owner: OwnerToken + lost: asyncio.Event + valid_until_monotonic: float + revoked: asyncio.Event = field(default_factory=asyncio.Event) + + def ensure_active(self) -> None: + if self.lost.is_set() or time.monotonic() >= self.valid_until_monotonic: + self.lost.set() + raise StoreOperationError("owner_lost", "Shard lease is no longer valid") + if self.revoked.is_set(): + raise StoreOperationError( + "owner_lost", "Shard lease is no longer valid (execution revoked)" + ) + + +@dataclass(frozen=True) +class ShardWindow: + progress_ms: int + upper_ms: int + redis_now_ms: int + + +@dataclass(frozen=True) +class SourceCursor: + due_at_ms: int + item_id: str + + +@dataclass(frozen=True) +class SourceItem: + item_id: str + due_at_ms: int + + +@dataclass(frozen=True) +class SourcePage: + items: tuple[SourceItem, ...] + next_cursor: SourceCursor | None + + +@dataclass(frozen=True) +class BucketRef: + item_type: str + shard_id: int + minute_ms: int + + +@dataclass(frozen=True) +class RegistryCursor: + score_ms: int + member: str + + +@dataclass(frozen=True) +class RegistryPage: + buckets: tuple[BucketRef, ...] + next_cursor: RegistryCursor | None + + +@dataclass(frozen=True) +class PublishResult: + status: Literal["published", "already_published"] + progress_ms: int + + +@dataclass(frozen=True) +class Claim: + bucket: BucketRef + item_id: str + claim_score: int + + +@dataclass(frozen=True) +class BucketItem: + item_id: str + score: int + + +@dataclass(frozen=True) +class BucketPage: + items: tuple[BucketItem, ...] + + +class AsyncExecutor(Protocol): + async def run(self, operation: Callable[[], Awaitable[T]]) -> T: ... + + +def task_is_cancelling(task: asyncio.Task | None) -> bool: + if task is None: + return False + cancelling = getattr(task, "cancelling", None) + return bool(cancelling()) if callable(cancelling) else task.cancelled() + + +def validate_item_type(item_type: str) -> None: + if not isinstance(item_type, str) or _ITEM_TYPE_RE.fullmatch(item_type) is None: + raise ExpiryError("invalid_identity", "Invalid resource item type") + + +def validate_identity(identity: ItemIdentity) -> None: + if not isinstance(identity, ItemIdentity): + raise ExpiryError("invalid_identity", "Invalid resource identity") + validate_item_type(identity.item_type) + for name in ("account_id", "user_id", "local_id"): + value = getattr(identity, name) + if not isinstance(value, str) or not value or "\x00" in value: + raise ExpiryError("invalid_identity", f"Invalid resource identity field: {name}") + + +def validate_report(task: ExpiryRegistration) -> None: + if not isinstance(task, ExpiryRegistration): + raise ExpiryError("invalid_report", "Invalid resource report") + validate_identity(task.identity) + if task.mode is not ExpiryMode.IDLE_ONESHOT: + raise ExpiryError("unsupported_mode", f"Unsupported timer mode: {task.mode}") + if type(task.due_at_ms) is not int or not 0 <= task.due_at_ms <= MAX_EXACT_INTEGER: + raise ExpiryError("invalid_report", "Due time is outside the supported range") + metadata_ttl_ms(task.timeout_seconds) + bucket_minute_ms(task.due_at_ms) + + +def minute_start_ms(timestamp_ms: int) -> int: + return timestamp_ms // MINUTE_MS * MINUTE_MS + + +def scan_cutoff_ms(now_ms: int) -> int: + return minute_start_ms(now_ms) + + +def bucket_minute_ms(due_at_ms: int) -> int: + if type(due_at_ms) is not int or not 0 <= due_at_ms <= MAX_EXACT_INTEGER: + raise ExpiryError("invalid_report", "Due time is outside the supported range") + minute = ((due_at_ms + MINUTE_MS - 1) // MINUTE_MS) * MINUTE_MS + if minute > MAX_EXACT_INTEGER: + raise ExpiryError("invalid_report", "Due time is outside the supported range") + return minute + + +def metadata_ttl_ms(timeout_seconds: int) -> int: + if type(timeout_seconds) is not int or not 0 < timeout_seconds <= MAX_EXACT_INTEGER // 2000: + raise ExpiryError("invalid_report", "Timeout is outside the supported range") + return max(timeout_seconds * 2000, MIN_METADATA_TTL_MS) + + +def encode_claim_score(attempt: int, started_at_seconds: int) -> int: + if type(attempt) is not int or not 0 <= attempt <= MAX_ATTEMPT: + raise ValueError("attempt out of range") + if type(started_at_seconds) is not int or not 0 < started_at_seconds < SCORE_TIME_BASE: + raise ValueError("claim timestamp out of range") + return attempt * SCORE_TIME_BASE + started_at_seconds + + +def decode_claim_score(score: int) -> tuple[int, int]: + if type(score) is not int or not 0 < score < (MAX_ATTEMPT + 1) * SCORE_TIME_BASE: + raise ValueError("invalid claim score") + attempt, started_at = divmod(score, SCORE_TIME_BASE) + if started_at == 0: + raise ValueError("invalid claim timestamp") + return attempt, started_at + + +def claim_layer_bounds(attempt: int) -> tuple[int, int]: + if type(attempt) is not int or not 0 <= attempt <= MAX_ATTEMPT: + raise ValueError("attempt out of range") + lower = 0 if attempt == 0 else attempt * SCORE_TIME_BASE + 1 + return lower, (attempt + 1) * SCORE_TIME_BASE - 1 + + +def encode_ready_claim_score(attempt: int) -> int: + return encode_claim_score(attempt, 1) diff --git a/openviking/scheduling/expiry/publisher.py b/openviking/scheduling/expiry/publisher.py new file mode 100644 index 0000000000..f491e05e5c --- /dev/null +++ b/openviking/scheduling/expiry/publisher.py @@ -0,0 +1,223 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Expiry publisher: source buckets to published callback buckets.""" + +from __future__ import annotations + +import asyncio +import time +from collections import defaultdict +from typing import Sequence + +from .api import SUPPORTED_ITEM_TYPES, ExpiryError +from .config import ExpiryConfig +from .keys import decode_item_id +from .metrics import ExpiryMetrics +from .models import ( + MINUTE_MS, + BucketRef, + OwnerRole, + PublishResult, + StoreOperationError, + bucket_minute_ms, +) +from .redis_store import RedisExpiryStore +from .shard_runner import ShardExecution, ShardRunner + + +async def _wait_until_next_round(stop: asyncio.Event, delay: float) -> None: + try: + await asyncio.wait_for(stop.wait(), delay) + except TimeoutError: + pass + + +class ExpiryPublisher: + def __init__( + self, + store: RedisExpiryStore, + runner: ShardRunner, + config: ExpiryConfig, + metrics: ExpiryMetrics, + ) -> None: + self._store = store + self._runner = runner + self._config = config + self._metrics = metrics + + async def run(self, stop: asyncio.Event) -> None: + """Own shard traversal and scheduling cadence, not lease lifecycle.""" + while not stop.is_set(): + for shard_id in range(self._config.shard_count): + if stop.is_set(): + break + try: + await self._runner.run_if_acquired( + OwnerRole.SCHEDULER, shard_id, self.process_shard, stop + ) + except StoreOperationError as exc: + if exc.code == "shard_not_stopped": + raise + self._metrics.count("timer_shard_round", outcome="failed") + except Exception: + self._metrics.count("timer_shard_round", outcome="failed") + if not stop.is_set(): + delay = max(0.05, 60.0 - time.time() % 60.0) + await _wait_until_next_round(stop, delay) + + async def process_shard(self, lease: ShardExecution, stop: asyncio.Event) -> None: + while not stop.is_set(): + lease.ensure_active() + window = await self._store.read_scan_window(lease.owner.shard_id) + progress = window.progress_ms + lease.ensure_active() + await self._cleanup_source(lease, stop, progress) + if window.upper_ms <= progress: + return + after_ms = progress + published_any = False + while after_ms < window.upper_ms and not stop.is_set(): + lease.ensure_active() + page = await self._store.read_source_page( + lease.owner.shard_id, + lower_exclusive_ms=after_ms, + upper_inclusive_ms=window.upper_ms, + after=None, + limit=1, + ) + if not page.items: + break + minute = bucket_minute_ms(page.items[0].due_at_ms) + result = await self._process_minute( + lease, + stop, + progress_ms=progress, + minute_ms=minute, + ) + if result is not None: + progress = result + published_any = True + after_ms = minute + if not published_any: + return + + async def _process_minute( + self, + lease: ShardExecution, + stop: asyncio.Event, + *, + progress_ms: int, + minute_ms: int, + ) -> int | None: + lease.ensure_active() + await self._store.reset_unpublished_minute( + lease.owner, + minute_ms=minute_ms, + expected_progress_ms=progress_ms, + item_types=tuple(sorted(SUPPORTED_ITEM_TYPES)), + ) + cursor = None + types_with_data: set[str] = set() + while True: + lease.ensure_active() + page = await self._store.read_source_page( + lease.owner.shard_id, + lower_exclusive_ms=max(progress_ms, minute_ms - MINUTE_MS), + upper_inclusive_ms=minute_ms, + after=cursor, + limit=self._config.scan_batch_size, + ) + by_type: dict[str, list[str]] = defaultdict(list) + invalid_counts: dict[str, int] = defaultdict(int) + for item in page.items: + try: + identity = decode_item_id(item.item_id) + except ExpiryError: + invalid_counts["invalid_identity"] += 1 + continue + if identity.item_type not in SUPPORTED_ITEM_TYPES: + invalid_counts["unsupported_type"] += 1 + continue + by_type[identity.item_type].append(item.item_id) + for reason, count in sorted(invalid_counts.items()): + self._metrics.count( + "timer_source_invalid", + value=count, + outcome=reason, + ) + self._metrics.log_invalid_source_items( + shard_id=lease.owner.shard_id, + minute_ms=minute_ms, + reason=reason, + count=count, + ) + for item_type, item_ids in by_type.items(): + lease.ensure_active() + ref = BucketRef(item_type, lease.owner.shard_id, minute_ms) + await self._store.stage_items(lease.owner, ref, item_ids) + types_with_data.add(item_type) + cursor = page.next_cursor + if cursor is None: + break + if not types_with_data: + # Invalid members can be removed without advancing the published watermark. + # Their metadata lives in other cluster slots and expires by TTL. + await self._cleanup_source(lease, stop, minute_ms) + return None + refs = tuple( + BucketRef(item_type, lease.owner.shard_id, minute_ms) + for item_type in sorted(types_with_data) + ) + result = await self._publish_completed_minute( + lease, + expected_progress_ms=progress_ms, + minute_ms=minute_ms, + buckets=refs, + ) + lease.ensure_active() + # This also removes skipped invalid members; their metadata expires by TTL. + await self._cleanup_source(lease, stop, result.progress_ms) + return result.progress_ms + + async def _cleanup_source( + self, lease: ShardExecution, stop: asyncio.Event, through_ms: int + ) -> None: + limit = self._config.cleanup_batch_size + while not stop.is_set(): + lease.ensure_active() + removed = await self._store.cleanup_source_page( + lease.owner.shard_id, + through_ms, + limit=limit, + ) + if removed < limit: + return + + async def _publish_completed_minute( + self, + lease: ShardExecution, + *, + expected_progress_ms: int, + minute_ms: int, + buckets: Sequence[BucketRef], + ) -> PublishResult: + while True: + lease.ensure_active() + try: + return await self._store.publish_minute( + lease.owner, + expected_progress_ms=expected_progress_ms, + minute_ms=minute_ms, + buckets=buckets, + ) + except StoreOperationError as exc: + if not exc.outcome_unknown: + raise + lease.ensure_active() + progress = await self._store.read_progress(lease.owner.shard_id) + if progress >= minute_ms: + return PublishResult("already_published", progress) + if progress != expected_progress_ms: + raise StoreOperationError( + "progress_conflict", "Timer progress changed after uncertain publish" + ) from exc diff --git a/openviking/scheduling/expiry/redis_store.py b/openviking/scheduling/expiry/redis_store.py new file mode 100644 index 0000000000..80e34b0e3f --- /dev/null +++ b/openviking/scheduling/expiry/redis_store.py @@ -0,0 +1,752 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Redis persistence for distributed expiry scheduling.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import time +from importlib import resources +from typing import Any, Literal, Sequence, cast +from urllib.parse import urlsplit + +from .api import ExpiryRegistration, ExpiryRegistrationResult, ItemIdentity +from .config import ExpiryRedisConfig +from .keys import ExpiryKeys +from .metrics import ExpiryMetrics +from .models import ( + MAX_EXACT_INTEGER, + SCORE_TIME_BASE, + AsyncExecutor, + BucketItem, + BucketPage, + BucketRef, + Claim, + ItemMetadata, + NamespaceSpec, + OwnerRole, + OwnerToken, + PublishResult, + RegistryCursor, + RegistryPage, + ShardWindow, + SourceCursor, + SourceItem, + SourcePage, + StoreOperationError, + claim_layer_bounds, + decode_claim_score, + encode_claim_score, + encode_ready_claim_score, + metadata_ttl_ms, +) + +RetryPolicy = Literal["read", "idempotent", "resolve_by_caller"] + + +class RedisExpiryStore: + def __init__( + self, + config: ExpiryRedisConfig, + keys: ExpiryKeys, + io: AsyncExecutor, + metrics: ExpiryMetrics, + ) -> None: + self._config = config + self._keys = keys + self._io = io + self._metrics = metrics + self._client: Any = None + self._scripts: dict[str, str] = {} + self._script_shas: dict[str, str] = {} + self._ready = False + + async def open(self) -> None: + await self._io.run(self._open_on_loop) + + async def close(self) -> None: + self._ready = False + if self._client is not None: + await self._io.run(self._close_on_loop) + + async def ensure_namespace(self, spec: NamespaceSpec) -> None: + key = self._keys.namespace() + expected = { + "shard_count": str(spec.shard_count), + "hash_name": spec.hash_name, + "hash_seed": str(spec.hash_seed), + "identity_encoding": spec.identity_encoding, + } + current = await self._command("hgetall", key) + initializing = True + if current: + normalized = {str(k): str(v) for k, v in current.items()} + for field, value in expected.items(): + if normalized.get(field) != value: + raise StoreOperationError( + "namespace_mismatch", f"Timer namespace mismatch: {field}" + ) + state = normalized.get("state") + if state not in {"INITIALIZING", "READY"}: + raise StoreOperationError("namespace_corrupt", "Invalid timer namespace state") + initializing = state == "INITIALIZING" + else: + created = dict(expected) + created["state"] = "INITIALIZING" + await self._command("hset", key, mapping=created, retry="idempotent") + for shard_id in range(spec.shard_count): + progress_key = self._keys.progress(shard_id) + if initializing: + await self._command("set", progress_key, "0", nx=True, retry="idempotent") + elif await self._command("get", progress_key) is None: + raise StoreOperationError( + "namespace_corrupt", f"Timer progress is missing for shard {shard_id}" + ) + if initializing: + await self._command("hset", key, "state", "READY", retry="idempotent") + self._ready = True + + async def ensure_metadata(self, item_id: str, task: ExpiryRegistration) -> None: + identity = task.identity + await self._eval( + "ensure_metadata", + [self._keys.metadata(item_id)], + [ + identity.item_type, + identity.local_id, + identity.account_id, + identity.user_id, + metadata_ttl_ms(task.timeout_seconds), + ], + retry="idempotent", + ) + + async def read_metadata(self, item_id: str) -> ItemMetadata | None: + raw = await self._command("hgetall", self._keys.metadata(item_id)) + if not raw: + return None + required = ("item_type", "account_id", "user_id", "local_id") + if any(name not in raw for name in required): + raise StoreOperationError("metadata_corrupt", "Timer metadata is incomplete") + return ItemMetadata(ItemIdentity(*(str(raw[name]) for name in required))) + + async def upsert_resource( + self, shard_id: int, item_id: str, task: ExpiryRegistration + ) -> ExpiryRegistrationResult: + raw = await self._eval( + "upsert_resource", + [self._keys.source(shard_id)], + [item_id, task.due_at_ms], + retry="resolve_by_caller", + ) + return ExpiryRegistrationResult(int(raw)) + + async def unregister_resource(self, shard_id: int, item_id: str) -> None: + await self._command("zrem", self._keys.source(shard_id), item_id, retry="idempotent") + await self._command("delete", self._keys.metadata(item_id), retry="idempotent") + + async def read_progress(self, shard_id: int) -> int: + raw = await self._command("get", self._keys.progress(shard_id)) + if raw is None: + raise StoreOperationError("namespace_corrupt", "Timer progress is missing") + try: + return int(raw) + except (TypeError, ValueError) as exc: + raise StoreOperationError("namespace_corrupt", "Timer progress is invalid") from exc + + async def read_scan_window(self, shard_id: int) -> ShardWindow: + progress = await self.read_progress(shard_id) + source_now_ms = await self._read_time_ms_for_key(self._keys.source(shard_id)) + upper = source_now_ms // 60_000 * 60_000 + if upper < progress: + raise StoreOperationError( + "clock_regression", "Source time is behind published progress" + ) + return ShardWindow(progress, upper, source_now_ms) + + async def read_source_page( + self, + shard_id: int, + *, + lower_exclusive_ms: int, + upper_inclusive_ms: int, + after: SourceCursor | None, + limit: int, + ) -> SourcePage: + items, next_raw = await self._read_zset_page( + self._keys.source(shard_id), + lower_exclusive=lower_exclusive_ms, + upper_inclusive=upper_inclusive_ms, + after_score=None if after is None else after.due_at_ms, + after_member=None if after is None else after.item_id, + limit=limit, + ) + page_items = tuple(SourceItem(item, score) for item, score in items) + cursor = SourceCursor(*next_raw) if next_raw else None + return SourcePage(page_items, cursor) + + async def reset_unpublished_minute( + self, + owner: OwnerToken, + *, + minute_ms: int, + expected_progress_ms: int, + item_types: Sequence[str], + ) -> None: + await self.assert_owner(owner) + current = await self.read_progress(owner.shard_id) + if current != expected_progress_ms or minute_ms <= current: + raise StoreOperationError("progress_conflict", "Timer progress changed") + keys = [ + self._keys.bucket(BucketRef(item_type, owner.shard_id, minute_ms)) + for item_type in item_types + ] + # Bucket keys deliberately have no shared cluster hash tag. Delete + # them one at a time so Redis Cluster never rejects this reset with a + # CROSSSLOT error. + for key in keys: + await self._command("unlink", key, retry="idempotent") + + async def stage_items( + self, owner: OwnerToken, bucket: BucketRef, item_ids: Sequence[str] + ) -> None: + if not item_ids: + return + await self.assert_owner(owner) + mapping = dict.fromkeys(item_ids, 0) + await self._command("zadd", self._keys.bucket(bucket), mapping, retry="resolve_by_caller") + + async def publish_minute( + self, + owner: OwnerToken, + *, + expected_progress_ms: int, + minute_ms: int, + buckets: Sequence[BucketRef], + ) -> PublishResult: + members = [self._keys.registry_member(ref) for ref in buckets] + raw = await self._eval( + "publish_minute", + [ + self._keys.registry(owner.shard_id), + self._keys.progress(owner.shard_id), + self._keys.owner(OwnerRole.SCHEDULER, owner.shard_id), + ], + [owner.token, expected_progress_ms, minute_ms, len(members), *members], + retry="resolve_by_caller", + ) + status = str(raw[0]) + if status not in {"published", "already_published"}: + raise StoreOperationError("namespace_corrupt", "Invalid timer publish result") + return PublishResult( + cast(Literal["published", "already_published"], status), + int(raw[1]), + ) + + async def cleanup_source_page(self, shard_id: int, through_ms: int, *, limit: int) -> int: + return int( + await self._eval( + "cleanup_source_page", + [self._keys.source(shard_id)], + [through_ms, limit], + retry="idempotent", + ) + ) + + async def read_registry_page( + self, shard_id: int, *, after: RegistryCursor | None, limit: int + ) -> RegistryPage: + items, next_raw = await self._read_zset_page( + self._keys.registry(shard_id), + lower_exclusive=-1, + upper_inclusive=None, + after_score=None if after is None else after.score_ms, + after_member=None if after is None else after.member, + limit=limit, + ) + buckets = tuple( + self._keys.parse_registry_member(str(member), expected_shard=shard_id) + for member, _score in items + ) + cursor = RegistryCursor(*next_raw) if next_raw else None + return RegistryPage(buckets, cursor) + + async def read_time_ms(self) -> int: + return self._decode_redis_time(await self._command("time")) + + @staticmethod + def _decode_redis_time(raw: Any) -> int: + try: + seconds, microseconds = (int(value) for value in raw) + except (TypeError, ValueError) as exc: + raise StoreOperationError("namespace_corrupt", "Invalid Redis time") from exc + if not 0 < seconds < SCORE_TIME_BASE or not 0 <= microseconds < 1_000_000: + raise StoreOperationError("namespace_corrupt", "Redis time is outside supported range") + return seconds * 1000 + microseconds // 1000 + + async def _read_time_ms_for_key(self, key: str) -> int: + """Read TIME from the primary owning key so source cutoffs share its clock.""" + + async def operation() -> Any: + if self._client is None: + raise StoreOperationError("not_started", "Redis timer store is closed") + kwargs: dict[str, object] = {} + if self._config.mode == "cluster": + node = self._client.get_node_from_key(key) + if node is None: + raise RuntimeError("Redis Cluster has no primary for the timer key") + kwargs["target_nodes"] = node + return await self._client.time(**kwargs) + + raw = await self._io.run(lambda: self._run_with_retry(operation, retry="read")) + return self._decode_redis_time(raw) + + async def _read_zset_page( + self, + key: str, + *, + lower_exclusive: int, + upper_inclusive: int | None, + after_score: int | None, + after_member: str | None, + limit: int, + ) -> tuple[tuple[tuple[str, int], ...], tuple[int, str] | None]: + if after_score is None: + upper = ( + await self._read_time_ms_for_key(key) + if upper_inclusive is None + else upper_inclusive + ) + raw_items = await self._command( + "zrangebyscore", + key, + f"({lower_exclusive}", + upper, + start=0, + num=limit + 1, + withscores=True, + ) + items = self._decode_zset_items( + raw_items, + lower_exclusive=lower_exclusive, + upper_inclusive=upper, + ) + has_more = len(items) > limit + items = items[:limit] + next_cursor = (items[-1][1], items[-1][0]) if has_more else None + return items, next_cursor + + raw = await self._eval( + "read_zset_page", + [key], + [ + lower_exclusive, + "" if upper_inclusive is None else upper_inclusive, + after_score, + after_member or "", + limit, + ], + retry="read", + ) + data = json.loads(raw) + items = tuple((str(member), int(score)) for member, score in data["items"]) + next_raw = data.get("next") + next_cursor = (int(next_raw[0]), str(next_raw[1])) if next_raw else None + return items, next_cursor + + @staticmethod + def _decode_zset_items( + raw_items: Any, + *, + lower_exclusive: int, + upper_inclusive: int, + ) -> tuple[tuple[str, int], ...]: + items: list[tuple[str, int]] = [] + try: + for member, raw_score in raw_items: + numeric_score = float(raw_score) + score = int(numeric_score) + if numeric_score != score or score <= lower_exclusive or score > upper_inclusive: + raise ValueError + items.append((str(member), score)) + except (TypeError, ValueError) as exc: + raise StoreOperationError( + "namespace_corrupt", "Timer sorted set contains an invalid score" + ) from exc + return tuple(items) + + async def read_bucket_page( + self, + bucket: BucketRef, + *, + attempt: int, + limit: int, + ) -> BucketPage: + lower, upper = claim_layer_bounds(attempt) + raw = await self._command( + "zrangebyscore", + self._keys.bucket(bucket), + lower, + upper, + start=0, + num=limit, + withscores=True, + ) + items: list[BucketItem] = [] + try: + for item_id, raw_score in raw: + numeric_score = float(raw_score) + score = int(numeric_score) + if numeric_score != score or not lower <= score <= upper: + raise ValueError + items.append(BucketItem(str(item_id), score)) + except (TypeError, ValueError) as exc: + raise StoreOperationError( + "namespace_corrupt", "Timer bucket contains an invalid score" + ) from exc + return BucketPage(tuple(items)) + + async def defer_registration( + self, + owner: OwnerToken, + bucket: BucketRef, + *, + next_poll_at_ms: int, + ) -> bool: + if ( + owner.role is not OwnerRole.CALLBACK + or owner.item_type != bucket.item_type + or owner.shard_id != bucket.shard_id + ): + raise StoreOperationError( + "invalid_owner", "Callback owner does not match the timer bucket" + ) + raw = await self._eval( + "defer_registration", + [ + self._keys.registry(bucket.shard_id), + self._keys.owner( + owner.role, + owner.shard_id, + item_type=owner.item_type, + ), + ], + [owner.token, self._keys.registry_member(bucket), next_poll_at_ms], + retry="idempotent", + ) + return bool(int(raw)) + + async def claim( + self, + bucket: BucketRef, + item: BucketItem, + *, + attempt: int, + started_at_seconds: int, + ) -> Claim | None: + if item.score == 0: + if attempt != 0: + raise StoreOperationError( + "namespace_corrupt", "Unclaimed expiry item is outside attempt zero" + ) + else: + item_attempt, _ = decode_claim_score(item.score) + if item_attempt != attempt: + raise StoreOperationError( + "namespace_corrupt", "Expiry item is outside its expected attempt layer" + ) + claim_score = encode_claim_score(attempt, started_at_seconds) + raw = await self._eval( + "cas_zset_member", + [self._keys.bucket(bucket)], + ["set", item.item_id, item.score, claim_score], + retry="resolve_by_caller", + ) + return Claim(bucket, item.item_id, claim_score) if bool(int(raw)) else None + + async def advance_retry(self, claim: Claim) -> bool: + attempt, _ = decode_claim_score(claim.claim_score) + retry_score = encode_ready_claim_score(attempt + 1) + raw = await self._eval( + "cas_zset_member", + [self._keys.bucket(claim.bucket)], + ["set", claim.item_id, claim.claim_score, retry_score], + retry="resolve_by_caller", + ) + return bool(int(raw)) + + async def ack(self, claim: Claim) -> bool: + raw = await self._eval( + "cas_zset_member", + [self._keys.bucket(claim.bucket)], + ["delete", claim.item_id, claim.claim_score, ""], + retry="idempotent", + ) + return bool(int(raw)) + + async def remove_registration_if_empty(self, bucket: BucketRef) -> bool: + if int(await self._command("zcard", self._keys.bucket(bucket))) != 0: + return False + await self._command( + "zrem", + self._keys.registry(bucket.shard_id), + self._keys.registry_member(bucket), + retry="idempotent", + ) + return True + + async def try_acquire_owner( + self, + role: OwnerRole, + shard_id: int, + token: str, + ttl_seconds: int, + *, + item_type: str | None = None, + ) -> OwnerToken | None: + ttl_ms = self._owner_ttl_ms(token, ttl_seconds) + # Wall-clock expiry is observational, so derive it from the conservative + # request start instead of adding TTL after the response arrives. + started_at_ms = time.time_ns() // 1_000_000 + owned = await self._command( + "set", + self._keys.owner(role, shard_id, item_type=item_type), + token, + nx=True, + px=ttl_ms, + retry="resolve_by_caller", + ) + if not owned: + return None + return OwnerToken(role, shard_id, token, started_at_ms + ttl_ms, item_type) + + async def renew_owner(self, owner: OwnerToken, ttl_seconds: int) -> OwnerToken | None: + ttl_ms = self._owner_ttl_ms(owner.token, ttl_seconds) + started_at_ms = time.time_ns() // 1_000_000 + raw = int( + await self._eval( + "compare_owner", + [ + self._keys.owner( + owner.role, + owner.shard_id, + item_type=owner.item_type, + ) + ], + ["renew", owner.token, ttl_ms], + retry="resolve_by_caller", + ) + ) + return ( + None + if raw == 0 + else OwnerToken( + owner.role, + owner.shard_id, + owner.token, + started_at_ms + ttl_ms, + owner.item_type, + ) + ) + + @staticmethod + def _owner_ttl_ms(token: str, ttl_seconds: int) -> int: + if not token: + raise StoreOperationError("invalid_owner", "Timer owner token is empty") + if type(ttl_seconds) is not int or not 0 < ttl_seconds <= MAX_EXACT_INTEGER // 1000: + raise StoreOperationError("invalid_integer", "Timer owner TTL is invalid") + return ttl_seconds * 1000 + + async def assert_owner(self, owner: OwnerToken) -> None: + raw = await self._command( + "get", + self._keys.owner( + owner.role, + owner.shard_id, + item_type=owner.item_type, + ), + ) + if raw != owner.token: + raise StoreOperationError("owner_lost", "Shard owner changed") + + async def release_owner(self, owner: OwnerToken) -> bool: + raw = await self._eval( + "compare_owner", + [ + self._keys.owner( + owner.role, + owner.shard_id, + item_type=owner.item_type, + ) + ], + ["release", owner.token, ""], + retry="idempotent", + ) + return bool(int(raw)) + + async def _open_on_loop(self) -> None: + try: + import redis.asyncio as redis_async + except ImportError as exc: + raise StoreOperationError( + "dependency_missing", + "Redis expiry scheduling requires the 'expiry' optional dependency", + ) from exc + endpoint = self._config.endpoints[0] + if "://" not in endpoint: + endpoint = ("rediss" if self._config.tls else "redis") + "://" + endpoint + password = self._config.password + if password is None and self._config.password_env: + password = os.environ.get(self._config.password_env) + kwargs = { + "username": self._config.username, + "password": password, + "decode_responses": True, + "socket_connect_timeout": self._config.connect_timeout_seconds, + "socket_timeout": self._config.socket_timeout_seconds, + "max_connections": self._config.max_connections, + } + if self._config.mode == "standalone": + self._client = redis_async.Redis.from_url(endpoint, **kwargs) + else: + from redis.asyncio.cluster import ClusterNode, RedisCluster + + nodes = [] + use_tls = self._config.tls + for raw in self._config.endpoints: + normalized = ( + raw + if "://" in raw + else (("rediss" if self._config.tls else "redis") + f"://{raw}") + ) + parsed = urlsplit(normalized) + use_tls = parsed.scheme == "rediss" + nodes.append(ClusterNode(parsed.hostname or "", parsed.port or 6379)) + self._client = RedisCluster(startup_nodes=nodes, ssl=use_tls, **kwargs) + try: + await self._client.ping() + except Exception as exc: + await self._close_on_loop() + raise self._translate_redis_error(exc, retry="read") from exc + + async def _close_on_loop(self) -> None: + client, self._client = self._client, None + if client is not None: + close = getattr(client, "aclose", None) or getattr(client, "close", None) + if close is not None: + result = close() + if result is not None: + await result + + async def _command( + self, name: str, *args: object, retry: RetryPolicy = "read", **kwargs: Any + ) -> Any: + async def operation() -> Any: + if self._client is None: + raise StoreOperationError("not_started", "Redis timer store is closed") + return await self._run_with_retry( + lambda: getattr(self._client, name)(*args, **kwargs), + retry=retry, + ) + + return await self._io.run(operation) + + async def _eval( + self, + script: str, + keys: Sequence[str], + args: Sequence[object], + *, + retry: RetryPolicy, + ) -> Any: + return await self._io.run(lambda: self._eval_on_loop(script, keys, args, retry=retry)) + + async def _eval_on_loop( + self, + script: str, + keys: Sequence[str], + args: Sequence[object], + *, + retry: RetryPolicy, + ) -> Any: + if self._client is None: + raise StoreOperationError("not_started", "Redis timer store is closed") + source = self._read_script(script) + sha = self._script_shas[script] + try: + return await self._run_with_retry( + lambda: self._client.evalsha(sha, len(keys), *keys, *args), + retry=retry, + ) + except StoreOperationError as exc: + if exc.code != "noscript": + raise + return await self._run_with_retry( + lambda: self._client.eval(source, len(keys), *keys, *args), + retry=retry, + ) + + async def _run_with_retry(self, operation: Any, *, retry: RetryPolicy) -> Any: + """Retry safe Redis operations inside the executor's total time budget.""" + delay = 0.01 + while True: + try: + return await operation() + except StoreOperationError: + raise + except Exception as exc: + translated = self._translate_redis_error(exc, retry=retry) + if retry == "resolve_by_caller" or not translated.retryable: + raise translated from exc + await asyncio.sleep(delay) + delay = min(delay * 2, 0.2) + + def _read_script(self, name: str) -> str: + if name not in self._scripts: + base = resources.files(__package__).joinpath("lua") + common = base.joinpath("_common.lua").read_text(encoding="utf-8") + body = base.joinpath(f"{name}.lua").read_text(encoding="utf-8") + self._scripts[name] = common + "\n" + body + self._script_shas[name] = hashlib.sha1( + self._scripts[name].encode("utf-8"), usedforsecurity=False + ).hexdigest() + return self._scripts[name] + + def _translate_redis_error(self, exc: Exception, *, retry: RetryPolicy) -> StoreOperationError: + message = str(exc) + normalized = message.lower() + known = ( + "noscript", + "invalid_identity", + "invalid_report", + "invalid_integer", + "invalid_owner", + "invalid_owner_action", + "invalid_claim_time", + "invalid_cas_action", + "invalid_publish", + "metadata_conflict", + "owner_lost", + "progress_conflict", + "namespace_corrupt", + "wrong_key_type", + ) + code = ( + "noscript" + if type(exc).__name__ == "NoScriptError" + else ( + "wrong_key_type" + if "wrongtype" in normalized + else next((item for item in known if item in normalized), "redis_error") + ) + ) + unknown = retry == "resolve_by_caller" and code == "redis_error" + return StoreOperationError( + code, + message or type(exc).__name__, + retryable=code == "redis_error", + outcome_unknown=unknown, + ) diff --git a/openviking/scheduling/expiry/registry.py b/openviking/scheduling/expiry/registry.py new file mode 100644 index 0000000000..23af7e965f --- /dev/null +++ b/openviking/scheduling/expiry/registry.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Private resource handler registry.""" + +from __future__ import annotations + +import inspect +from dataclasses import dataclass +from types import MappingProxyType +from typing import Mapping + +from .api import SUPPORTED_ITEM_TYPES, ExpiryError, ExpiryHandler, HandlerOptions +from .models import validate_item_type + + +@dataclass(frozen=True) +class HandlerBinding: + handler: ExpiryHandler + options: HandlerOptions + + +class HandlerRegistry: + def __init__(self) -> None: + self._bindings: dict[str, HandlerBinding] = {} + self._frozen = False + + def register(self, item_type: str, handler: ExpiryHandler, options: HandlerOptions) -> None: + if self._frozen: + raise ExpiryError("registration_closed", "Timer handler registration is closed") + validate_item_type(item_type) + if item_type not in SUPPORTED_ITEM_TYPES: + raise ExpiryError("unregistered_type", f"Timer type is not supported: {item_type}") + handle = getattr(handler, "handle", None) + if not callable(handle) or not inspect.iscoroutinefunction(handle): + raise ExpiryError("invalid_handler", "Timer handler.handle must be async") + if not isinstance(options, HandlerOptions): + raise ExpiryError("invalid_handler", "Invalid timer handler options") + if type(options.concurrency) is not int or options.concurrency <= 0: + raise ExpiryError("invalid_handler", "Handler concurrency must be positive") + if type(options.max_attempts) is not int or not 1 <= options.max_attempts <= 256: + raise ExpiryError("invalid_handler", "Handler max_attempts must be in [1, 256]") + if item_type in self._bindings: + raise ExpiryError("duplicate_handler", f"Handler already registered: {item_type}") + self._bindings[item_type] = HandlerBinding(handler, options) + + def freeze(self) -> Mapping[str, HandlerBinding]: + self._frozen = True + return MappingProxyType(dict(self._bindings)) diff --git a/openviking/scheduling/expiry/runtime.py b/openviking/scheduling/expiry/runtime.py new file mode 100644 index 0000000000..2ecd5cbd11 --- /dev/null +++ b/openviking/scheduling/expiry/runtime.py @@ -0,0 +1,398 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Expiry lifecycle and dedicated Redis event loop.""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import threading +import time +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable, Coroutine, TypeVar + +from .api import ExpiryScheduler +from .config import ExpiryConfig +from .consumer import ExpiryConsumer +from .keys import HASH_NAME, HASH_SEED, IDENTITY_ENCODING, ExpiryKeys, encode_component +from .leases import ShardLeaseCoordinator +from .metrics import ExpiryMetrics +from .models import NamespaceSpec, StoreOperationError +from .publisher import ExpiryPublisher +from .redis_store import RedisExpiryStore +from .registry import HandlerRegistry +from .service import ExpiryRuntimeState, _ExpirySchedulerFacade +from .shard_runner import ShardRunner + +T = TypeVar("T") + + +@dataclass(eq=False) +class _RedisRequest: + # Unlike run_coroutine_threadsafe's Future, this future is completed only + # when the Redis-loop task AND its operation have actually terminated. + finished: concurrent.futures.Future = field(default_factory=concurrent.futures.Future) + cancelled: threading.Event = field(default_factory=threading.Event) + task: asyncio.Task | None = None # Read and written only on the Redis loop. + + +class RedisLoopExecutor: + def __init__(self, *, max_inflight: int, request_budget_seconds: float) -> None: + self._max_inflight = max_inflight + self._request_budget_seconds = request_budget_seconds + self._loop: asyncio.AbstractEventLoop | None = None + self._thread: threading.Thread | None = None + self._gate: asyncio.Semaphore | None = None + self._accepting = False + self._ready = threading.Event() + self._requests: set[_RedisRequest] = set() + self._requests_lock = threading.Lock() + self._stop_requested = False + + async def start(self) -> None: + if self._accepting: + return + if self._thread is not None: + raise self._not_stopped() + self._ready.clear() + self._stop_requested = False + self._thread = threading.Thread( + target=self._thread_main, name="openviking-redis-expiry", daemon=True + ) + self._thread.start() + ready = await asyncio.to_thread(self._ready.wait, 5.0) + if not ready or self._loop is None: + raise RuntimeError("Expiry Redis loop failed to start") + self._accepting = True + + async def run(self, operation: Callable[[], Awaitable[T]]) -> T: + loop = self._loop + # Nested store.close calls during run_cleanup are already covered by + # their tracked parent request, and must be allowed after quiescing. + if loop is not None and asyncio.get_running_loop() is loop: + return await self._execute(operation) + if not self._accepting or loop is None: + raise StoreOperationError("not_started", "Redis expiry I/O loop is not running") + request = self._submit(operation) + finished = asyncio.wrap_future(request.finished) + try: + return await asyncio.shield(finished) + except asyncio.CancelledError: + self._cancel(request) + # Keep the caller visible to ShardRunner until the actual operation + # exits. Repeated cancellation must not sever this lifetime bridge. + await self._join(finished) + raise + + async def quiesce(self, timeout_seconds: float) -> None: + """Reject new work and observe actual remote termination before closing Redis.""" + self._accepting = False + with self._requests_lock: + requests = list(self._requests) + if requests: + futures = {asyncio.wrap_future(request.finished) for request in requests} + done, pending = await asyncio.wait(futures, timeout=max(0.0, timeout_seconds)) + for future in done: + if not future.cancelled(): + future.exception() + # A caller might have already stopped awaiting a failed request. + for future in pending: + future.add_done_callback(self._consume_exception) + if pending: + raise self._not_stopped() + + async def run_cleanup(self, operation: Callable[[], Awaitable[T]], timeout_seconds: float) -> T: + """Run storage cleanup on its own loop without reopening public admission.""" + request = self._submit(operation, cleanup=True) + finished = asyncio.wrap_future(request.finished) + try: + done, _ = await asyncio.wait({finished}, timeout=max(0.0, timeout_seconds)) + except BaseException: + self._cancel(request) + finished.add_done_callback(self._consume_exception) + raise + if finished not in done: + self._cancel(request) + finished.add_done_callback(self._consume_exception) + raise self._not_stopped() + return finished.result() + + async def close(self, timeout_seconds: float) -> None: + deadline = time.monotonic() + max(0.0, timeout_seconds) + await self.quiesce(timeout_seconds) + loop, thread = self._loop, self._thread + if loop is not None and not self._stop_requested: + self._stop_requested = True + loop.call_soon_threadsafe(loop.stop) + if thread is not None: + await asyncio.to_thread(thread.join, max(0.0, deadline - time.monotonic())) + if thread.is_alive(): + # Retain both references so callers can retry, and cannot start + # another I/O thread while the old one still owns dependencies. + raise self._not_stopped() + self._loop = None + self._thread = None + self._gate = None + + def _submit( + self, operation: Callable[[], Awaitable[T]], *, cleanup: bool = False + ) -> _RedisRequest: + loop = self._loop + if loop is None or self._stop_requested: + raise StoreOperationError("not_started", "Redis expiry I/O loop is not running") + request = _RedisRequest() + # Register before scheduling: shutdown must also see not-yet-started I/O. + with self._requests_lock: + self._requests.add(request) + + async def cleanup_operation() -> T: + return await operation() + + def start_request() -> None: + request.task = asyncio.create_task( + cleanup_operation() if cleanup else self._execute(operation) + ) + request.task.add_done_callback(lambda task: self._request_done(request, task)) + if request.cancelled.is_set(): + request.task.cancel() + + try: + loop.call_soon_threadsafe(start_request) + except BaseException: + with self._requests_lock: + self._requests.discard(request) + raise + return request + + def _cancel(self, request: _RedisRequest) -> None: + if request.cancelled.is_set(): + return + request.cancelled.set() + + def cancel_request() -> None: + if request.task is not None and not request.task.done(): + request.task.cancel() + + loop = self._loop + if loop is not None: + loop.call_soon_threadsafe(cancel_request) + + def _request_done(self, request: _RedisRequest, task: asyncio.Task) -> None: + if task.cancelled(): + request.finished.set_exception(asyncio.CancelledError()) + else: + error = task.exception() + if error is not None: + request.finished.set_exception(error) + else: + request.finished.set_result(task.result()) + with self._requests_lock: + self._requests.discard(request) + + @staticmethod + def _consume_exception(future: asyncio.Future) -> None: + if not future.cancelled(): + future.exception() + + @staticmethod + async def _join(future: asyncio.Future) -> None: + while not future.done(): + try: + await asyncio.shield(future) + except (asyncio.CancelledError, Exception): + pass + if not future.cancelled(): + future.exception() + + @staticmethod + def _not_stopped() -> StoreOperationError: + return StoreOperationError( + "shard_not_stopped", "Redis expiry work or its I/O thread has not terminated" + ) + + def _thread_main(self) -> None: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self._loop = loop + self._gate = asyncio.Semaphore(self._max_inflight) + self._ready.set() + try: + loop.run_forever() + finally: + pending = asyncio.all_tasks(loop) + for task in pending: + task.cancel() + if pending: + loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + async def _execute(self, operation: Callable[[], Awaitable[T]]) -> T: + gate = self._gate + if gate is None: + raise StoreOperationError("not_started", "Redis expiry I/O loop is not ready") + + async def guarded() -> T: + async with gate: + return await operation() + + task = asyncio.create_task(guarded()) + try: + done, _ = await asyncio.wait({task}, timeout=self._request_budget_seconds) + if task not in done: + task.cancel() + await self._join(task) + raise StoreOperationError( + "redis_timeout", + "Redis expiry request exceeded its budget", + retryable=True, + outcome_unknown=True, + ) + return task.result() + finally: + if not task.done(): + task.cancel() + await self._join(task) + if not task.cancelled(): + task.exception() + + +class ExpiryRuntime: + def __init__( + self, + config: ExpiryConfig, + *, + metrics: ExpiryMetrics | None = None, + ) -> None: + self._config = config + self._metrics = metrics or ExpiryMetrics() + self._keys = ExpiryKeys(encode_component(config.service_name)) + self._io = RedisLoopExecutor( + max_inflight=config.redis.max_inflight_requests, + request_budget_seconds=config.redis.request_budget_seconds, + ) + self._store = RedisExpiryStore(config.redis, self._keys, self._io, self._metrics) + self._registry = HandlerRegistry() + self._state = ExpiryRuntimeState() + self._facade = _ExpirySchedulerFacade( + self._store, + self._registry, + self._state, + shard_count=config.shard_count, + reporting_enabled=config.reporting_enabled, + metrics=self._metrics, + ) + self._leases = ShardLeaseCoordinator(self._store, config, self._metrics) + self._shard_runner = ShardRunner(self._leases, config, self._metrics) + self._scheduler: ExpiryPublisher | None = None + self._consumers: dict[str, ExpiryConsumer] = {} + self._tasks: dict[str, asyncio.Task[None]] = {} + self._stop = asyncio.Event() + self._storage_started = False + self._storage_closed = False + + @property + def timer(self) -> ExpiryScheduler: + return self._facade + + async def start_storage(self) -> None: + if self._storage_started: + return + await self._io.start() + try: + await self._store.open() + await self._store.ensure_namespace( + NamespaceSpec( + self._config.shard_count, + HASH_NAME, + HASH_SEED, + IDENTITY_ENCODING, + ) + ) + except Exception: + try: + await self._store.close() + finally: + await self._io.close(self._config.shutdown_timeout_seconds) + raise + self._storage_started = True + self._storage_closed = False + self._state.storage_ready = True + + async def start_roles(self) -> None: + if not self._storage_started: + raise RuntimeError("Expiry storage is not started") + if self._state.accepting_calls: + return + bindings = self._registry.freeze() + self._stop.clear() + if self._config.scheduler_enabled: + self._scheduler = ExpiryPublisher( + self._store, self._shard_runner, self._config, self._metrics + ) + self._spawn_role("scheduler", self._scheduler.run(self._stop)) + if self._config.callback_enabled: + for item_type, binding in bindings.items(): + consumer = ExpiryConsumer( + self._store, + self._shard_runner, + item_type, + binding, + self._config, + self._metrics, + ) + self._consumers[item_type] = consumer + self._spawn_role( + f"callback:{item_type}", + consumer.run(self._stop), + ) + self._state.accepting_calls = True + + async def wait(self) -> None: + if not self._tasks: + await self._stop.wait() + return + done, _ = await asyncio.wait(self._tasks.values(), return_when=asyncio.FIRST_COMPLETED) + for task in done: + await task + + async def stop_roles(self) -> None: + self._state.accepting_calls = False + self._stop.set() + tasks = set(self._tasks.values()) + if tasks: + _, pending = await asyncio.wait(tasks, timeout=self._config.shutdown_timeout_seconds) + for task in pending: + task.cancel() + if pending: + await asyncio.wait(pending, timeout=self._config.redis.request_budget_seconds) + for name, task in list(self._tasks.items()): + if task.done(): + if not task.cancelled(): + task.exception() + self._tasks.pop(name, None) + settled = await self._shard_runner.wait_pending(self._config.redis.request_budget_seconds) + if self._tasks or not settled: + raise StoreOperationError( + "shard_not_stopped", "Expiry work is still running; dependencies remain open" + ) + + async def close(self) -> None: + await self.stop_roles() + if not self._storage_started: + return + # Stop admission and check real Redis-loop tasks, not just their + # application-loop wrappers, before closing any dependency. + await self._io.quiesce(self._config.shutdown_timeout_seconds) + try: + if not self._storage_closed: + await self._io.run_cleanup(self._store.close, self._config.shutdown_timeout_seconds) + self._storage_closed = True + finally: + await self._io.close(self._config.shutdown_timeout_seconds) + self._storage_started = False + self._state.storage_ready = False + + def _spawn_role(self, name: str, operation: Coroutine[Any, Any, None]) -> None: + task: asyncio.Task[None] = asyncio.create_task(operation, name=f"expiry-{name}") + self._tasks[name] = task diff --git a/openviking/scheduling/expiry/service.py b/openviking/scheduling/expiry/service.py new file mode 100644 index 0000000000..6bff8cf626 --- /dev/null +++ b/openviking/scheduling/expiry/service.py @@ -0,0 +1,97 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Private facade implementing the public ExpiryScheduler protocol.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .api import ( + SUPPORTED_ITEM_TYPES, + ExpiryError, + ExpiryHandler, + ExpiryRegistration, + ExpiryRegistrationResult, + HandlerOptions, + ItemIdentity, +) +from .keys import encode_item_id, shard_for +from .metrics import ExpiryMetrics +from .models import StoreOperationError, validate_identity, validate_report +from .redis_store import RedisExpiryStore +from .registry import HandlerRegistry + + +@dataclass +class ExpiryRuntimeState: + storage_ready: bool = False + accepting_calls: bool = False + + +class _ExpirySchedulerFacade: + def __init__( + self, + store: RedisExpiryStore, + registry: HandlerRegistry, + state: ExpiryRuntimeState, + *, + shard_count: int, + reporting_enabled: bool, + metrics: ExpiryMetrics, + ) -> None: + self._store = store + self._registry = registry + self._state = state + self._metrics = metrics + self._shard_count = shard_count + self._reporting_enabled = reporting_enabled + + async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: + self._require_ready() + if not self._reporting_enabled: + raise ExpiryError("reporting_disabled", "Timer reporting is disabled") + validate_report(task) + self._ensure_supported_type(task.identity.item_type) + try: + item_id = encode_item_id(task.identity) + shard_id = shard_for(item_id, self._shard_count) + result = await self._store.upsert_resource(shard_id, item_id, task) + await self._store.ensure_metadata(item_id, task) + except StoreOperationError as exc: + raise self._business_error(exc) from exc + self._metrics.count("timer_report", item_type=task.identity.item_type, outcome="written") + return result + + async def unregister(self, identity: ItemIdentity) -> None: + self._require_ready() + validate_identity(identity) + self._ensure_supported_type(identity.item_type) + item_id = encode_item_id(identity) + try: + await self._store.unregister_resource(shard_for(item_id, self._shard_count), item_id) + except StoreOperationError as exc: + raise self._business_error(exc) from exc + + def register_handler( + self, + item_type: str, + handler: ExpiryHandler, + *, + options: HandlerOptions | None = None, + ) -> None: + self._registry.register(item_type, handler, options or HandlerOptions()) + + @staticmethod + def _ensure_supported_type(item_type: str) -> None: + if item_type not in SUPPORTED_ITEM_TYPES: + raise ExpiryError("unregistered_type", f"Timer type is not supported: {item_type}") + + def _require_ready(self) -> None: + if not (self._state.storage_ready and self._state.accepting_calls): + raise ExpiryError("not_started", "Expiry scheduling is not ready") + + def _business_error(self, exc: StoreOperationError) -> ExpiryError: + public_codes = {"invalid_identity", "invalid_report", "unregistered_type"} + code = exc.code if exc.code in public_codes else "timer_unavailable" + self._metrics.count("timer_store_error", outcome=exc.code) + return ExpiryError(code, str(exc), retryable=exc.retryable or code == "timer_unavailable") diff --git a/openviking/scheduling/expiry/shard_runner.py b/openviking/scheduling/expiry/shard_runner.py new file mode 100644 index 0000000000..05ca3d6777 --- /dev/null +++ b/openviking/scheduling/expiry/shard_runner.py @@ -0,0 +1,353 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Bounded execution of business work under a temporary shard lease.""" + +from __future__ import annotations + +import asyncio +import time +from typing import Awaitable, Callable + +from .config import ExpiryConfig +from .leases import ShardLeaseCoordinator +from .metrics import ExpiryMetrics +from .models import ( + OwnerRole, + OwnerToken, + ShardLease, + StoreOperationError, + task_is_cancelling, +) + +ShardProcessor = Callable[["ShardExecution", asyncio.Event], Awaitable[None]] +RunnerScope = tuple[OwnerRole, str | None] + + +class ShardExecution: + """A revocable checkpoint and the strong owner of a processor's children. + + Cancellation only stops local work. It does not prove that an external + request or callback has stopped, so cancelled work never authorizes release. + """ + + def __init__(self, lease: ShardLease) -> None: + self._lease = lease + self._children: set[asyncio.Task[None]] = set() + self._waiters: set[asyncio.Task[object]] = set() + self._error: BaseException | None = None + self._outcome_unknown = False + + @property + def owner(self) -> OwnerToken: + return self._lease.owner + + @property + def shard_id(self) -> int: + return self.owner.shard_id + + def ensure_active(self) -> None: + task = asyncio.current_task() + if task_is_cancelling(task): + # A handler may swallow child cancellation and return normally. Mark + # uncertainty before its next store operation, not in a late callback. + self._outcome_unknown = True + raise asyncio.CancelledError + self._lease.ensure_active() + + def spawn(self, operation: Callable[[], Awaitable[None]]) -> asyncio.Task[None]: + self.ensure_active() + + async def guarded() -> None: + self.ensure_active() + await operation() + + task = asyncio.create_task(guarded()) + self._children.add(task) + task.add_done_callback(self._child_done) + return task + + async def wait(self, delay_seconds: float, stop: asyncio.Event) -> None: + """Wait for a poll, stop, or invalidation, checking both sides of the wait.""" + self.ensure_active() + deadline = time.monotonic() + max(0.0, delay_seconds) + waiters = { + asyncio.create_task(stop.wait()), + asyncio.create_task(self._lease.lost.wait()), + asyncio.create_task(self._lease.revoked.wait()), + } + self._waiters.update(waiters) + for task in waiters: + task.add_done_callback(self._waiter_done) + try: + while True: + self.ensure_active() + now = time.monotonic() + if stop.is_set() or now >= deadline: + return + await asyncio.wait( + waiters, + timeout=max(0.0, min(deadline, self._lease.valid_until_monotonic) - now), + return_when=asyncio.FIRST_COMPLETED, + ) + self.ensure_active() + finally: + for task in waiters: + task.cancel() + + def _waiter_done(self, task: asyncio.Task[object]) -> None: + self._waiters.discard(task) + _consume_exception(task) + + def _child_done(self, task: asyncio.Task[None]) -> None: + self._children.discard(task) + if task.cancelled() or task_is_cancelling(task): + self._outcome_unknown = True + if task.cancelled(): + return + error = task.exception() + if error is not None: + self._error = self._error or error + if isinstance(error, StoreOperationError) and error.outcome_unknown: + self._outcome_unknown = True + + def _revoke(self) -> None: + self._lease.revoked.set() + + +def _consume_exception(task: asyncio.Task[object]) -> None: + if not task.cancelled(): + task.exception() + + +class ShardRunner: + """Run one shard at a time per role, retaining all work that cannot stop.""" + + def __init__( + self, + leases: ShardLeaseCoordinator, + config: ExpiryConfig, + metrics: ExpiryMetrics, + ) -> None: + self._leases = leases + self._config = config + self._metrics = metrics + self._active: set[RunnerScope] = set() + self._pending: dict[RunnerScope, set[asyncio.Task[object]]] = {} + + @property + def has_pending(self) -> bool: + return any(not task.done() for tasks in self._pending.values() for task in tasks) + + async def wait_pending(self, timeout: float) -> bool: + """Observe retained work for at most timeout seconds, without cancelling it.""" + tasks = {task for tasks in self._pending.values() for task in tasks if not task.done()} + if tasks: + await asyncio.wait(tasks, timeout=max(0.0, timeout)) + return not self.has_pending + + async def run_if_acquired( + self, + role: OwnerRole, + shard_id: int, + process: ShardProcessor, + stop: asyncio.Event, + *, + item_type: str | None = None, + ) -> bool: + scope = (role, item_type) + if scope in self._active or any(not task.done() for task in self._pending.get(scope, ())): + raise self._not_stopped(scope) + if stop.is_set(): + return False + self._active.add(scope) + try: + # Shield the request so caller cancellation cannot force an unbounded + # join of a Redis implementation which suppresses cancellation. + acquire = asyncio.create_task( + self._leases.try_acquire(role, shard_id, item_type=item_type) + ) + try: + lease = await asyncio.shield(acquire) + except BaseException: + acquire.cancel() + await self._settle(scope, {acquire}) + raise + if lease is None: + return False + await self._run(scope, ShardExecution(lease), process, stop) + return True + finally: + self._active.discard(scope) + + async def _run( + self, + scope: RunnerScope, + execution: ShardExecution, + process: ShardProcessor, + stop: asyncio.Event, + ) -> None: + lease = execution._lease + + async def guarded_process() -> None: + execution.ensure_active() + await process(execution, stop) + + processor = asyncio.create_task(guarded_process()) + renewing = asyncio.Event() + heartbeat = asyncio.create_task(self._heartbeat(lease, renewing)) + stopped = asyncio.create_task(stop.wait()) + lost = asyncio.create_task(lease.lost.wait()) + tasks = {processor, heartbeat, stopped, lost} + forced = False + error: BaseException | None = None + try: + grace_deadline: float | None = None + while True: + now = time.monotonic() + if lease.lost.is_set() or now >= lease.valid_until_monotonic: + lease.lost.set() + forced = True + break + if processor.done(): + break + if heartbeat.done(): + # A heartbeat is only supposed to finish when ownership is lost. + lease.lost.set() + forced = True + break + if stop.is_set() and grace_deadline is None: + grace_deadline = now + self._config.shutdown_timeout_seconds + if grace_deadline is not None and now >= grace_deadline: + forced = True + break + deadline = lease.valid_until_monotonic + watched = {processor, heartbeat, lost} + if grace_deadline is None: + watched.add(stopped) + else: + deadline = min(deadline, grace_deadline) + await asyncio.wait( + watched, + timeout=max(0.0, deadline - now), + return_when=asyncio.FIRST_COMPLETED, + ) + except BaseException as exc: + forced = True + error = exc + finally: + execution._revoke() + # A processor returning while it still owns detached operations is a + # lifecycle violation, even if those operations cooperate with cancel. + children = {task for task in execution._children if not task.done()} + if children: + if processor.done() and not forced: + error = error or StoreOperationError( + "shard_children_pending", + "Shard processor returned with unfinished children", + ) + forced = True + if not processor.done() or processor.cancelled() or task_is_cancelling(processor): + forced = True + # Mark synchronously before cancel: a Redis implementation may catch + # cancellation and return a success, which must not revive ownership. + if renewing.is_set() and not heartbeat.done(): + lease.lost.set() + local_tasks = tasks | children | execution._waiters + for task in local_tasks: + if not task.done(): + task.cancel() + await self._settle(scope, local_tasks) + + # task.done() is terminal even if the task raised an ordinary exception. + # Read the exception before deciding release, then propagate it afterwards. + if not processor.cancelled(): + error = error or processor.exception() + error = error or execution._error + unknown = execution._outcome_unknown or ( + isinstance(error, StoreOperationError) and error.outcome_unknown + ) + if time.monotonic() >= lease.valid_until_monotonic: + lease.lost.set() + if not forced and not unknown and not lease.lost.is_set(): + release = asyncio.create_task(self._leases.release(lease)) + try: + done, _ = await asyncio.wait( + {release}, timeout=self._config.redis.request_budget_seconds + ) + except BaseException: + lease.lost.set() + release.cancel() + await self._settle(scope, {release}) + raise + if release not in done: + lease.lost.set() + release.cancel() + await self._settle(scope, {release}) + else: + release.result() + if error is not None: + raise error + + async def _heartbeat(self, lease: ShardLease, renewing: asyncio.Event) -> None: + while True: + await asyncio.sleep(self._config.owner_renew_interval_seconds) + renewing.set() + try: + if not await self._leases.renew(lease): + return + finally: + renewing.clear() + + async def _settle(self, scope: RunnerScope, tasks: set[asyncio.Task]) -> None: + """Never couple the deadline to completion of a cancelled task.""" + if not tasks: + return + # Retain before awaiting: a second caller cancellation must not orphan + # work while cleanup itself is suspended. Each callback owns its exact + # set, not a later run's same-role state slot. + retained = self._pending.setdefault(scope, set()) + for task in tasks: + if not task.done() and task not in retained: + retained.add(task) + + def pending_done( + finished: asyncio.Task[object], + scope: RunnerScope = scope, + retained: set[asyncio.Task[object]] = retained, + ) -> None: + self._pending_done(scope, retained, finished) + + task.add_done_callback(pending_done) + try: + done, _ = await asyncio.wait(tasks, timeout=self._config.redis.request_budget_seconds) + except asyncio.CancelledError as exc: + if any(not task.done() for task in tasks): + raise self._not_stopped(scope) from exc + raise + for task in done: + _consume_exception(task) + retained.discard(task) + if not retained and self._pending.get(scope) is retained: + self._pending.pop(scope) + if any(not task.done() for task in tasks): + self._metrics.count("timer_shard_stop", outcome="pending") + raise self._not_stopped(scope) + + def _pending_done( + self, + scope: RunnerScope, + retained: set[asyncio.Task[object]], + task: asyncio.Task[object], + ) -> None: + _consume_exception(task) + retained.discard(task) + if not retained and self._pending.get(scope) is retained: + self._pending.pop(scope) + + @staticmethod + def _not_stopped(scope: RunnerScope) -> StoreOperationError: + role, item_type = scope + label = role.value if item_type is None else f"{role.value}:{item_type}" + return StoreOperationError( + "shard_not_stopped", f"The {label} shard still has active local work" + ) diff --git a/openviking/service/core.py b/openviking/service/core.py index 434ed1b64b..3b938b713a 100644 --- a/openviking/service/core.py +++ b/openviking/service/core.py @@ -60,6 +60,7 @@ if TYPE_CHECKING: from openviking.config.vlm import VLMResolver + from openviking.scheduling.expiry.runtime import ExpiryRuntime from openviking.session.compressor_v3 import SessionCompressorV3 @@ -103,6 +104,7 @@ def __init__( self._uri_mutation_coordinator = UriMutationCoordinator() self._watch_scheduler: Optional[WatchScheduler] = None self._session_auto_commit_scheduler: Optional[SessionAutoCommitScheduler] = None + self._expiry_runtime: Optional["ExpiryRuntime"] = None self._encryptor: Optional[Any] = None self._privacy_config_service: Optional[UserPrivacyConfigService] = None self._runtime_config_manager: Optional[Any] = None @@ -577,7 +579,17 @@ async def initialize(self) -> None: except Exception: session_auto_commit_config = SessionAutoCommitConfig() self._session_service.set_session_auto_commit_config(session_auto_commit_config) - if session_auto_commit_config.enabled: + expiry_config = get_openviking_config().expiry + expiry_enabled = expiry_config.enabled + session_expiry_enabled = ( + session_auto_commit_config.enabled + and getattr(session_auto_commit_config, "idle_backend", "legacy") == "redis" + ) + has_expiry_work = session_expiry_enabled + if ( + session_auto_commit_config.enabled + and getattr(session_auto_commit_config, "idle_backend", "legacy") == "legacy" + ): self._session_auto_commit_scheduler = SessionAutoCommitScheduler( self._session_service, session_auto_commit_config, @@ -649,6 +661,23 @@ async def initialize(self) -> None: self._queue_manager.start() logger.info("QueueManager workers started") + if expiry_enabled and has_expiry_work: + from openviking.scheduling.expiry.runtime import ExpiryRuntime + + self._expiry_runtime = ExpiryRuntime(expiry_config) + try: + if session_expiry_enabled: + self._session_service.configure_expiry( + self._expiry_runtime.timer, + reporting_enabled=expiry_config.reporting_enabled, + ) + await self._expiry_runtime.start_storage() + await self._expiry_runtime.start_roles() + except Exception: + await self._stop_expiry() + raise + logger.info("Expiry roles started") + # Preflight the MinerU endpoint when it will be used, so endpoint # misconfiguration or a stopped service surfaces now instead of on the # first PDF import. Required for strategy="mineru"; advisory for "auto". @@ -663,6 +692,7 @@ async def initialize(self) -> None: logger.info("MinerU preflight passed: %s", pdf_config.mineru_endpoint) except RuntimeError as exc: if pdf_config.strategy == "mineru": + await self._stop_expiry() raise logger.warning( "MinerU preflight failed (fallback will retry on first parse): %s", exc @@ -673,11 +703,32 @@ async def initialize(self) -> None: self._initialized = True logger.info("OpenVikingService initialized") + async def _stop_expiry(self) -> None: + """Stop expiry work before releasing dependencies used by callbacks.""" + runtime = self._expiry_runtime + if runtime is None: + self._session_service.set_session_timer_adapter(None) + return + try: + await runtime.close() + except Exception as exc: + logger.exception("Expiry shutdown failed") + if getattr(exc, "code", None) == "shard_not_stopped": + # Retain the runtime and leave QueueFS/AGFS open for still-running + # callbacks. The caller can retry close after they terminate. + raise + else: + self._session_service.set_session_timer_adapter(None) + self._expiry_runtime = None + logger.info("Expiry stopped") + async def close(self) -> None: """Close OpenViking and release resources.""" await self._resource_service.close_background_tasks() vlm_resolver = getattr(self, "_vlm_resolver", None) + await self._stop_expiry() + if self._runtime_config_manager: await self._runtime_config_manager.stop_refresh_loop() self._runtime_config_manager = None diff --git a/openviking/service/session_service.py b/openviking/service/session_service.py index c33f071290..48a95a3a39 100644 --- a/openviking/service/session_service.py +++ b/openviking/service/session_service.py @@ -7,8 +7,9 @@ """ import asyncio +import json from dataclasses import replace -from datetime import datetime +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Dict, List, Optional from openviking.core.namespace import canonical_session_uri @@ -28,13 +29,14 @@ ) from openviking.service.task_tracker import get_task_tracker from openviking.session import Session +from openviking.session.archive_store import is_storage_not_found from openviking.session.auto_commit_policy import AutoCommitPolicy from openviking.session.memory.memory_type_registry import get_default_registry from openviking.session.memory_policy import MemoryPolicy from openviking.storage.viking_fs import VikingFS from openviking.storage.vikingdb_manager import VikingDBManager from openviking.utils.tags import normalize_search_tags -from openviking.utils.time_utils import parse_iso_datetime +from openviking.utils.time_utils import get_current_timestamp, parse_iso_datetime from openviking_cli.exceptions import ( AlreadyExistsError, NotFoundError, @@ -45,9 +47,20 @@ from openviking_cli.utils.config.memory_config import SessionAutoCommitConfig logger = get_logger(__name__) +_SESSION_META_LOCK_TIMEOUT_SECONDS = 30.0 + + +def _parse_utc_timestamp(value: str) -> datetime: + parsed = parse_iso_datetime(value) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + if TYPE_CHECKING: from openviking.config.vlm import VLMResolver + from openviking.scheduling.expiry import ExpiryScheduler + from openviking.service.session_timer import SessionTimerAdapter from openviking.session.compressor_v3 import SessionCompressorV3 from openviking.usage_reporter import UsageReporter @@ -79,6 +92,15 @@ def __init__( self._auto_commit_inflight_lock = asyncio.Lock() # Strong refs so spawned tasks aren't GC'd mid-await. self._auto_commit_tasks: set[asyncio.Task] = set() + self._session_timer_adapter: Optional["SessionTimerAdapter"] = None + + def configure_expiry(self, scheduler: "ExpiryScheduler", *, reporting_enabled: bool) -> None: + from openviking.service.session_timer import install_session_timer + + install_session_timer(scheduler, self, reporting_enabled=reporting_enabled) + + def set_session_timer_adapter(self, adapter: Optional["SessionTimerAdapter"]) -> None: + self._session_timer_adapter = adapter def set_dependencies( self, @@ -218,6 +240,7 @@ def session( memory_policy_provider=lambda: self._get_user_memory_policy(ctx), usage_reporter=self._usage_reporter, vlm_resolver=self._vlm_resolver, + timer_hook=self._session_timer_adapter, ) async def _get_user_memory_policy(self, ctx: RequestContext) -> Optional[Dict[str, Any]]: @@ -362,6 +385,12 @@ async def delete(self, session_id: str, ctx: RequestContext) -> bool: raise NotFoundError(session_id, "session") await self._viking_fs.rm(session_uri, recursive=True, ctx=ctx) + if self._session_timer_adapter is not None: + from openviking.scheduling.expiry import SESSION_ITEM_TYPE, ItemIdentity + + await self._session_timer_adapter.unregister_after_delete( + ItemIdentity(SESSION_ITEM_TYPE, ctx.account_id, ctx.user.user_id, session_id) + ) logger.info(f"Deleted session: {session_id}") self._record_lifecycle_metric("delete", "ok") return True @@ -454,6 +483,118 @@ async def commit_async( self._record_archive_metric("ok" if result.get("archived") else "skip") return result + async def commit_on_timeout(self, session_id: str, ctx: RequestContext) -> Dict[str, Any]: + """Validate once, then reuse the ordinary session commit API. + + Appends after validation are allowed: commit_async owns the snapshot + boundary and its existing lock/queue/recovery behavior. This wrapper + neither rechecks idle under that lock nor waits for memory extraction. + """ + try: + session = await self.get(session_id, ctx, auto_create=False) + except Exception as exc: + if not is_storage_not_found(exc): + raise + return {"status": "skipped", "reason": "not_found"} + decision = self._check_timeout_commit(session, datetime.now(timezone.utc)) + if decision is not None: + return decision + result = await session.commit_async( + keep_recent_count=0, + persist_keep_recent_count=False, + record_auto_commit_success=True, + ) + if result.get("status") == "accepted": + await self._record_idle_commit_time(session_id, ctx, get_current_timestamp()) + self._record_lifecycle_metric("commit", "ok" if result.get("status") else "error") + self._record_archive_metric("ok" if result.get("archived") else "skip") + return result + + @staticmethod + def _check_timeout_commit(session: Session, now: datetime) -> Optional[Dict[str, Any]]: + """Check the freshly loaded state without inspecting archive internals.""" + policy_data = session.meta.auto_commit_policy + if policy_data is None: + return {"status": "skipped", "reason": "idle_disabled"} + timeout_seconds = AutoCommitPolicy.from_dict(policy_data).idle_timeout_seconds + if timeout_seconds <= 0: + return {"status": "skipped", "reason": "idle_disabled"} + if not session.messages: + return {"status": "skipped", "reason": "no_messages"} + if now.tzinfo is None: + now = now.replace(tzinfo=timezone.utc) + now_ms = int(now.timestamp() * 1000) + accepted_at = session.meta.last_idle_commit_accepted_at + if accepted_at: + accepted = _parse_utc_timestamp(accepted_at) + if (now - accepted).total_seconds() < timeout_seconds: + return { + "status": "deferred", + "reason": "idle_cooldown", + "timeout_seconds": timeout_seconds, + "due_at_ms": now_ms + timeout_seconds * 1000, + } + if not session.meta.last_message_at: + raise RuntimeError("Session last_message_at is missing") + last_message_at = _parse_utc_timestamp(session.meta.last_message_at) + activity_due_ms = int(last_message_at.timestamp() * 1000) + timeout_seconds * 1000 + if now_ms < activity_due_ms: + return { + "status": "deferred", + "reason": "idle_activity", + "timeout_seconds": timeout_seconds, + "due_at_ms": activity_due_ms, + } + return None + + async def _record_idle_commit_time( + self, session_id: str, ctx: RequestContext, accepted_at: str + ) -> None: + """Best-effort patch of the latest metadata after an accepted commit. + + Acquire the session lock only after commit_async returns. Re-read and + patch the raw dictionary rather than saving a stale Session.meta, so a + concurrent append/config change and unknown fields are preserved. + """ + try: + self._ensure_initialized() + fs = self._viking_fs + session_uri = canonical_session_uri(ctx, session_id) + session_path = fs._uri_to_path(session_uri, ctx=ctx) + lease = await fs._async_agfs.pathlock_acquire_exact( + session_path, timeout_secs=_SESSION_META_LOCK_TIMEOUT_SECONDS + ) + try: + # A missing meta (including a session deleted meanwhile) must + # not be replaced with stale data or recreate the session. + content = await fs.read_file(f"{session_uri}/.meta.json", ctx=ctx) + metadata = json.loads(content) + if not isinstance(metadata, dict): + raise ValueError("Session metadata must be a JSON object") + previous = metadata.get("last_idle_commit_accepted_at") + if previous: + try: + previous_dt = _parse_utc_timestamp(previous) + except (TypeError, ValueError): + logger.warning("Replacing invalid idle commit time for %s", session_id) + else: + if previous_dt >= _parse_utc_timestamp(accepted_at): + return + metadata["last_idle_commit_accepted_at"] = accepted_at + metadata["updated_at"] = get_current_timestamp() + await fs.write_file( + f"{session_uri}/.meta.json", + json.dumps(metadata, ensure_ascii=False), + ctx=ctx, + lease_ref=lease, + ) + finally: + await fs._async_agfs.pathlock_release(lease) + except Exception: + # Recording failure must not turn an already accepted commit into + # a retry. Cancellation still propagates (it is a BaseException). + logger.warning("Failed to record idle commit time for %s", session_id, exc_info=True) + async def get_commit_task(self, task_id: str, ctx: RequestContext) -> Optional[Dict[str, Any]]: """Query background commit task status by task_id for the calling owner.""" task = await get_task_tracker().get( diff --git a/openviking/service/session_timer.py b/openviking/service/session_timer.py new file mode 100644 index 0000000000..7900e1fba4 --- /dev/null +++ b/openviking/service/session_timer.py @@ -0,0 +1,152 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Session business adapter for the generic distributed resource timer.""" + +from __future__ import annotations + +import asyncio +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any + +from openviking.scheduling.expiry import ( + SESSION_ITEM_TYPE, + CallbackRequest, + CallbackResult, + CallbackStatus, + ExpiryMode, + ExpiryRegistration, + ExpiryScheduler, + HandlerOptions, + ItemIdentity, +) +from openviking.server.identity import RequestContext, Role +from openviking.session.auto_commit_policy import AutoCommitPolicy +from openviking.session.session import SessionTimerSnapshot +from openviking_cli.session.user_id import UserIdentifier +from openviking_cli.utils import get_logger + +if TYPE_CHECKING: + from openviking.service.session_service import SessionService + +logger = get_logger(__name__) + + +def install_session_timer( + timer: ExpiryScheduler, + sessions: "SessionService", + *, + reporting_enabled: bool, +) -> None: + timer.register_handler( + SESSION_ITEM_TYPE, + SessionTimeoutHandler(sessions, timer), + options=HandlerOptions(concurrency=4, max_attempts=3), + ) + sessions.set_session_timer_adapter(SessionTimerAdapter(timer) if reporting_enabled else None) + + +def effective_idle_timeout(policy: dict | None) -> int | None: + if policy is None: + return None + timeout = AutoCommitPolicy.from_dict(policy).idle_timeout_seconds + return timeout if timeout > 0 else None + + +def timestamp_ms(value: str) -> int: + normalized = value[:-1] + "+00:00" if value.endswith("Z") else value + parsed = datetime.fromisoformat(normalized) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return int(parsed.astimezone(timezone.utc).timestamp() * 1000) + + +class SessionTimerAdapter: + def __init__(self, timer: ExpiryScheduler) -> None: + self._timer = timer + + def build_report(self, snapshot: SessionTimerSnapshot) -> ExpiryRegistration | None: + timeout = effective_idle_timeout(snapshot.auto_commit_policy) + if timeout is None: + return None + return ExpiryRegistration( + identity=ItemIdentity( + SESSION_ITEM_TYPE, + snapshot.account_id, + snapshot.user_id, + snapshot.session_id, + ), + due_at_ms=timestamp_ms(snapshot.last_message_at) + timeout * 1000, + timeout_seconds=timeout, + mode=ExpiryMode.IDLE_ONESHOT, + ) + + async def report_after_append(self, snapshot: SessionTimerSnapshot) -> None: + try: + report = self.build_report(snapshot) + if report is not None: + await self._timer.report(report) + except asyncio.CancelledError: + raise + except Exception as exc: + self.record_report_failure("report_failed", exc) + + async def unregister_after_delete(self, identity: ItemIdentity) -> None: + try: + await self._timer.unregister(identity) + except asyncio.CancelledError: + raise + except Exception as exc: + self.record_report_failure("unregister_failed", exc) + + def record_report_failure(self, reason: str, exc: Exception | None = None) -> None: + logger.warning( + "Session timer operation failed reason=%s error=%s", + reason, + type(exc).__name__ if exc is not None else "unknown", + exc_info=exc is not None, + ) + + +class SessionTimeoutHandler: + def __init__(self, sessions: "SessionService", timer: ExpiryScheduler) -> None: + self._sessions = sessions + self._timer = timer + + async def handle(self, request: CallbackRequest) -> CallbackResult: + result = await self._sessions.commit_on_timeout( + request.identity.local_id, self._make_context(request.identity) + ) + if result.get("status") == "deferred": + await self._report_deferred(request.identity, result) + return CallbackResult(CallbackStatus.COMPLETED, "idle_rescheduled") + return self._to_callback_result(result) + + async def _report_deferred(self, identity: ItemIdentity, result: dict[str, Any]) -> None: + if result.get("reason") not in {"idle_cooldown", "idle_activity"}: + raise ValueError("Unexpected deferred session result") + await self._timer.report( + ExpiryRegistration( + identity=identity, + due_at_ms=result["due_at_ms"], + timeout_seconds=result["timeout_seconds"], + mode=ExpiryMode.IDLE_ONESHOT, + ) + ) + + def _make_context(self, identity: ItemIdentity) -> RequestContext: + return RequestContext( + user=UserIdentifier(account_id=identity.account_id, user_id=identity.user_id), + role=Role.USER, + ) + + def _to_callback_result(self, result: dict[str, Any]) -> CallbackResult: + status = result.get("status") + if status == "accepted": + return CallbackResult(CallbackStatus.COMPLETED, "accepted") + if status == "skipped" and result.get("reason") in { + "not_found", + "idle_disabled", + "no_messages", + }: + return CallbackResult(CallbackStatus.COMPLETED, str(result["reason"])) + return CallbackResult(CallbackStatus.RETRY, "unexpected_commit_result") diff --git a/openviking/session/session.py b/openviking/session/session.py index 2ab01e8f3a..a5006a6e00 100644 --- a/openviking/session/session.py +++ b/openviking/session/session.py @@ -6,12 +6,13 @@ """ import asyncio +import copy import inspect import json import re from dataclasses import dataclass, field, replace from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Protocol from uuid import uuid4 from openviking.core.context import ContextLevel @@ -320,6 +321,9 @@ class SessionMeta: # Timestamp of the most recent successful auto-commit, surfaced via session # GET and used to throttle auto-commit frequency. last_auto_commit_at: str = "" + # Timeout commits record their accepted time externally; this is separate + # from the legacy auto-commit statistic and is not a Phase-1 ready marker. + last_idle_commit_accepted_at: str = "" # Default custom scalar tags applied to event memories extracted from this # session. Maps to config.memory_extraction_config.events.tags in the API. # None means no session default; a commit may still override per-call. @@ -350,6 +354,7 @@ def to_dict(self) -> Dict[str, Any]: ), "last_message_at": self.last_message_at, "last_auto_commit_at": self.last_auto_commit_at, + "last_idle_commit_accepted_at": self.last_idle_commit_accepted_at, } if self.total_message_count is not None: data["total_message_count"] = self.total_message_count @@ -404,10 +409,26 @@ def from_dict(cls, data: Dict[str, Any]) -> "SessionMeta": auto_commit_policy=data.get("auto_commit_policy"), last_message_at=data.get("last_message_at", ""), last_auto_commit_at=data.get("last_auto_commit_at", ""), + last_idle_commit_accepted_at=data.get("last_idle_commit_accepted_at", ""), event_search_tags=data.get("event_search_tags"), ) +@dataclass(frozen=True) +class SessionTimerSnapshot: + account_id: str + user_id: str + session_id: str + last_message_at: str + auto_commit_policy: Optional[Dict[str, Any]] + + +class SessionTimerHook(Protocol): + async def report_after_append(self, snapshot: SessionTimerSnapshot) -> None: ... + + def record_report_failure(self, reason: str, exc: Optional[Exception] = None) -> None: ... + + class Session: """Session management class - Message = role + parts.""" @@ -427,6 +448,7 @@ def __init__( agent_evolution_enabled_provider: Optional[Callable[[], bool | Awaitable[bool]]] = None, memory_policy_provider: Optional[MemoryPolicyProvider] = None, vlm_resolver: Optional["VLMResolver"] = None, + timer_hook: Optional[SessionTimerHook] = None, ): self._viking_fs = viking_fs self._vikingdb_manager = vikingdb_manager @@ -466,6 +488,7 @@ def __init__( self._memory_policy_provider = memory_policy_provider self._usage_reporter = usage_reporter self._vlm_resolver = vlm_resolver + self._timer_hook = timer_hook async def _get_vlm_config(self): if self._vlm_resolver is None: @@ -711,6 +734,9 @@ async def _append_messages_authoritatively(self, messages: List[Message]) -> Non lease = await self._viking_fs._async_agfs.pathlock_acquire_exact( session_path, timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS ) + snapshot: Optional[SessionTimerSnapshot] = None + snapshot_error: Optional[Exception] = None + metadata_reload_error: Optional[Exception] = None try: live_messages_missing = False try: @@ -727,10 +753,11 @@ async def _append_messages_authoritatively(self, messages: List[Message]) -> Non ctx=self.ctx, ) self._meta = SessionMeta.from_dict(json.loads(meta_content)) - except Exception: + except Exception as exc: # Legacy/malformed metadata must not prevent an otherwise safe # append. Message correctness remains rooted in messages.jsonl. self._meta = in_memory_meta + metadata_reload_error = exc await self._apply_appended_messages_to_state(messages) batch_content = "".join(message.to_jsonl() + "\n" for message in messages) @@ -747,8 +774,42 @@ async def _append_messages_authoritatively(self, messages: List[Message]) -> Non ctx=self.ctx, ) await self._save_meta() + if self._timer_hook is not None: + try: + # _save_meta() made this exact in-memory state authoritative + # under the Session path lock, even when the preceding reload + # failed. Reporting it prevents the previous due time from + # firing early and repairs a missing timer registration. + snapshot = self._build_timer_snapshot_locked() + except Exception as exc: + snapshot_error = exc finally: await self._viking_fs._async_agfs.pathlock_release(lease) + if self._timer_hook is not None: + if metadata_reload_error is not None: + logger.warning( + "Session metadata reload failed before append; reporting repaired state", + exc_info=metadata_reload_error, + ) + if snapshot is not None: + await self._timer_hook.report_after_append(snapshot) + else: + self._timer_hook.record_report_failure("snapshot_build_failed", snapshot_error) + + def _build_timer_snapshot_locked(self) -> SessionTimerSnapshot: + if not self._meta.last_message_at: + raise ValueError("Session last_message_at is missing") + account_id = self._meta.created_by_account_id or self.ctx.account_id + user_id = self._meta.created_by_user_id or self.ctx.user.user_id + if not account_id or not user_id: + raise ValueError("Session owner identity is incomplete") + return SessionTimerSnapshot( + account_id=account_id, + user_id=user_id, + session_id=self.session_id, + last_message_at=self._meta.last_message_at, + auto_commit_policy=copy.deepcopy(self._meta.auto_commit_policy), + ) async def _apply_appended_messages_to_state(self, messages: List[Message]) -> None: """Update in-memory counters after an authoritative root reload.""" diff --git a/openviking_cli/expiry_bootstrap.py b/openviking_cli/expiry_bootstrap.py new file mode 100644 index 0000000000..1fdedd28e1 --- /dev/null +++ b/openviking_cli/expiry_bootstrap.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Standalone distributed expiry publisher.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import signal +from pathlib import Path + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="OpenViking expiry publisher") + parser.add_argument("--config", required=True, type=Path) + return parser.parse_args(argv) + + +def load_expiry_config(path: Path): + from openviking.scheduling.expiry.config import ExpiryConfig + + raw = json.loads(os.path.expandvars(path.expanduser().read_text(encoding="utf-8-sig"))) + section = dict(raw.get("expiry", {})) + section["reporting_enabled"] = False + section["callback_enabled"] = False + config = ExpiryConfig.model_validate(section) + if not config.scheduler_enabled: + raise ValueError("expiry.scheduler_enabled must be true for expiry CLI") + return config + + +def install_signal_handlers(stop: asyncio.Event) -> None: + loop = asyncio.get_running_loop() + for signum in (signal.SIGINT, signal.SIGTERM): + try: + loop.add_signal_handler(signum, stop.set) + except NotImplementedError: + signal.signal(signum, lambda *_args: loop.call_soon_threadsafe(stop.set)) + + +async def run_publisher(config, stop: asyncio.Event) -> None: + from openviking.scheduling.expiry.runtime import ExpiryRuntime + + runtime = ExpiryRuntime(config) + try: + await runtime.start_storage() + await runtime.start_roles() + wait_task = asyncio.create_task(runtime.wait()) + stop_task = asyncio.create_task(stop.wait()) + done, _ = await asyncio.wait({wait_task, stop_task}, return_when=asyncio.FIRST_COMPLETED) + if wait_task in done: + await wait_task + finally: + await runtime.close() + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv) + config = load_expiry_config(args.config) + + async def runner() -> None: + stop = asyncio.Event() + install_signal_handlers(stop) + await run_publisher(config, stop) + + asyncio.run(runner()) + return 0 + except KeyboardInterrupt: + return 0 + except Exception: + logging.exception("Expiry publisher failed") + return 1 diff --git a/openviking_cli/utils/config/__init__.py b/openviking_cli/utils/config/__init__.py index 75b31354a8..93e82f17d5 100644 --- a/openviking_cli/utils/config/__init__.py +++ b/openviking_cli/utils/config/__init__.py @@ -53,6 +53,7 @@ SYSTEM_CONFIG_DIR, ) from .embedding_config import EmbeddingConfig +from .expiry_config import ExpiryConfig, ExpiryRedisConfig from .git_config import GitConfig, GitLocalConfig, GitS3Config from .glob_config import GlobConfig, GlobEngine from .grep_config import GrepConfig, GrepEngine @@ -146,6 +147,8 @@ "GrepEngine", "RerankConfig", "RetrievalConfig", + "ExpiryConfig", + "ExpiryRedisConfig", "StorageConfig", "OpenGaussConfig", "VectorDBBackendConfig", diff --git a/openviking_cli/utils/config/expiry_config.py b/openviking_cli/utils/config/expiry_config.py new file mode 100644 index 0000000000..fd7ca560a6 --- /dev/null +++ b/openviking_cli/utils/config/expiry_config.py @@ -0,0 +1,5 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig + +__all__ = ["ExpiryConfig", "ExpiryRedisConfig"] diff --git a/openviking_cli/utils/config/memory_config.py b/openviking_cli/utils/config/memory_config.py index 92e358f8fd..0dc876ff57 100644 --- a/openviking_cli/utils/config/memory_config.py +++ b/openviking_cli/utils/config/memory_config.py @@ -21,6 +21,13 @@ class SessionAutoCommitConfig(BaseModel): "started. When disabled, neither happens." ), ) + idle_backend: Literal["legacy", "redis"] = Field( + default="legacy", + description=( + "Idle timeout discovery backend. 'legacy' scans session metadata; " + "'redis' uses distributed expiry scheduling." + ), + ) check_interval_seconds: float = Field(default=600.0, gt=0) scan_rate_limit_files_per_second: float = Field( default=2.0, diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py index 6e9be1905d..31205c65ca 100644 --- a/openviking_cli/utils/config/open_viking_config.py +++ b/openviking_cli/utils/config/open_viking_config.py @@ -29,6 +29,7 @@ from .context_gateway_config import ContextGatewayConfig from .embedding_config import EmbeddingConfig from .encryption_config import EncryptionConfig +from .expiry_config import ExpiryConfig from .git_config import GitConfig from .glob_config import GlobConfig from .grep_config import GrepConfig @@ -444,6 +445,28 @@ def _inherit_git_defaults_from_agfs(cls, data: Any) -> Any: memory: MemoryConfig = Field(default_factory=MemoryConfig, description="Memory configuration") + expiry: ExpiryConfig = Field( + default_factory=ExpiryConfig, + description="Distributed expiry scheduling configuration", + ) + + @model_validator(mode="after") + def _validate_expiry_backend(self) -> "OpenVikingConfig": + self._check_expiry_backend() + return self + + def _check_expiry_backend(self) -> None: + idle = self.memory.session_auto_commit + if ( + idle.enabled + and idle.idle_backend == "redis" + and not self.expiry.reporting_enabled + ): + raise ValueError( + "memory.session_auto_commit.idle_backend='redis' requires " + "expiry.reporting_enabled=true" + ) + agent_evolution: AgentEvolutionConfig = RuntimeField( default_factory=AgentEvolutionConfig, description="Dynamic cluster default for Agent Evolution.", @@ -563,6 +586,7 @@ def from_dict(cls, config: Dict[str, Any]) -> "OpenVikingConfig": path_prefix="memory", ) ) from e + instance._check_expiry_backend() # Apply parser configurations for parser_type in parser_types: diff --git a/pyproject.toml b/pyproject.toml index ea4c2a373f..a7423fcf2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,9 @@ dependencies = [ [project.optional-dependencies] +expiry = [ + "redis>=5,<6", +] context-gateway = [ "aiohttp>=3.13.3", "async-timeout>=5.0.1", @@ -214,6 +217,7 @@ ov = "openviking_cli.rust_cli:main" # Rust CLI 入口（极简包装器） openviking = "openviking_cli.rust_cli:main" # Rust CLI 入口（放弃 python CLI） openviking-server = "openviking_cli.server_bootstrap:main" # also: `openviking-server ingest ...` openviking-context-gateway = "context_gateway.cli:main" +openviking-timer = "openviking_cli.expiry_bootstrap:main" vikingbot = "vikingbot.cli.commands:app" [tool.setuptools_scm] @@ -229,6 +233,7 @@ exclude = ["tests*", "docs*", "examples*"] [tool.setuptools.package-data] context_gateway = ["client-rules.json"] +"openviking.scheduling.expiry" = ["lua/*.lua"] "vikingbot.studio.providers.feishu" = ["icon.png"] openviking = [ "prompts/templates/**/*.yaml", diff --git a/setup.py b/setup.py index 0cbea08808..65335cf098 100644 --- a/setup.py +++ b/setup.py @@ -579,6 +579,7 @@ def has_ext_modules(self): cmdclass=cmdclass, package_data={ "context_gateway": ["client-rules.json"], + "openviking.scheduling.expiry": ["lua/*.lua"], "openviking": [ "lib/ragfs_python*.so", "lib/ragfs_python*.pyd", diff --git a/tests/scheduling/expiry/test_config.py b/tests/scheduling/expiry/test_config.py new file mode 100644 index 0000000000..c935f6f335 --- /dev/null +++ b/tests/scheduling/expiry/test_config.py @@ -0,0 +1,101 @@ +import pytest +from pydantic import ValidationError + +from openviking.scheduling.expiry.config import ExpiryConfig +from openviking_cli.utils.config.open_viking_config import OpenVikingConfig + + +def test_timer_disabled_by_default_without_redis_dependency(): + assert ExpiryConfig().enabled is False + + +def test_enabled_expiry_requires_service_name_and_endpoint(): + with pytest.raises(ValidationError): + ExpiryConfig(reporting_enabled=True) + + +def test_shard_count_is_fixed_for_the_initial_protocol(): + with pytest.raises(ValidationError): + ExpiryConfig(shard_count=256) + + +@pytest.mark.parametrize("value", [0, 4097]) +def test_cleanup_batch_size_is_bounded(value): + with pytest.raises(ValidationError): + ExpiryConfig(cleanup_batch_size=value) + + +def test_callback_requires_reporting_for_cooldown_reschedule(): + with pytest.raises(ValidationError): + ExpiryConfig( + service_name="documents", + callback_enabled=True, + redis={"endpoints": ["localhost:6379"]}, + ) + + +def test_cluster_rejects_mixed_transport_schemes(): + with pytest.raises(ValidationError, match="one transport scheme"): + ExpiryConfig( + service_name="documents", + reporting_enabled=True, + redis={ + "mode": "cluster", + "endpoints": ["redis://one:6379", "rediss://two:6379"], + }, + ) + + +def test_redis_password_can_be_configured_directly(): + config = ExpiryConfig( + service_name="documents", + reporting_enabled=True, + redis={ + "endpoints": ["localhost:6379"], + "password": "secret", + }, + ) + + assert config.redis.password == "secret" + + +def test_redis_password_sources_are_mutually_exclusive(): + with pytest.raises(ValidationError, match="mutually exclusive"): + ExpiryConfig( + service_name="documents", + reporting_enabled=True, + redis={ + "endpoints": ["localhost:6379"], + "password": "secret", + "password_env": "REDIS_PASSWORD", + }, + ) + + +def test_redis_idle_backend_requires_reporting(): + with pytest.raises(ValueError, match="reporting_enabled"): + OpenVikingConfig.from_dict( + { + "memory": { + "session_auto_commit": { + "enabled": True, + "idle_backend": "redis", + } + } + } + ) + + +def test_expiry_top_level_config_is_parsed(): + config = OpenVikingConfig.from_dict( + { + "expiry": { + "service_name": "documents", + "reporting_enabled": True, + "redis": {"endpoints": ["localhost:6379"]}, + } + } + ) + + assert config.expiry.service_name == "documents" + assert config.expiry.reporting_enabled is True diff --git a/tests/scheduling/expiry/test_consumer.py b/tests/scheduling/expiry/test_consumer.py new file mode 100644 index 0000000000..cec9b2c3f5 --- /dev/null +++ b/tests/scheduling/expiry/test_consumer.py @@ -0,0 +1,668 @@ +import asyncio + +import pytest + +from openviking.scheduling.expiry.api import ( + CallbackResult, + CallbackStatus, + HandlerOptions, + ItemIdentity, +) +from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig +from openviking.scheduling.expiry.consumer import ExpiryConsumer +from openviking.scheduling.expiry.keys import encode_item_id +from openviking.scheduling.expiry.leases import ShardLeaseCoordinator +from openviking.scheduling.expiry.metrics import ExpiryMetrics +from openviking.scheduling.expiry.models import ( + BucketItem, + BucketPage, + BucketRef, + Claim, + OwnerRole, + OwnerToken, + RegistryPage, + StoreOperationError, + claim_layer_bounds, + decode_claim_score, + encode_claim_score, + encode_ready_claim_score, +) +from openviking.scheduling.expiry.registry import HandlerBinding +from openviking.scheduling.expiry.shard_runner import ShardRunner + + +class Store: + def __init__( + self, + *, + empty=False, + attempt=0, + now_ms=200_000, + lose_on_renew=False, + ): + self.identity = ItemIdentity("session", "account", "user", "session-id") + self.bucket = BucketRef("session", 0, 60_000) + self.item_id = encode_item_id(self.identity) + self.score = None if empty else encode_claim_score(attempt, 123) + self.now_ms = now_ms + self.empty = empty + self.lose_on_renew = lose_on_renew + self.claim_requests = [] + self.claims = [] + self.advanced = [] + self.acked = [] + self.released = [] + self.metadata_reads = [] + self.renew_started = asyncio.Event() + self.allow_renew = asyncio.Event() + self.callback_finished = asyncio.Event() + self.deferred = [] + + async def try_acquire_owner(self, role, shard_id, token, ttl_seconds, *, item_type=None): + return OwnerToken(role, shard_id, token, 9999999999999, item_type) + + async def renew_owner(self, owner, ttl_seconds): + self.renew_started.set() + if self.lose_on_renew: + await self.allow_renew.wait() + return None + return owner + + async def release_owner(self, owner): + self.released.append(owner) + return True + + async def read_registry_page(self, shard_id, *, after, limit): + return RegistryPage(() if self.score is None else (self.bucket,), None) + + async def read_time_ms(self): + return self.now_ms + + async def read_bucket_page(self, bucket, *, attempt, limit): + if self.score is None: + return BucketPage(()) + lower, upper = claim_layer_bounds(attempt) + if lower <= self.score <= upper: + return BucketPage((BucketItem(self.item_id, self.score),)) + return BucketPage(()) + + async def claim(self, bucket, item, *, attempt, started_at_seconds): + self.claim_requests.append( + (bucket, item, {"attempt": attempt, "started_at_seconds": started_at_seconds}) + ) + if self.score != item.score: + return None + self.score = encode_claim_score(attempt, started_at_seconds) + claim = Claim(bucket, item.item_id, self.score) + self.claims.append(claim) + return claim + + async def advance_retry(self, claim): + if self.score != claim.claim_score: + return False + attempt, _ = decode_claim_score(claim.claim_score) + self.score = encode_ready_claim_score(attempt + 1) + self.advanced.append(claim) + return True + + async def read_metadata(self, item_id): + self.metadata_reads.append(item_id) + return None + + async def ack(self, claim): + if self.score != claim.claim_score: + return False + self.score = None + self.acked.append(claim) + return True + + async def remove_registration_if_empty(self, bucket): + return self.score is None + + async def defer_registration(self, owner, bucket, *, next_poll_at_ms): + self.deferred.append((owner, bucket, next_poll_at_ms)) + return True + + +class Handler: + def __init__(self, *, status=CallbackStatus.COMPLETED, error=None, resist_cancellation=False): + self.status = status + self.error = error + self.resist_cancellation = resist_cancellation + self.started = asyncio.Event() + self.cancelled = asyncio.Event() + self.release = asyncio.Event() + self.finished = asyncio.Event() + self.requests = [] + + async def handle(self, request): + self.requests.append(request) + self.started.set() + try: + while not self.release.is_set(): + try: + await self.release.wait() + except asyncio.CancelledError: + self.cancelled.set() + if not self.resist_cancellation: + raise + if self.error is not None: + raise self.error + return CallbackResult(self.status, "test callback") + finally: + self.finished.set() + + +class Runner: + def __init__(self, *, stop_on_shard=0, error=None): + self.calls = [] + self.stop_on_shard = stop_on_shard + self.error = error + + async def run_if_acquired(self, role, shard_id, process, stop, *, item_type=None): + self.calls.append((role, item_type, shard_id, process)) + if shard_id == self.stop_on_shard: + stop.set() + if self.error is not None and shard_id == 0: + raise self.error + return False + + +def config(**overrides): + values = { + "shard_count": 1, + "owner_renew_interval_seconds": 0.01, + "callback_poll_seconds": 0.01, + "shutdown_timeout_seconds": 0.1, + "processing_timeout_seconds": 2, + "redis": ExpiryRedisConfig.model_construct(request_budget_seconds=0.05), + } + values.update(overrides) + return ExpiryConfig.model_construct(**values) + + +def consumer(store, handler, *, max_attempts=3, metrics=None): + timer_config = config() + metrics = metrics or ExpiryMetrics() + coordinator = ShardLeaseCoordinator(store, timer_config, metrics) + shard_runner = ShardRunner(coordinator, timer_config, metrics) + binding = HandlerBinding(handler, HandlerOptions(concurrency=1, max_attempts=max_attempts)) + return ( + ExpiryConsumer( + store, + shard_runner, + "session", + binding, + timer_config, + metrics, + start_shard=0, + ), + shard_runner, + ) + + +def worker_consumer(store, shard_runner, timer_config): + return ExpiryConsumer( + store, + shard_runner, + "session", + HandlerBinding(Handler(), HandlerOptions()), + timer_config, + ExpiryMetrics(), + start_shard=0, + ) + + +@pytest.mark.asyncio +async def test_empty_registry_finishes_and_releases_owner_without_callback(): + store = Store(empty=True) + handler = Handler() + resource_consumer, shard_runner = consumer(store, handler) + assert ( + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + is True + ) + assert handler.requests == [] + assert store.claim_requests == [] + assert len(store.released) == 1 + assert shard_runner.has_pending is False + + +@pytest.mark.asyncio +async def test_completed_callback_with_missing_metadata_acks_claim_and_releases_owner(): + store = Store() + handler = Handler() + handler.release.set() + resource_consumer, shard_runner = consumer(store, handler) + assert ( + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + is True + ) + assert [request.identity for request in handler.requests] == [store.identity] + assert store.metadata_reads == [store.item_id] + assert store.acked == [store.claims[0]] + assert store.claim_requests[0][2]["attempt"] == 0 + assert len(store.released) == 1 + assert shard_runner.has_pending is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["retry", "exception"]) +@pytest.mark.parametrize("last_attempt", [False, True]) +async def test_failed_callback_advances_immediately_and_acks_after_last_attempt( + failure, last_attempt +): + store = Store(attempt=1 if last_attempt else 0) + handler = Handler( + status=CallbackStatus.RETRY, + error=RuntimeError("callback failed") if failure == "exception" else None, + ) + resource_consumer, shard_runner = consumer(store, handler, max_attempts=2) + stop = asyncio.Event() + run_task = asyncio.create_task( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + stop, + item_type="session", + ) + ) + try: + await asyncio.wait_for(handler.started.wait(), timeout=2.0) + callback_tasks = set(resource_consumer._state.tasks) + assert callback_tasks + handler.release.set() + await asyncio.wait_for(handler.finished.wait(), timeout=2.0) + finished, pending = await asyncio.wait(callback_tasks, timeout=2.0) + assert finished == callback_tasks + assert not pending + assert await asyncio.wait_for(run_task, timeout=2.0) is True + assert len(store.acked) == 1 + assert len(store.advanced) == (0 if last_attempt else 1) + assert len(handler.requests) == (1 if last_attempt else 2) + assert store.deferred == [] + assert len(store.released) == 1 + assert shard_runner.has_pending is False + finally: + stop.set() + handler.release.set() + await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) + assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("result", ["completed", "retry", "exception"]) +@pytest.mark.parametrize("last_attempt", [False, True]) +async def test_cancelled_callback_never_acks_after_drain_when_handler_suppresses_cancel( + result, last_attempt +): + store = Store(attempt=1 if last_attempt else 0) + handler = Handler( + status=CallbackStatus.COMPLETED if result == "completed" else CallbackStatus.RETRY, + error=RuntimeError("callback failed") if result == "exception" else None, + resist_cancellation=True, + ) + resource_consumer, shard_runner = consumer(store, handler, max_attempts=2) + stop = asyncio.Event() + run_task = asyncio.create_task( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + stop, + item_type="session", + ) + ) + try: + await asyncio.wait_for(handler.started.wait(), timeout=2.0) + with pytest.raises(StoreOperationError) as error: + await resource_consumer.drain(0) + assert error.value.code == "shard_not_stopped" + await asyncio.wait_for(handler.cancelled.wait(), timeout=2.0) + # Keep the shard running: cancellation of the child alone must block ACK, + # without relying on owner loss or the runner's shutdown revocation. + assert not run_task.done() + assert store.acked == [] + assert store.released == [] + handler.release.set() + await asyncio.wait_for(handler.finished.wait(), timeout=2.0) + await resource_consumer.drain(1.0) + assert store.acked == [] + stop.set() + assert await asyncio.wait_for(run_task, timeout=2.0) is True + assert len(handler.requests) == 1 + assert store.released == [] + assert shard_runner.has_pending is False + finally: + stop.set() + handler.release.set() + await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) + assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True + + +@pytest.mark.asyncio +async def test_timed_out_last_attempt_is_retried_without_becoming_exhausted(): + store = Store(attempt=2) + handler = Handler() + handler.release.set() + resource_consumer, shard_runner = consumer(store, handler) + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + assert len(store.acked) == 1 + assert len(store.claims) == 1 + assert decode_claim_score(store.claims[0].claim_score)[0] == 2 + assert len(handler.requests) == 1 + assert len(store.released) == 1 + assert shard_runner.has_pending is False + + +@pytest.mark.asyncio +async def test_waiting_bucket_defers_registration_to_exact_timeout_deadline(): + store = Store(now_ms=124_000) + handler = Handler() + resource_consumer, shard_runner = consumer(store, handler) + + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + + assert handler.requests == [] + assert store.claim_requests == [] + assert len(store.deferred) == 1 + assert store.deferred[0][2] == 125_000 + + +@pytest.mark.asyncio +async def test_waiting_attempt_does_not_block_ready_item_in_next_attempt(): + store = Store(now_ms=124_000) + retry_identity = ItemIdentity("session", "account", "user", "retry") + retry_item_id = encode_item_id(retry_identity) + retry_score = encode_ready_claim_score(1) + + async def read_bucket_page(bucket, *, attempt, limit): + if attempt == 0: + return ( + BucketPage(()) + if store.score is None + else BucketPage((BucketItem(store.item_id, store.score),)) + ) + if attempt == 1 and retry_score is not None: + return BucketPage((BucketItem(retry_item_id, retry_score),)) + return BucketPage(()) + + async def claim(bucket, item, *, attempt, started_at_seconds): + nonlocal retry_score + store.claim_requests.append( + (bucket, item, {"attempt": attempt, "started_at_seconds": started_at_seconds}) + ) + if item.item_id != retry_item_id or item.score != retry_score: + return None + retry_score = encode_claim_score(attempt, started_at_seconds) + result = Claim(bucket, item.item_id, retry_score) + store.claims.append(result) + return result + + async def ack(claim): + nonlocal retry_score + if claim.item_id != retry_item_id or claim.claim_score != retry_score: + return False + retry_score = None + store.acked.append(claim) + return True + + store.read_bucket_page = read_bucket_page + store.claim = claim + store.ack = ack + handler = Handler() + handler.release.set() + resource_consumer, shard_runner = consumer(store, handler) + + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + + assert [request.identity for request in handler.requests] == [retry_identity] + assert len(store.acked) == 1 + assert store.deferred[0][2] == 125_000 + + +@pytest.mark.asyncio +async def test_claim_conflict_rescans_without_duplicate_callback(): + store = Store() + original_claim = store.claim + conflicts = 1 + + async def claim_with_conflict(*args, **kwargs): + nonlocal conflicts + if conflicts: + conflicts -= 1 + store.claim_requests.append((args[0], args[1], kwargs)) + return None + return await original_claim(*args, **kwargs) + + store.claim = claim_with_conflict + handler = Handler() + handler.release.set() + resource_consumer, shard_runner = consumer(store, handler) + + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + + assert len(store.claim_requests) == 2 + assert len(handler.requests) == 1 + assert len(store.acked) == 1 + + +@pytest.mark.asyncio +async def test_last_explicit_failure_emits_exhausted_metric_and_removes_item(): + events = [] + metrics = ExpiryMetrics(lambda name, value, labels: events.append((name, value, labels))) + store = Store(attempt=1) + handler = Handler(status=CallbackStatus.RETRY) + handler.release.set() + resource_consumer, shard_runner = consumer( + store, + handler, + max_attempts=2, + metrics=metrics, + ) + + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + + assert store.score is None + assert len(store.acked) == 1 + assert ( + "timer_callback_exhausted", + 1.0, + {"item_type": "session", "outcome": "callback_retry"}, + ) in events + + +@pytest.mark.asyncio +async def test_shard_visit_filters_type_and_processes_only_one_minute(): + store = Store() + second = BucketRef("session", 0, 120_000) + watch = BucketRef("watch", 0, 60_000) + + async def read_registry_page(shard_id, *, after, limit): + return RegistryPage((watch, store.bucket, second), None) + + store.read_registry_page = read_registry_page + handler = Handler() + handler.release.set() + resource_consumer, shard_runner = consumer(store, handler) + + await shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ) + + assert len(handler.requests) == 1 + assert store.claim_requests[0][0] == store.bucket + assert all(bucket != second for bucket, _, _ in store.claim_requests) + + +@pytest.mark.asyncio +async def test_owner_loss_keeps_resistant_callback_pending_and_prevents_late_ack(): + store = Store(lose_on_renew=True) + handler = Handler(resist_cancellation=True) + resource_consumer, shard_runner = consumer(store, handler) + stop = asyncio.Event() + run_task = asyncio.create_task( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + stop, + item_type="session", + ) + ) + try: + await asyncio.wait_for(handler.started.wait(), timeout=2.0) + await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) + store.allow_renew.set() + await asyncio.wait_for(handler.cancelled.wait(), timeout=2.0) + finished, _ = await asyncio.wait({run_task}, timeout=0.3) + assert run_task in finished, ( + "Consumer cleanup must not join a resistant callback indefinitely" + ) + with pytest.raises(StoreOperationError) as error: + run_task.result() + assert error.value.code == "shard_not_stopped" + assert shard_runner.has_pending is True + assert await asyncio.wait_for(shard_runner.wait_pending(0.01), timeout=2.0) is False + assert store.acked == [] + assert store.released == [] + # The pending child still owns its reserved slot; do not reset local + # accounting while its cancellation has not completed. + assert resource_consumer._state.reserved == 1 + handler.release.set() + assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True + assert handler.finished.is_set() + assert store.acked == [] + assert store.released == [] + assert resource_consumer._state.reserved == 0 + assert not resource_consumer._state.tasks + assert shard_runner.has_pending is False + finally: + stop.set() + store.allow_renew.set() + handler.release.set() + await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) + assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True + + +@pytest.mark.asyncio +async def test_consumer_worker_delegates_callback_role_and_stops_mid_shard_round(): + store = Store(empty=True) + shard_runner = Runner(stop_on_shard=1) + resource_consumer = worker_consumer(store, shard_runner, config(shard_count=3)) + await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) + assert [(role, item_type, shard_id) for role, item_type, shard_id, _ in shard_runner.calls] == [ + (OwnerRole.CALLBACK, "session", 0), + (OwnerRole.CALLBACK, "session", 1), + ] + assert all( + process == resource_consumer.process_shard for _, _, _, process in shard_runner.calls + ) + + +@pytest.mark.asyncio +async def test_consumer_worker_does_not_enter_runner_when_already_stopped(): + shard_runner = Runner() + resource_consumer = worker_consumer(Store(empty=True), shard_runner, config()) + stop = asyncio.Event() + stop.set() + await resource_consumer.run(stop) + assert shard_runner.calls == [] + + +@pytest.mark.asyncio +async def test_consumer_worker_propagates_pending_work_instead_of_acquiring_next_shard(): + shard_runner = Runner( + stop_on_shard=1, + error=StoreOperationError("shard_not_stopped", "callback still running"), + ) + resource_consumer = worker_consumer(Store(empty=True), shard_runner, config(shard_count=3)) + with pytest.raises(StoreOperationError) as error: + await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) + assert error.value.code == "shard_not_stopped" + assert len(shard_runner.calls) == 1 + + +@pytest.mark.asyncio +async def test_consumer_worker_continues_shard_round_after_processor_error(): + shard_runner = Runner(stop_on_shard=1, error=RuntimeError("callback failed")) + resource_consumer = worker_consumer(Store(empty=True), shard_runner, config(shard_count=3)) + await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) + assert [shard_id for _, _, shard_id, _ in shard_runner.calls] == [0, 1] + + +@pytest.mark.asyncio +async def test_consumer_worker_propagates_cancellation_instead_of_retrying(): + shard_runner = Runner(error=asyncio.CancelledError()) + resource_consumer = worker_consumer(Store(empty=True), shard_runner, config()) + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) + assert len(shard_runner.calls) == 1 diff --git a/tests/scheduling/expiry/test_facade_and_registry.py b/tests/scheduling/expiry/test_facade_and_registry.py new file mode 100644 index 0000000000..68921ae168 --- /dev/null +++ b/tests/scheduling/expiry/test_facade_and_registry.py @@ -0,0 +1,130 @@ +import pytest + +from openviking.scheduling.expiry import ( + CallbackResult, + CallbackStatus, + ExpiryError, + ExpiryRegistration, + ExpiryRegistrationResult, + HandlerOptions, + ItemIdentity, +) +from openviking.scheduling.expiry.metrics import ExpiryMetrics +from openviking.scheduling.expiry.models import StoreOperationError +from openviking.scheduling.expiry.registry import HandlerRegistry +from openviking.scheduling.expiry.service import ExpiryRuntimeState, _ExpirySchedulerFacade + + +class Handler: + async def handle(self, request): + return CallbackResult(CallbackStatus.COMPLETED, "ok") + + +class Store: + def __init__(self): + self.calls = [] + self.fail_metadata = False + + async def upsert_resource(self, shard_id, item_id, task): + self.calls.append("resource") + return ExpiryRegistrationResult(task.due_at_ms) + + async def ensure_metadata(self, item_id, task): + self.calls.append("metadata") + if self.fail_metadata: + raise StoreOperationError("redis_error", "metadata failed", retryable=True) + + async def unregister_resource(self, shard_id, item_id): + self.calls.append("unregister") + + +def test_registry_validation_and_freeze(): + registry = HandlerRegistry() + registry.register("session", Handler(), HandlerOptions(concurrency=2, max_attempts=3)) + bindings = registry.freeze() + assert bindings["session"].options.max_attempts == 3 + with pytest.raises(ExpiryError) as exc: + registry.register("watch", Handler(), HandlerOptions()) + assert exc.value.code == "registration_closed" + + +def test_registry_rejects_unsupported_type(): + registry = HandlerRegistry() + with pytest.raises(ExpiryError) as exc: + registry.register("watch", Handler(), HandlerOptions()) + assert exc.value.code == "unregistered_type" + + +@pytest.mark.asyncio +async def test_report_writes_resource_before_metadata(): + store = Store() + state = ExpiryRuntimeState(storage_ready=True, accepting_calls=True) + facade = _ExpirySchedulerFacade( + store, + HandlerRegistry(), + state, + shard_count=8, + reporting_enabled=True, + metrics=ExpiryMetrics(), + ) + result = await facade.report( + ExpiryRegistration(ItemIdentity("session", "a", "u", "s"), 1234, 60) + ) + assert result.accepted_due_at_ms == 1234 + assert store.calls == ["resource", "metadata"] + + +@pytest.mark.asyncio +async def test_metadata_failure_does_not_roll_back_resource(): + store = Store() + store.fail_metadata = True + facade = _ExpirySchedulerFacade( + store, + HandlerRegistry(), + ExpiryRuntimeState(storage_ready=True, accepting_calls=True), + shard_count=8, + reporting_enabled=True, + metrics=ExpiryMetrics(), + ) + with pytest.raises(ExpiryError) as exc: + await facade.report( + ExpiryRegistration(ItemIdentity("session", "a", "u", "s"), 1234, 60) + ) + assert exc.value.code == "timer_unavailable" + assert store.calls == ["resource", "metadata"] + + +@pytest.mark.asyncio +async def test_report_rejects_unsupported_type_before_store_write(): + store = Store() + state = ExpiryRuntimeState(storage_ready=True, accepting_calls=True) + facade = _ExpirySchedulerFacade( + store, + HandlerRegistry(), + state, + shard_count=8, + reporting_enabled=True, + metrics=ExpiryMetrics(), + ) + report = ExpiryRegistration(ItemIdentity("watch", "a", "u", "s"), 1234, 60) + with pytest.raises(ExpiryError) as exc: + await facade.report(report) + assert exc.value.code == "unregistered_type" + assert store.calls == [] + + +@pytest.mark.asyncio +async def test_unregister_rejects_unsupported_type_before_store_write(): + store = Store() + facade = _ExpirySchedulerFacade( + store, + HandlerRegistry(), + ExpiryRuntimeState(storage_ready=True, accepting_calls=True), + shard_count=8, + reporting_enabled=True, + metrics=ExpiryMetrics(), + ) + with pytest.raises(ExpiryError) as exc: + await facade.unregister(ItemIdentity("watch", "a", "u", "s")) + assert exc.value.code == "unregistered_type" + assert store.calls == [] diff --git a/tests/scheduling/expiry/test_leases.py b/tests/scheduling/expiry/test_leases.py new file mode 100644 index 0000000000..a881e11e2e --- /dev/null +++ b/tests/scheduling/expiry/test_leases.py @@ -0,0 +1,538 @@ +import asyncio +import gc +import time +import weakref +from types import SimpleNamespace + +import pytest + +from openviking.scheduling.expiry import leases as leases_module +from openviking.scheduling.expiry import models as models_module +from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig +from openviking.scheduling.expiry.leases import ShardLeaseCoordinator +from openviking.scheduling.expiry.metrics import ExpiryMetrics +from openviking.scheduling.expiry.models import ( + OwnerRole, + OwnerToken, + ShardLease, + StoreOperationError, +) +from openviking.scheduling.expiry.shard_runner import ShardExecution, ShardRunner + + +class Store: + def __init__(self, *, busy=False, renew_outcome="success", block_renew=False): + self.busy = busy + self.renew_outcome = renew_outcome + self.block_renew = block_renew + self.renew_started = asyncio.Event() + self.allow_renew = asyncio.Event() + self.acquired = [] + self.renewed = [] + self.released = [] + + async def try_acquire_owner(self, role, shard_id, token, ttl_seconds, *, item_type=None): + self.acquired.append((role, item_type, shard_id, token, ttl_seconds)) + if self.busy: + return None + return OwnerToken(role, shard_id, token, 9999999999999, item_type) + + async def renew_owner(self, owner, ttl_seconds): + self.renewed.append((owner, ttl_seconds)) + self.renew_started.set() + if self.block_renew: + await self.allow_renew.wait() + if self.renew_outcome == "error": + raise RuntimeError("renew failed") + if self.renew_outcome == "lost": + return None + return OwnerToken( + owner.role, + owner.shard_id, + owner.token, + owner.expires_at_ms + 1, + owner.item_type, + ) + + async def release_owner(self, owner): + self.released.append(owner) + return True + + +def lease(): + return ShardLease( + OwnerToken(OwnerRole.SCHEDULER, 0, "token", 9999999999999), + asyncio.Event(), + float("inf"), + ) + + +def config(**overrides): + values = { + "shard_count": 1, + "owner_renew_interval_seconds": 0.01, + "shutdown_timeout_seconds": 1.0, + "redis": ExpiryRedisConfig.model_construct(request_budget_seconds=0.05), + } + values.update(overrides) + return ExpiryConfig.model_construct(**values) + + +def runner(store, timer_config=None): + timer_config = timer_config or config() + metrics = ExpiryMetrics() + coordinator = ShardLeaseCoordinator(store, timer_config, metrics) + return ShardRunner(coordinator, timer_config, metrics) + + +@pytest.mark.asyncio +async def test_coordinator_acquires_renews_and_releases_matching_owner(): + store = Store() + coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) + acquired = await coordinator.try_acquire(OwnerRole.CALLBACK, 3, item_type="session") + assert acquired is not None + assert acquired.owner.role is OwnerRole.CALLBACK + assert acquired.owner.item_type == "session" + assert acquired.owner.shard_id == 3 + assert len(acquired.owner.token) == 32 + int(acquired.owner.token, 16) + assert acquired.valid_until_monotonic > time.monotonic() + original = acquired.owner + assert await coordinator.renew(acquired) is True + assert acquired.owner != original + assert acquired.owner.token == original.token + await coordinator.release(acquired) + assert store.released == [acquired.owner] + + +@pytest.mark.asyncio +async def test_coordinator_returns_none_when_shard_is_busy(): + store = Store(busy=True) + coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) + assert await coordinator.try_acquire(OwnerRole.SCHEDULER, 0) is None + assert store.renewed == [] + assert store.released == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["lost", "error"]) +async def test_failed_renewal_marks_lease_lost_and_cannot_revive(outcome): + store = Store(renew_outcome=outcome) + coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) + acquired = await coordinator.try_acquire(OwnerRole.SCHEDULER, 0) + original = acquired.owner + assert await coordinator.renew(acquired) is False + assert acquired.lost.is_set() + store.renew_outcome = "success" + assert await coordinator.renew(acquired) is False + await coordinator.release(acquired) + assert acquired.owner == original + assert len(store.renewed) == 1 + assert store.released == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("invalidation", ["deadline", "lost", "revoked"]) +async def test_successful_renew_response_cannot_revive_invalidated_lease(monkeypatch, invalidation): + clock = SimpleNamespace(now=10.0) + local_time = SimpleNamespace(monotonic=lambda: clock.now) + monkeypatch.setattr(leases_module, "time", local_time) + monkeypatch.setattr(models_module, "time", local_time) + store = Store(block_renew=True) + coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) + acquired = await coordinator.try_acquire(OwnerRole.SCHEDULER, 0) + original = acquired.owner + original_deadline = acquired.valid_until_monotonic + renew_task = asyncio.create_task(coordinator.renew(acquired)) + try: + await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) + if invalidation == "deadline": + clock.now = original_deadline + else: + getattr(acquired, invalidation).set() + store.allow_renew.set() + assert await asyncio.wait_for(renew_task, timeout=2.0) is False + assert acquired.owner == original + assert acquired.valid_until_monotonic == original_deadline + with pytest.raises(StoreOperationError, match="no longer valid"): + acquired.ensure_active() + finally: + store.allow_renew.set() + await asyncio.wait_for(asyncio.gather(renew_task, return_exceptions=True), timeout=2.0) + + +@pytest.mark.asyncio +async def test_release_does_not_delete_expired_owner(): + store = Store() + coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) + acquired = lease() + acquired.valid_until_monotonic = 0.0 + await coordinator.release(acquired) + assert acquired.lost.is_set() + assert store.released == [] + + +@pytest.mark.asyncio +async def test_runner_releases_normally_completed_processor_and_children(): + store = Store() + shard_runner = runner(store) + observed = [] + + async def process(execution, stop): + execution.ensure_active() + observed.append((execution.owner, execution.shard_id)) + + async def child(): + execution.ensure_active() + observed.append("child") + + await execution.spawn(child) + + assert ( + await asyncio.wait_for( + shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, asyncio.Event()), + timeout=2.0, + ) + is True + ) + assert observed[0][1] == 0 + assert observed[1] == "child" + assert store.released == [observed[0][0]] + assert shard_runner.has_pending is False + + +@pytest.mark.asyncio +async def test_runner_busy_shard_does_not_start_processor(): + store = Store(busy=True) + shard_runner = runner(store) + called = [] + + async def process(execution, stop): + called.append(execution) + + assert ( + await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, asyncio.Event()) + is False + ) + assert called == [] + assert store.released == [] + assert shard_runner.has_pending is False + + +@pytest.mark.asyncio +async def test_callback_runner_scopes_active_work_by_item_type(): + store = Store() + shard_runner = runner(store) + session_started = asyncio.Event() + release_session = asyncio.Event() + + async def session_process(execution, stop): + session_started.set() + await release_session.wait() + execution.ensure_active() + + async def watch_process(execution, stop): + execution.ensure_active() + + session_task = asyncio.create_task( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + session_process, + asyncio.Event(), + item_type="session", + ) + ) + try: + await asyncio.wait_for(session_started.wait(), timeout=2.0) + assert ( + await shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + watch_process, + asyncio.Event(), + item_type="watch", + ) + is True + ) + finally: + release_session.set() + await asyncio.wait_for(session_task, timeout=2.0) + + +@pytest.mark.asyncio +async def test_runner_does_not_acquire_after_stop(): + store = Store() + shard_runner = runner(store) + stop = asyncio.Event() + stop.set() + + async def process(execution, stop): + pytest.fail("A stopped runner must not invoke the processor") + + assert await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, stop) is False + assert store.acquired == [] + + +@pytest.mark.asyncio +async def test_stop_grace_allows_processor_to_finish_and_release(): + store = Store() + shard_runner = runner(store) + stop = asyncio.Event() + started = asyncio.Event() + observed = [] + + async def process(execution, current_stop): + started.set() + await current_stop.wait() + execution.ensure_active() + observed.append(execution.owner) + + run_task = asyncio.create_task( + shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, stop) + ) + try: + await asyncio.wait_for(started.wait(), timeout=2.0) + stop.set() + assert await asyncio.wait_for(run_task, timeout=2.0) is True + assert store.released == observed + assert shard_runner.has_pending is False + finally: + stop.set() + await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) + + +@pytest.mark.asyncio +async def test_renewal_loss_cancels_processor_during_stop_grace(): + store = Store(renew_outcome="lost", block_renew=True) + shard_runner = runner(store) + stop = asyncio.Event() + process_started = asyncio.Event() + stop_observed = asyncio.Event() + process_cancelled = asyncio.Event() + release_process = asyncio.Event() + + async def process(execution, current_stop): + process_started.set() + try: + await current_stop.wait() + stop_observed.set() + await release_process.wait() + except asyncio.CancelledError: + process_cancelled.set() + raise + + run_task = asyncio.create_task( + shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, stop) + ) + cancelled_task = asyncio.create_task(process_cancelled.wait()) + try: + await asyncio.wait_for(process_started.wait(), timeout=2.0) + await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) + stop.set() + await asyncio.wait_for(stop_observed.wait(), timeout=2.0) + await asyncio.sleep(0) + await asyncio.sleep(0) + store.allow_renew.set() + finished, _ = await asyncio.wait({cancelled_task}, timeout=0.3) + assert cancelled_task in finished, "Lease loss must not wait for the stop grace to expire" + await asyncio.wait_for(run_task, timeout=2.0) + assert store.released == [] + assert shard_runner.has_pending is False + finally: + stop.set() + store.allow_renew.set() + release_process.set() + cancelled_task.cancel() + await asyncio.gather(cancelled_task, return_exceptions=True) + await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) + + +@pytest.mark.asyncio +async def test_processor_exception_releases_active_lease(): + store = Store() + shard_runner = runner(store) + owners = [] + + async def process(execution, stop): + owners.append(execution.owner) + raise RuntimeError("processor failed") + + with pytest.raises(RuntimeError, match="processor failed"): + await asyncio.wait_for( + shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, asyncio.Event()), + timeout=2.0, + ) + assert store.released == owners + assert shard_runner.has_pending is False + + +@pytest.mark.asyncio +async def test_cancellation_resistant_processor_is_bounded_retained_and_blocks_same_role(): + store = Store() + shard_runner = runner(store, config(shutdown_timeout_seconds=0.02)) + stop = asyncio.Event() + started = asyncio.Event() + cancelled = asyncio.Event() + release_process = asyncio.Event() + process_refs = [] + + async def process(execution, current_stop): + process_refs.append(weakref.ref(asyncio.current_task())) + started.set() + while not release_process.is_set(): + try: + await release_process.wait() + except asyncio.CancelledError: + cancelled.set() + + async def completed(execution, current_stop): + execution.ensure_active() + + run_task = asyncio.create_task( + shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, stop) + ) + try: + await asyncio.wait_for(started.wait(), timeout=2.0) + stop.set() + await asyncio.wait_for(cancelled.wait(), timeout=2.0) + finished, _ = await asyncio.wait({run_task}, timeout=0.3) + assert run_task in finished, ( + "Cleanup must not wait indefinitely for cancellation resistance" + ) + with pytest.raises(StoreOperationError) as error: + run_task.result() + assert error.value.code == "shard_not_stopped" + assert shard_runner.has_pending is True + gc.collect() + assert process_refs[0]() is not None + assert not process_refs[0]().done() + assert await asyncio.wait_for(shard_runner.wait_pending(0.01), timeout=2.0) is False + acquisition_count = len(store.acquired) + with pytest.raises(StoreOperationError) as error: + await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 1, completed, asyncio.Event()) + assert error.value.code == "shard_not_stopped" + assert len(store.acquired) == acquisition_count + assert store.released == [] + # Pending work blocks only its own role. + assert ( + await shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + completed, + asyncio.Event(), + item_type="session", + ) + is True + ) + release_process.set() + assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True + assert shard_runner.has_pending is False + assert ( + await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, completed, asyncio.Event()) + is True + ) + assert len(store.released) == 2 + finally: + stop.set() + release_process.set() + await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) + assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True + + +@pytest.mark.asyncio +async def test_lease_loss_cancels_spawned_child_and_prevents_late_ack(): + store = Store(renew_outcome="lost", block_renew=True) + shard_runner = runner(store) + child_started = asyncio.Event() + child_cancelled = asyncio.Event() + child_finished = asyncio.Event() + release_child = asyncio.Event() + acked = [] + executions = [] + + async def process(execution, stop): + executions.append(execution) + + async def child(): + child_started.set() + try: + while not release_child.is_set(): + try: + await release_child.wait() + except asyncio.CancelledError: + child_cancelled.set() + execution.ensure_active() + acked.append("ack") + finally: + child_finished.set() + + execution.spawn(child) + await execution.wait(60.0, stop) + + run_task = asyncio.create_task( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + process, + asyncio.Event(), + item_type="session", + ) + ) + try: + await asyncio.wait_for(child_started.wait(), timeout=2.0) + await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) + store.allow_renew.set() + await asyncio.wait_for(child_cancelled.wait(), timeout=2.0) + finished, _ = await asyncio.wait({run_task}, timeout=0.3) + assert run_task in finished + with pytest.raises(StoreOperationError) as error: + run_task.result() + assert error.value.code == "shard_not_stopped" + assert shard_runner.has_pending is True + assert store.released == [] + called = [] + + async def forbidden_child(): + called.append(True) + + with pytest.raises(StoreOperationError): + executions[0].spawn(forbidden_child) + assert called == [] + release_child.set() + await asyncio.wait_for(child_finished.wait(), timeout=2.0) + assert await shard_runner.wait_pending(1.0) is True + assert acked == [] + assert store.released == [] + assert shard_runner.has_pending is False + finally: + store.allow_renew.set() + release_child.set() + await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) + assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("invalidation", ["lost", "revoked"]) +async def test_execution_wait_wakes_and_rechecks_invalidation(invalidation): + acquired = lease() + execution = ShardExecution(acquired) + wait_task = asyncio.create_task(execution.wait(60.0, asyncio.Event())) + try: + await asyncio.sleep(0) + getattr(acquired, invalidation).set() + with pytest.raises(StoreOperationError): + await asyncio.wait_for(wait_task, timeout=2.0) + finally: + getattr(acquired, invalidation).set() + await asyncio.wait_for(asyncio.gather(wait_task, return_exceptions=True), timeout=2.0) + + +@pytest.mark.asyncio +async def test_execution_wait_rechecks_local_deadline_without_renewal(): + acquired = lease() + acquired.valid_until_monotonic = time.monotonic() + 0.03 + execution = ShardExecution(acquired) + with pytest.raises(StoreOperationError): + await asyncio.wait_for(execution.wait(60.0, asyncio.Event()), timeout=2.0) + assert acquired.lost.is_set() diff --git a/tests/scheduling/expiry/test_models_and_keys.py b/tests/scheduling/expiry/test_models_and_keys.py new file mode 100644 index 0000000000..aa5def2e8f --- /dev/null +++ b/tests/scheduling/expiry/test_models_and_keys.py @@ -0,0 +1,102 @@ +import pytest + +from openviking.scheduling.expiry import ( + ExpiryError, + ExpiryMode, + ExpiryRegistration, + ItemIdentity, +) +from openviking.scheduling.expiry.keys import ( + ExpiryKeys, + decode_component, + decode_item_id, + encode_component, + encode_item_id, + shard_for, +) +from openviking.scheduling.expiry.models import ( + SCORE_TIME_BASE, + BucketRef, + OwnerRole, + bucket_minute_ms, + claim_layer_bounds, + decode_claim_score, + encode_claim_score, + encode_ready_claim_score, + metadata_ttl_ms, + validate_report, +) + + +def test_identity_round_trip_is_canonical_and_isolated(): + identity = ItemIdentity("session", "account:a", "用户/{x}", "session:1") + item_id = encode_item_id(identity) + assert decode_item_id(item_id) == identity + assert "用户" not in item_id + assert "{" not in item_id + assert shard_for(item_id, 512) == shard_for(item_id, 512) + + +@pytest.mark.parametrize("encoded", ["=", "YQ==", "***", "_"]) +def test_decode_component_rejects_non_canonical_values(encoded): + with pytest.raises(ExpiryError, match="encoded identity"): + decode_component(encoded) + + +def test_component_round_trip_preserves_unicode(): + value = "账户/with spaces:✓" + assert decode_component(encode_component(value)) == value + + +def test_timer_keys_only_use_a_control_hash_tag_where_required(): + keys = ExpiryKeys(encode_component("cluster/resource")) + assert keys.service_key == encode_component("cluster/resource") + ref = BucketRef("session", 7, 120_000) + assert "{" not in keys.source(7) + assert "{" not in keys.bucket(ref) + tag = keys.control_tag(7) + assert keys.progress(7) == f"resource_sche:process:consume_progress:{tag}" + assert keys.registry(7) == f"resource_consume:bucket_register:{tag}" + scheduler_owner = keys.owner(OwnerRole.SCHEDULER, 7) + callback_owner = keys.owner(OwnerRole.CALLBACK, 7, item_type="session") + assert scheduler_owner == f"resource_timer:owner:scheduler:{tag}" + assert callback_owner == f"resource_timer:owner:callback:session:{tag}" + + +def test_callback_owner_requires_item_type_scope(): + keys = ExpiryKeys("resource") + with pytest.raises(ValueError, match="requires an item type"): + keys.owner(OwnerRole.CALLBACK, 7) + with pytest.raises(ValueError, match="must not include"): + keys.owner(OwnerRole.SCHEDULER, 7, item_type="session") + + +@pytest.mark.parametrize( + ("timeout", "expected"), + [(3600, 86_400_000), (43_200, 86_400_000), (46_800, 93_600_000)], +) +def test_metadata_ttl(timeout, expected): + assert metadata_ttl_ms(timeout) == expected + + +def test_minute_and_claim_boundaries(): + assert bucket_minute_ms(60_000) == 60_000 + assert bucket_minute_ms(60_001) == 120_000 + score = encode_claim_score(2, 123) + assert decode_claim_score(score) == (2, 123) + assert claim_layer_bounds(0) == (0, SCORE_TIME_BASE - 1) + assert claim_layer_bounds(2) == ( + 2 * SCORE_TIME_BASE + 1, + 3 * SCORE_TIME_BASE - 1, + ) + assert encode_ready_claim_score(2) == 2 * SCORE_TIME_BASE + 1 + + +def test_report_rejects_fixed_mode_and_boolean_numbers(): + identity = ItemIdentity("session", "a", "u", "s") + with pytest.raises(ExpiryError) as exc: + validate_report(ExpiryRegistration(identity, 100, 1, ExpiryMode.FIXED)) + assert exc.value.code == "unsupported_mode" + with pytest.raises(ExpiryError) as exc: + validate_report(ExpiryRegistration(identity, True, 1)) + assert exc.value.code == "invalid_report" diff --git a/tests/scheduling/expiry/test_publisher.py b/tests/scheduling/expiry/test_publisher.py new file mode 100644 index 0000000000..7f172deaf9 --- /dev/null +++ b/tests/scheduling/expiry/test_publisher.py @@ -0,0 +1,306 @@ +import asyncio + +import pytest + +from openviking.scheduling.expiry import ItemIdentity +from openviking.scheduling.expiry.config import ExpiryConfig +from openviking.scheduling.expiry.keys import encode_item_id +from openviking.scheduling.expiry.metrics import ExpiryMetrics +from openviking.scheduling.expiry.models import ( + OwnerRole, + OwnerToken, + PublishResult, + ShardLease, + ShardWindow, + SourceCursor, + SourceItem, + SourcePage, + StoreOperationError, +) +from openviking.scheduling.expiry.publisher import ExpiryPublisher +from openviking.scheduling.expiry.shard_runner import ShardExecution + + +class Store: + def __init__(self, *, item=None, items=None, cleanup_results=None, upper_ms=60_000): + self.items = list(items if items is not None else (() if item is None else (item,))) + self.progress = 0 + self.upper_ms = upper_ms + self.staged = [] + self.published = [] + self.cleaned = [] + self.cleanup_results = list(cleanup_results or ()) + self.resets = [] + + async def read_scan_window(self, shard_id): + return ShardWindow(self.progress, self.upper_ms, self.upper_ms + 123) + + async def cleanup_source_page(self, shard_id, through_ms, *, limit): + self.cleaned.append((through_ms, limit)) + return self.cleanup_results.pop(0) if self.cleanup_results else 0 + + async def read_source_page( + self, shard_id, *, lower_exclusive_ms, upper_inclusive_ms, after, limit + ): + items = sorted(self.items, key=lambda item: (item.due_at_ms, item.item_id)) + items = [ + item + for item in items + if lower_exclusive_ms < item.due_at_ms <= upper_inclusive_ms + and (after is None or (item.due_at_ms, item.item_id) > (after.due_at_ms, after.item_id)) + ] + page_items = items[:limit] + next_cursor = None + if len(items) > limit: + last = page_items[-1] + next_cursor = SourceCursor(last.due_at_ms, last.item_id) + return SourcePage(tuple(page_items), next_cursor) + + async def reset_unpublished_minute(self, owner, **kwargs): + self.resets.append(kwargs) + return None + + async def stage_items(self, owner, bucket, item_ids): + self.staged.append((bucket, tuple(item_ids))) + + async def publish_minute(self, owner, **kwargs): + self.published.append(kwargs) + self.progress = kwargs["minute_ms"] + return PublishResult("published", kwargs["minute_ms"]) + + +def lease(): + return ShardLease( + OwnerToken(OwnerRole.SCHEDULER, 0, "token", 9999999999999), + asyncio.Event(), + float("inf"), + ) + + +def config(): + return ExpiryConfig.model_construct( + shard_count=1, + scan_batch_size=8, + cleanup_batch_size=8, + registry_page_size=8, + ) + + +@pytest.mark.asyncio +async def test_empty_scan_does_not_publish_or_advance_progress(): + store = Store() + scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) + await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) + assert store.published == [] + assert store.cleaned == [(0, 8)] + + +@pytest.mark.asyncio +async def test_non_empty_minute_is_fully_staged_before_publish(): + item_id = encode_item_id(ItemIdentity("session", "a", "u", "s")) + store = Store(item=SourceItem(item_id, 60_000)) + scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) + await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) + assert store.staged[0][1] == (item_id,) + assert store.published[0]["minute_ms"] == 60_000 + assert store.cleaned == [(0, 8), (60_000, 8), (60_000, 8)] + assert store.resets[0]["item_types"] == ("session",) + + +@pytest.mark.asyncio +async def test_source_cleanup_drains_bounded_pages(): + store = Store(cleanup_results=[8, 8, 3]) + scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) + + await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) + + assert store.cleaned == [(0, 8), (0, 8), (0, 8)] + + +@pytest.mark.asyncio +async def test_source_cleanup_stops_between_pages(): + stop = asyncio.Event() + + class StoppingStore(Store): + async def cleanup_source_page(self, shard_id, through_ms, *, limit): + removed = await super().cleanup_source_page(shard_id, through_ms, limit=limit) + stop.set() + return removed + + store = StoppingStore(cleanup_results=[8, 8]) + scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) + + await scheduler.process_shard(ShardExecution(lease()), stop) + + assert store.cleaned == [(0, 8)] + + +@pytest.mark.asyncio +async def test_source_cleanup_rechecks_lease_between_pages(): + shard_lease = lease() + + class LosingStore(Store): + async def cleanup_source_page(self, shard_id, through_ms, *, limit): + removed = await super().cleanup_source_page(shard_id, through_ms, limit=limit) + shard_lease.lost.set() + return removed + + store = LosingStore(cleanup_results=[8, 8]) + scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) + + with pytest.raises(StoreOperationError, match="Shard lease is no longer valid"): + await scheduler.process_shard(ShardExecution(shard_lease), asyncio.Event()) + + assert store.cleaned == [(0, 8)] + + +@pytest.mark.asyncio +async def test_unsupported_source_item_is_cleaned_without_advancing_progress(timer_logs): + item_id = encode_item_id(ItemIdentity("watch", "a", "u", "s")) + store = Store(item=SourceItem(item_id, 60_000)) + events = [] + metrics = ExpiryMetrics(lambda name, value, labels: events.append((name, value, labels))) + scheduler = ExpiryPublisher(store, None, config(), metrics) + + await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) + + assert store.staged == [] + assert store.published == [] + assert store.progress == 0 + assert store.cleaned == [(0, 8), (60_000, 8)] + assert events == [ + ( + "timer_source_invalid", + 1.0, + {"item_type": "", "outcome": "unsupported_type"}, + ) + ] + assert "reason=unsupported_type count=1" in timer_logs.text + assert item_id not in timer_logs.text + + +@pytest.mark.asyncio +async def test_all_invalid_minute_does_not_block_later_valid_minute(): + invalid_id = encode_item_id(ItemIdentity("watch", "a", "u", "invalid")) + valid_id = encode_item_id(ItemIdentity("session", "a", "u", "valid")) + store = Store( + items=( + SourceItem(invalid_id, 60_000), + SourceItem(valid_id, 120_000), + ), + upper_ms=120_000, + ) + scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) + + await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) + + assert store.staged[0][1] == (valid_id,) + assert store.published[0]["expected_progress_ms"] == 0 + assert store.published[0]["minute_ms"] == 120_000 + assert store.progress == 120_000 + assert store.cleaned == [(0, 8), (60_000, 8), (120_000, 8), (120_000, 8)] + + +@pytest.mark.asyncio +async def test_malformed_source_item_is_skipped_while_valid_item_is_published( + timer_logs, +): + valid_id = encode_item_id(ItemIdentity("session", "a", "u", "valid")) + malformed_id = "not-an-item-id" + store = Store( + items=( + SourceItem(valid_id, 60_000), + SourceItem(malformed_id, 60_000), + ) + ) + scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) + + await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) + + assert store.staged[0][1] == (valid_id,) + assert len(store.published[0]["buckets"]) == 1 + assert store.published[0]["minute_ms"] == 60_000 + assert store.cleaned == [(0, 8), (60_000, 8), (60_000, 8)] + assert "reason=invalid_identity count=1" in timer_logs.text + assert malformed_id not in timer_logs.text + + +@pytest.fixture +def timer_logs(caplog): + from openviking.scheduling.expiry.metrics import logger + + logger.addHandler(caplog.handler) + try: + yield caplog + finally: + logger.removeHandler(caplog.handler) + + +class Runner: + def __init__(self, *, stop_on_shard=0, error=None): + self.stop_on_shard = stop_on_shard + self.error = error + self.calls = [] + + async def run_if_acquired(self, role, shard_id, process, stop): + self.calls.append((role, shard_id, process)) + if shard_id == self.stop_on_shard: + stop.set() + if self.error is not None and shard_id == 0: + raise self.error + return False + + +@pytest.mark.asyncio +async def test_scheduler_worker_delegates_scheduler_role_and_stops_mid_shard_round(): + shard_runner = Runner(stop_on_shard=1) + timer_config = ExpiryConfig.model_construct(shard_count=3) + scheduler = ExpiryPublisher(Store(), shard_runner, timer_config, ExpiryMetrics()) + await asyncio.wait_for(scheduler.run(asyncio.Event()), timeout=2.0) + assert [(role, shard_id) for role, shard_id, _ in shard_runner.calls] == [ + (OwnerRole.SCHEDULER, 0), + (OwnerRole.SCHEDULER, 1), + ] + assert all(process == scheduler.process_shard for _, _, process in shard_runner.calls) + + +@pytest.mark.asyncio +async def test_scheduler_worker_does_not_enter_runner_when_already_stopped(): + shard_runner = Runner() + scheduler = ExpiryPublisher(Store(), shard_runner, config(), ExpiryMetrics()) + stop = asyncio.Event() + stop.set() + await scheduler.run(stop) + assert shard_runner.calls == [] + + +@pytest.mark.asyncio +async def test_scheduler_worker_propagates_cancellation_instead_of_retrying(): + shard_runner = Runner(error=asyncio.CancelledError()) + scheduler = ExpiryPublisher(Store(), shard_runner, config(), ExpiryMetrics()) + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(scheduler.run(asyncio.Event()), timeout=2.0) + assert len(shard_runner.calls) == 1 + + +@pytest.mark.asyncio +async def test_scheduler_worker_propagates_pending_work_instead_of_acquiring_next_shard(): + shard_runner = Runner( + stop_on_shard=1, + error=StoreOperationError("shard_not_stopped", "scheduler still running"), + ) + timer_config = ExpiryConfig.model_construct(shard_count=3) + scheduler = ExpiryPublisher(Store(), shard_runner, timer_config, ExpiryMetrics()) + with pytest.raises(StoreOperationError) as error: + await asyncio.wait_for(scheduler.run(asyncio.Event()), timeout=2.0) + assert error.value.code == "shard_not_stopped" + assert len(shard_runner.calls) == 1 + + +@pytest.mark.asyncio +async def test_scheduler_worker_continues_shard_round_after_processor_error(): + shard_runner = Runner(stop_on_shard=1, error=RuntimeError("scan failed")) + timer_config = ExpiryConfig.model_construct(shard_count=3) + scheduler = ExpiryPublisher(Store(), shard_runner, timer_config, ExpiryMetrics()) + await asyncio.wait_for(scheduler.run(asyncio.Event()), timeout=2.0) + assert [shard_id for _, shard_id, _ in shard_runner.calls] == [0, 1] diff --git a/tests/scheduling/expiry/test_redis_loop.py b/tests/scheduling/expiry/test_redis_loop.py new file mode 100644 index 0000000000..0c21f0e223 --- /dev/null +++ b/tests/scheduling/expiry/test_redis_loop.py @@ -0,0 +1,338 @@ +import asyncio +import threading + +import pytest + +from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig +from openviking.scheduling.expiry.models import StoreOperationError +from openviking.scheduling.expiry.runtime import ExpiryRuntime, RedisLoopExecutor +from openviking.scheduling.expiry.service import ExpiryRuntimeState + + +class Runner: + has_pending = False + + async def wait_pending(self, timeout): + return True + + +class Store: + def __init__(self, finished): + self.finished = finished + self.close_calls = 0 + self.closed_while_running = False + + async def close(self): + self.close_calls += 1 + self.closed_while_running |= not self.finished.is_set() + + +def runtime(io, store): + result = object.__new__(ExpiryRuntime) + result._config = ExpiryConfig.model_construct( + shutdown_timeout_seconds=0.03, + redis=ExpiryRedisConfig.model_construct(request_budget_seconds=0.03), + ) + result._state = ExpiryRuntimeState(storage_ready=True, accepting_calls=True) + result._stop = asyncio.Event() + result._tasks = {} + result._storage_started = True + result._storage_closed = False + result._store = store + result._io = io + result._shard_runner = Runner() + return result + + +@pytest.mark.asyncio +async def test_runtime_preserves_dependencies_until_cancelled_remote_operation_really_exits(): + io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=10.0) + await io.start() + thread = io._thread + loop = io._loop + started = threading.Event() + cancelled = threading.Event() + release = threading.Event() + finished = threading.Event() + + async def operation(): + started.set() + try: + while not release.is_set(): + try: + await asyncio.sleep(0.005) + except asyncio.CancelledError: + cancelled.set() + finally: + finished.set() + + store = Store(finished) + timer_runtime = runtime(io, store) + request = asyncio.create_task(io.run(operation)) + close_task = None + try: + assert await asyncio.to_thread(started.wait, 2.0) + request.cancel() + assert await asyncio.to_thread(cancelled.wait, 2.0) + close_task = asyncio.create_task(timer_runtime.close()) + done, _ = await asyncio.wait({close_task}, timeout=0.5) + assert close_task in done, "Shutdown must remain bounded even across the Redis loop" + with pytest.raises(StoreOperationError) as error: + close_task.result() + assert error.value.code == "shard_not_stopped" + assert store.close_calls == 0 + assert not store.closed_while_running + assert timer_runtime._storage_started + assert timer_runtime._state.storage_ready + assert io._thread is thread + assert io._loop is loop + assert thread.is_alive() + assert not finished.is_set() + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(request), timeout=2.0) + assert await asyncio.to_thread(finished.wait, 2.0) + await asyncio.wait_for(timer_runtime.close(), timeout=2.0) + assert store.close_calls == 1 + assert not store.closed_while_running + assert not thread.is_alive() + assert io._thread is None + assert not timer_runtime._storage_started + finally: + release.set() + await asyncio.wait_for(asyncio.gather(request, return_exceptions=True), timeout=2.0) + if close_task is not None: + await asyncio.wait_for(asyncio.gather(close_task, return_exceptions=True), timeout=2.0) + # Also join the captured thread: the old implementation incorrectly + # discarded the only executor reference after its timed join expired. + if thread.is_alive(): + assert await asyncio.to_thread(finished.wait, 2.0) + await io.close(2.0) + await asyncio.to_thread(thread.join, 2.0) + if io._thread is not None: + await io.close(2.0) + + +@pytest.mark.asyncio +async def test_executor_cancelled_wrapper_stays_pending_until_remote_task_exits(): + io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=10.0) + await io.start() + started = threading.Event() + cancelled = threading.Event() + release = threading.Event() + finished = threading.Event() + + async def operation(): + started.set() + try: + while not release.is_set(): + try: + await asyncio.sleep(0.005) + except asyncio.CancelledError: + cancelled.set() + finally: + finished.set() + + request = asyncio.create_task(io.run(operation)) + try: + assert await asyncio.to_thread(started.wait, 2.0) + request.cancel() + assert await asyncio.to_thread(cancelled.wait, 2.0) + done, _ = await asyncio.wait({request}, timeout=0.03) + assert request not in done, "A cancelled bridge is not proof of remote task termination" + # Repeated cancellation during cleanup must not orphan remote work. + request.cancel() + done, _ = await asyncio.wait({request}, timeout=0.03) + assert request not in done + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(request), timeout=2.0) + assert finished.is_set() + finally: + release.set() + await asyncio.wait_for(asyncio.gather(request, return_exceptions=True), timeout=2.0) + await io.close(2.0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("raises", [False, True]) +async def test_executor_propagates_remote_result_or_exception_and_closes_cleanly(raises): + io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) + await io.start() + thread = io._thread + + async def operation(): + if raises: + raise ValueError("remote error") + return 42 + + try: + if raises: + with pytest.raises(ValueError, match="remote error"): + await io.run(operation) + else: + assert await io.run(operation) == 42 + await io.quiesce(0.1) + with pytest.raises(StoreOperationError) as error: + await io.run(operation) + assert error.value.code == "not_started" + finally: + await io.close(2.0) + assert not thread.is_alive() + + +@pytest.mark.asyncio +async def test_cleanup_runs_nested_io_without_reopening_admission_or_deadlocking_gate(): + io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) + await io.start() + observed = [] + + async def operation(): + observed.append(asyncio.get_running_loop()) + return 42 + + async def cleanup(): + return await io.run(operation) + + try: + await io.quiesce(0.1) + assert await io.run_cleanup(cleanup, 1.0) == 42 + assert observed == [io._loop] + with pytest.raises(StoreOperationError) as error: + await io.run(operation) + assert error.value.code == "not_started" + finally: + await io.close(2.0) + + +@pytest.mark.asyncio +async def test_resistant_storage_cleanup_is_bounded_and_retains_thread_for_retry(): + io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) + await io.start() + thread = io._thread + started = threading.Event() + cancelled = threading.Event() + release = threading.Event() + finished = threading.Event() + + class ResistantStore(Store): + async def close(self): + self.close_calls += 1 + if finished.is_set(): + return + started.set() + try: + while not release.is_set(): + try: + await asyncio.sleep(0.005) + except asyncio.CancelledError: + cancelled.set() + finally: + finished.set() + + store = ResistantStore(finished) + timer_runtime = runtime(io, store) + close_task = asyncio.create_task(timer_runtime.close()) + try: + assert await asyncio.to_thread(started.wait, 2.0) + assert await asyncio.to_thread(cancelled.wait, 2.0) + done, _ = await asyncio.wait({close_task}, timeout=0.5) + assert close_task in done + with pytest.raises(StoreOperationError) as error: + close_task.result() + assert error.value.code == "shard_not_stopped" + assert timer_runtime._storage_started + assert timer_runtime._state.storage_ready + assert io._thread is thread + assert thread.is_alive() + assert not finished.is_set() + release.set() + assert await asyncio.to_thread(finished.wait, 2.0) + await asyncio.wait_for(timer_runtime.close(), timeout=2.0) + assert not thread.is_alive() + assert not timer_runtime._storage_started + finally: + release.set() + await asyncio.wait_for(asyncio.gather(close_task, return_exceptions=True), timeout=2.0) + assert await asyncio.to_thread(finished.wait, 2.0) + await io.close(2.0) + + +@pytest.mark.asyncio +async def test_request_timeout_never_reports_success_when_operation_suppresses_cancel(): + io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=0.03) + await io.start() + cancelled = threading.Event() + release = threading.Event() + finished = threading.Event() + + async def operation(): + try: + while not release.is_set(): + try: + await asyncio.sleep(0.005) + except asyncio.CancelledError: + cancelled.set() + return "late success" + finally: + finished.set() + + request = asyncio.create_task(io.run(operation)) + try: + assert await asyncio.to_thread(cancelled.wait, 2.0) + done, _ = await asyncio.wait({request}, timeout=0.03) + assert request not in done + release.set() + with pytest.raises(StoreOperationError) as error: + await asyncio.wait_for(request, timeout=2.0) + assert error.value.code == "redis_timeout" + assert error.value.outcome_unknown + assert finished.is_set() + finally: + release.set() + await asyncio.wait_for(asyncio.gather(request, return_exceptions=True), timeout=2.0) + await io.close(2.0) + + +@pytest.mark.asyncio +async def test_executor_retains_loop_and_thread_if_timed_join_does_not_finish(): + class Loop: + def __init__(self): + self.stops = 0 + + def is_running(self): + return True + + def stop(self): + self.stops += 1 + + def call_soon_threadsafe(self, callback): + callback() + + class Thread: + def __init__(self): + self.alive = True + self.budgets = [] + + def join(self, timeout): + self.budgets.append(timeout) + + def is_alive(self): + return self.alive + + io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) + loop, thread = Loop(), Thread() + io._loop, io._thread, io._accepting = loop, thread, True + with pytest.raises(StoreOperationError) as error: + await io.close(0.01) + assert error.value.code == "shard_not_stopped" + assert io._loop is loop + assert io._thread is thread + assert io._accepting is False + thread.alive = False + await io.close(0.01) + assert io._loop is None + assert io._thread is None + assert len(thread.budgets) == 2 + assert all(0.0 <= budget <= 0.01 for budget in thread.budgets) + assert loop.stops == 1 diff --git a/tests/scheduling/expiry/test_redis_store.py b/tests/scheduling/expiry/test_redis_store.py new file mode 100644 index 0000000000..85fd760e8d --- /dev/null +++ b/tests/scheduling/expiry/test_redis_store.py @@ -0,0 +1,411 @@ +from unittest.mock import AsyncMock + +import pytest + +from openviking.scheduling.expiry.config import ExpiryRedisConfig +from openviking.scheduling.expiry.keys import ExpiryKeys +from openviking.scheduling.expiry.metrics import ExpiryMetrics +from openviking.scheduling.expiry.models import ( + BucketItem, + BucketRef, + Claim, + OwnerRole, + OwnerToken, + RegistryCursor, + SourceCursor, + SourceItem, + StoreOperationError, + encode_claim_score, + encode_ready_claim_score, +) +from openviking.scheduling.expiry.redis_store import RedisExpiryStore + + +class InlineExecutor: + async def run(self, operation): + return await operation() + + +class FlakyClient: + def __init__(self, failures): + self.failures = list(failures) + self.calls = 0 + + async def get(self, _key): + self.calls += 1 + if self.failures: + raise self.failures.pop(0) + return "value" + + +class ScriptClient: + def __init__(self): + self.calls = [] + + async def evalsha(self, sha, key_count, *args): + self.calls.append(("evalsha", sha, key_count, args)) + error_type = type("NoScriptError", (RuntimeError,), {}) + raise error_type("No matching script. Please use EVAL.") + + async def eval(self, source, key_count, *args): + self.calls.append(("eval", source, key_count, args)) + return "loaded" + + +class ClusterTimeClient: + def __init__(self): + self.node = object() + self.key = None + self.target = None + + def get_node_from_key(self, key): + self.key = key + return self.node + + async def time(self, *, target_nodes): + self.target = target_nodes + return (123, 456_789) + + +def store(client): + result = RedisExpiryStore( + ExpiryRedisConfig(), ExpiryKeys("resource"), InlineExecutor(), ExpiryMetrics() + ) + result._client = client + return result + + +@pytest.mark.asyncio +async def test_safe_command_retries_transient_redis_failure(): + client = FlakyClient([ConnectionError("temporary")]) + + assert await store(client)._command("get", "key", retry="read") == "value" + assert client.calls == 2 + + +@pytest.mark.asyncio +async def test_uncertain_write_is_left_for_caller_to_resolve(): + client = FlakyClient([ConnectionError("temporary")]) + + with pytest.raises(StoreOperationError) as raised: + await store(client)._command("get", "key", retry="resolve_by_caller") + + assert client.calls == 1 + assert raised.value.outcome_unknown is True + + +@pytest.mark.asyncio +async def test_protocol_error_is_not_retried(): + client = FlakyClient([RuntimeError("wrong_key_type")]) + + with pytest.raises(StoreOperationError) as raised: + await store(client)._command("get", "key", retry="idempotent") + + assert client.calls == 1 + assert raised.value.code == "wrong_key_type" + + +@pytest.mark.asyncio +async def test_evalsha_falls_back_to_eval_on_cache_miss(): + client = ScriptClient() + timer_store = store(client) + + result = await timer_store._eval("upsert_resource", ["source"], ["item", 1], retry="read") + + assert result == "loaded" + assert [call[0] for call in client.calls] == ["evalsha", "eval"] + + +@pytest.mark.asyncio +async def test_bucket_reset_uses_single_key_cluster_commands(monkeypatch): + timer_store = store(FlakyClient([])) + monkeypatch.setattr(timer_store, "assert_owner", AsyncMock()) + monkeypatch.setattr(timer_store, "read_progress", AsyncMock(return_value=0)) + command = AsyncMock() + monkeypatch.setattr(timer_store, "_command", command) + + await timer_store.reset_unpublished_minute( + OwnerToken(OwnerRole.SCHEDULER, 3, "owner", 999), + minute_ms=60_000, + expected_progress_ms=0, + item_types=("session", "watch"), + ) + + assert [call.args[:2] for call in command.await_args_list] == [ + ("unlink", timer_store._keys.bucket(BucketRef("session", 3, 60_000))), + ("unlink", timer_store._keys.bucket(BucketRef("watch", 3, 60_000))), + ] + + +@pytest.mark.asyncio +async def test_source_cleanup_page_uses_single_key_atomic_script(monkeypatch): + timer_store = store(FlakyClient([])) + evaluate = AsyncMock(return_value=7) + monkeypatch.setattr(timer_store, "_eval", evaluate) + + removed = await timer_store.cleanup_source_page(3, 60_000, limit=128) + + assert removed == 7 + evaluate.assert_awaited_once_with( + "cleanup_source_page", + [timer_store._keys.source(3)], + [60_000, 128], + retry="idempotent", + ) + + +@pytest.mark.asyncio +async def test_registry_page_uses_score_cursor(monkeypatch): + timer_store = store(FlakyClient([])) + evaluate = AsyncMock(return_value='{"items":[["session:3:60000","70000"]],"next":null}') + monkeypatch.setattr(timer_store, "_eval", evaluate) + + page = await timer_store.read_registry_page( + 3, + after=RegistryCursor(65_000, "session:3:0"), + limit=128, + ) + + assert page.buckets == (BucketRef("session", 3, 60_000),) + evaluate.assert_awaited_once_with( + "read_zset_page", + [timer_store._keys.registry(3)], + [-1, "", 65_000, "session:3:0", 128], + retry="read", + ) + + +@pytest.mark.asyncio +async def test_waiting_bucket_can_defer_shared_registration(monkeypatch): + timer_store = store(FlakyClient([])) + evaluate = AsyncMock(return_value=1) + monkeypatch.setattr(timer_store, "_eval", evaluate) + bucket = BucketRef("session", 3, 60_000) + owner = OwnerToken( + OwnerRole.CALLBACK, + 3, + "owner", + 999, + "session", + ) + + assert ( + await timer_store.defer_registration( + owner, + bucket, + next_poll_at_ms=180_000, + ) + is True + ) + + evaluate.assert_awaited_once_with( + "defer_registration", + [ + timer_store._keys.registry(3), + timer_store._keys.owner( + OwnerRole.CALLBACK, + 3, + item_type="session", + ), + ], + ["owner", timer_store._keys.registry_member(bucket), 180_000], + retry="idempotent", + ) + + +@pytest.mark.asyncio +async def test_redis_time_is_returned_in_milliseconds(monkeypatch): + timer_store = store(FlakyClient([])) + command = AsyncMock(return_value=(123, 456_789)) + monkeypatch.setattr(timer_store, "_command", command) + + assert await timer_store.read_time_ms() == 123_456 + command.assert_awaited_once_with("time") + + +@pytest.mark.asyncio +async def test_cluster_time_is_routed_to_key_primary(): + client = ClusterTimeClient() + timer_store = store(client) + timer_store._config = ExpiryRedisConfig.model_construct(mode="cluster") + + assert await timer_store._read_time_ms_for_key("source-key") == 123_456 + assert client.key == "source-key" + assert client.target is client.node + + +@pytest.mark.asyncio +async def test_source_first_page_uses_native_score_range(monkeypatch): + timer_store = store(FlakyClient([])) + command = AsyncMock(return_value=[("a", 100.0), ("b", 100.0), ("c", 200.0)]) + monkeypatch.setattr(timer_store, "_command", command) + + page = await timer_store.read_source_page( + 3, + lower_exclusive_ms=0, + upper_inclusive_ms=1_000, + after=None, + limit=2, + ) + + assert page.items == (SourceItem("a", 100), SourceItem("b", 100)) + assert page.next_cursor == SourceCursor(100, "b") + command.assert_awaited_once_with( + "zrangebyscore", + timer_store._keys.source(3), + "(0", + 1_000, + start=0, + num=3, + withscores=True, + ) + + +@pytest.mark.asyncio +async def test_source_continuation_uses_stable_cursor_script(monkeypatch): + timer_store = store(FlakyClient([])) + evaluate = AsyncMock(return_value='{"items":[["c","200"]],"next":["200","c"]}') + monkeypatch.setattr(timer_store, "_eval", evaluate) + + page = await timer_store.read_source_page( + 3, + lower_exclusive_ms=0, + upper_inclusive_ms=1_000, + after=SourceCursor(100, "b"), + limit=2, + ) + + assert page.items == (SourceItem("c", 200),) + assert page.next_cursor == SourceCursor(200, "c") + evaluate.assert_awaited_once_with( + "read_zset_page", + [timer_store._keys.source(3)], + [0, 1_000, 100, "b", 2], + retry="read", + ) + + +@pytest.mark.asyncio +async def test_bucket_page_reads_one_attempt_layer_from_offset_zero(monkeypatch): + timer_store = store(FlakyClient([])) + command = AsyncMock(return_value=[("item", float(encode_claim_score(2, 123)))]) + monkeypatch.setattr(timer_store, "_command", command) + bucket = BucketRef("session", 3, 60_000) + + page = await timer_store.read_bucket_page( + bucket, + attempt=2, + limit=128, + ) + + assert page.items == (BucketItem("item", encode_claim_score(2, 123)),) + lower = encode_claim_score(2, 1) + upper = encode_claim_score(3, 1) - 2 + command.assert_awaited_once_with( + "zrangebyscore", + timer_store._keys.bucket(bucket), + lower, + upper, + start=0, + num=128, + withscores=True, + ) + + +@pytest.mark.asyncio +async def test_claim_and_retry_use_score_compare_and_set(monkeypatch): + timer_store = store(FlakyClient([])) + evaluate = AsyncMock(side_effect=[1, 1]) + monkeypatch.setattr(timer_store, "_eval", evaluate) + bucket = BucketRef("session", 3, 60_000) + old_score = encode_claim_score(1, 123) + + claim = await timer_store.claim( + bucket, + BucketItem("item", old_score), + attempt=1, + started_at_seconds=200, + ) + + assert claim == Claim(bucket, "item", encode_claim_score(1, 200)) + evaluate.assert_awaited_once_with( + "cas_zset_member", + [timer_store._keys.bucket(bucket)], + ["set", "item", old_score, encode_claim_score(1, 200)], + retry="resolve_by_caller", + ) + + evaluate.reset_mock() + assert await timer_store.advance_retry(claim) is True + evaluate.assert_awaited_once_with( + "cas_zset_member", + [timer_store._keys.bucket(bucket)], + ["set", "item", claim.claim_score, encode_ready_claim_score(2)], + retry="resolve_by_caller", + ) + + +@pytest.mark.asyncio +async def test_ack_uses_score_compare_and_delete(monkeypatch): + timer_store = store(FlakyClient([])) + evaluate = AsyncMock(return_value=1) + monkeypatch.setattr(timer_store, "_eval", evaluate) + claim = Claim(BucketRef("session", 3, 60_000), "item", encode_claim_score(1, 123)) + + assert await timer_store.ack(claim) is True + evaluate.assert_awaited_once_with( + "cas_zset_member", + [timer_store._keys.bucket(claim.bucket)], + ["delete", "item", claim.claim_score, ""], + retry="idempotent", + ) + + +@pytest.mark.asyncio +async def test_owner_acquire_uses_native_set_nx_px(monkeypatch): + timer_store = store(FlakyClient([])) + command = AsyncMock(return_value=True) + monkeypatch.setattr(timer_store, "_command", command) + + owner = await timer_store.try_acquire_owner( + OwnerRole.CALLBACK, + 3, + "token", + 120, + item_type="session", + ) + + assert owner is not None + assert owner.token == "token" + command.assert_awaited_once_with( + "set", + timer_store._keys.owner(OwnerRole.CALLBACK, 3, item_type="session"), + "token", + nx=True, + px=120_000, + retry="resolve_by_caller", + ) + + +@pytest.mark.asyncio +async def test_owner_renew_and_release_share_compare_script(monkeypatch): + timer_store = store(FlakyClient([])) + evaluate = AsyncMock(side_effect=[1, 1]) + monkeypatch.setattr(timer_store, "_eval", evaluate) + owner = OwnerToken(OwnerRole.SCHEDULER, 3, "token", 1) + key = timer_store._keys.owner(OwnerRole.SCHEDULER, 3) + + assert await timer_store.renew_owner(owner, 120) is not None + assert await timer_store.release_owner(owner) is True + assert evaluate.await_args_list[0].args == ( + "compare_owner", + [key], + ["renew", "token", 120_000], + ) + assert evaluate.await_args_list[0].kwargs == {"retry": "resolve_by_caller"} + assert evaluate.await_args_list[1].args == ( + "compare_owner", + [key], + ["release", "token", ""], + ) + assert evaluate.await_args_list[1].kwargs == {"retry": "idempotent"} diff --git a/tests/scheduling/expiry/test_runtime.py b/tests/scheduling/expiry/test_runtime.py new file mode 100644 index 0000000000..ea3bfddd57 --- /dev/null +++ b/tests/scheduling/expiry/test_runtime.py @@ -0,0 +1,338 @@ +import asyncio + +import pytest + +from openviking.scheduling.expiry import runtime as runtime_module +from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig +from openviking.scheduling.expiry.models import StoreOperationError +from openviking.scheduling.expiry.runtime import ExpiryRuntime +from openviking.scheduling.expiry.service import ExpiryRuntimeState +from openviking.service.core import OpenVikingService + + +class Store: + def __init__(self): + self.close_calls = 0 + + async def close(self): + self.close_calls += 1 + + +class IO: + def __init__(self): + self.close_budgets = [] + + async def quiesce(self, timeout_seconds): + pass + + async def run_cleanup(self, operation, timeout_seconds): + return await operation() + + async def close(self, timeout_seconds): + self.close_budgets.append(timeout_seconds) + + +class Runner: + def __init__(self, *, pending=False): + self.settled = asyncio.Event() + if not pending: + self.settled.set() + self.wait_budgets = [] + + @property + def has_pending(self): + return not self.settled.is_set() + + async def wait_pending(self, timeout): + self.wait_budgets.append(timeout) + if not self.has_pending: + return True + waiter = asyncio.create_task(self.settled.wait()) + try: + finished, _ = await asyncio.wait({waiter}, timeout=timeout) + return waiter in finished + finally: + waiter.cancel() + await asyncio.gather(waiter, return_exceptions=True) + + +def runtime(*, pending=False): + # Exercise the actual shutdown helpers without constructing Redis clients, + # starting an I/O thread, or replacing the helpers under test. + result = object.__new__(ExpiryRuntime) + result._config = ExpiryConfig.model_construct( + shutdown_timeout_seconds=0.02, + redis=ExpiryRedisConfig.model_construct(request_budget_seconds=0.03), + ) + result._state = ExpiryRuntimeState() + result._state.accepting_calls = True + result._state.storage_ready = True + result._stop = asyncio.Event() + result._tasks = {} + result._storage_started = True + result._storage_closed = False + result._store = Store() + result._io = IO() + result._shard_runner = Runner(pending=pending) + return result + + +@pytest.mark.asyncio +async def test_start_roles_creates_one_callback_task_per_item_type(monkeypatch): + created = [] + + class Consumer: + def __init__(self, store, runner, item_type, binding, config, metrics): + self.item_type = item_type + created.append(item_type) + + async def run(self, stop): + await stop.wait() + + class Registry: + def freeze(self): + return {"session": object(), "watch": object()} + + timer_runtime = object.__new__(ExpiryRuntime) + timer_runtime._storage_started = True + timer_runtime._state = ExpiryRuntimeState() + timer_runtime._config = ExpiryConfig.model_construct( + scheduler_enabled=False, + callback_enabled=True, + ) + timer_runtime._registry = Registry() + timer_runtime._stop = asyncio.Event() + timer_runtime._store = object() + timer_runtime._shard_runner = object() + timer_runtime._metrics = object() + timer_runtime._scheduler = None + timer_runtime._consumers = {} + spawned = {} + + def spawn_role(name, operation): + spawned[name] = operation + + timer_runtime._spawn_role = spawn_role + monkeypatch.setattr(runtime_module, "ExpiryConsumer", Consumer) + + await timer_runtime.start_roles() + + assert created == ["session", "watch"] + assert set(timer_runtime._consumers) == {"session", "watch"} + assert set(spawned) == {"callback:session", "callback:watch"} + for operation in spawned.values(): + operation.close() + + +@pytest.mark.asyncio +async def test_stop_roles_clears_cooperative_tasks_before_closing_dependencies(): + timer_runtime = runtime() + started = asyncio.Event() + finished = asyncio.Event() + + async def role(): + started.set() + await timer_runtime._stop.wait() + finished.set() + + role_task = asyncio.create_task(role()) + timer_runtime._tasks["callback"] = role_task + try: + await asyncio.wait_for(started.wait(), timeout=2.0) + await asyncio.wait_for(timer_runtime.stop_roles(), timeout=2.0) + assert finished.is_set() + assert role_task.done() + assert timer_runtime._tasks == {} + assert timer_runtime._state.accepting_calls is False + assert timer_runtime._store.close_calls == 0 + assert timer_runtime._io.close_budgets == [] + await asyncio.wait_for(timer_runtime.close(), timeout=2.0) + assert timer_runtime._store.close_calls == 1 + assert timer_runtime._io.close_budgets == [0.02] + assert timer_runtime._storage_started is False + assert timer_runtime._state.storage_ready is False + finally: + timer_runtime._stop.set() + await asyncio.wait_for(asyncio.gather(role_task, return_exceptions=True), timeout=2.0) + + +@pytest.mark.asyncio +async def test_resistant_role_has_bounded_stop_and_retains_tasks_and_open_dependencies(): + timer_runtime = runtime() + started = asyncio.Event() + cancelled = asyncio.Event() + release_role = asyncio.Event() + + async def role(): + started.set() + while not release_role.is_set(): + try: + await release_role.wait() + except asyncio.CancelledError: + cancelled.set() + + role_task = asyncio.create_task(role()) + timer_runtime._tasks["callback"] = role_task + stop_task = None + close_task = None + try: + await asyncio.wait_for(started.wait(), timeout=2.0) + stop_task = asyncio.create_task(timer_runtime.stop_roles()) + await asyncio.wait_for(cancelled.wait(), timeout=2.0) + finished, _ = await asyncio.wait({stop_task}, timeout=0.3) + assert stop_task in finished, ( + "Role shutdown must not join cancellation resistance indefinitely" + ) + with pytest.raises(StoreOperationError) as error: + stop_task.result() + assert error.value.code == "shard_not_stopped" + assert timer_runtime._tasks == {"callback": role_task} + assert not role_task.done() + assert timer_runtime._state.accepting_calls is False + close_task = asyncio.create_task(timer_runtime.close()) + finished, _ = await asyncio.wait({close_task}, timeout=0.3) + assert close_task in finished + with pytest.raises(StoreOperationError) as error: + close_task.result() + assert error.value.code == "shard_not_stopped" + assert timer_runtime._tasks == {"callback": role_task} + assert timer_runtime._store.close_calls == 0 + assert timer_runtime._io.close_budgets == [] + assert timer_runtime._storage_started is True + assert timer_runtime._state.storage_ready is True + finally: + # Release resistant work before joining shutdown tasks; a cancellation + # watchdog must never become an unbounded join of the resistant role. + release_role.set() + tasks = [task for task in (role_task, stop_task, close_task) if task is not None] + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=2.0) + await asyncio.wait_for(timer_runtime.close(), timeout=2.0) + assert timer_runtime._tasks == {} + assert timer_runtime._store.close_calls == 1 + assert timer_runtime._io.close_budgets == [0.02] + + +@pytest.mark.asyncio +async def test_runner_pending_blocks_close_after_role_task_has_exited(): + timer_runtime = runtime(pending=True) + + async def role(): + return None + + role_task = asyncio.create_task(role()) + await role_task + timer_runtime._tasks["callback"] = role_task + close_task = asyncio.create_task(timer_runtime.close()) + try: + finished, _ = await asyncio.wait({close_task}, timeout=0.3) + assert close_task in finished + with pytest.raises(StoreOperationError) as error: + close_task.result() + assert error.value.code == "shard_not_stopped" + assert timer_runtime._tasks == {} + assert timer_runtime._shard_runner.has_pending is True + assert timer_runtime._store.close_calls == 0 + assert timer_runtime._io.close_budgets == [] + assert timer_runtime._storage_started is True + assert timer_runtime._state.storage_ready is True + finally: + timer_runtime._shard_runner.settled.set() + await asyncio.wait_for(asyncio.gather(close_task, return_exceptions=True), timeout=2.0) + await asyncio.wait_for(timer_runtime.close(), timeout=2.0) + assert timer_runtime._store.close_calls == 1 + assert timer_runtime._io.close_budgets == [0.02] + assert timer_runtime._storage_started is False + + +@pytest.mark.asyncio +async def test_runtime_retries_thread_join_without_resubmitting_closed_storage_cleanup(): + timer_runtime = runtime() + + class SlowThreadIO(IO): + def __init__(self): + super().__init__() + self.stopping = False + + async def run_cleanup(self, operation, timeout_seconds): + if self.stopping: + raise StoreOperationError("not_started", "Redis loop already stopping") + return await operation() + + async def close(self, timeout_seconds): + self.close_budgets.append(timeout_seconds) + if not self.stopping: + self.stopping = True + raise StoreOperationError("shard_not_stopped", "I/O thread still exiting") + + timer_runtime._io = SlowThreadIO() + with pytest.raises(StoreOperationError) as error: + await timer_runtime.close() + assert error.value.code == "shard_not_stopped" + assert timer_runtime._storage_started + assert timer_runtime._state.storage_ready + assert timer_runtime._store.close_calls == 1 + await timer_runtime.close() + assert timer_runtime._store.close_calls == 1 + assert timer_runtime._io.close_budgets == [0.02, 0.02] + assert not timer_runtime._storage_started + assert not timer_runtime._state.storage_ready + + +class SessionService: + def __init__(self): + self.adapter = object() + self.adapter_updates = [] + + def set_session_timer_adapter(self, adapter): + self.adapter_updates.append(adapter) + self.adapter = adapter + + +class Runtime: + def __init__(self, *, error=None): + self.error = error + self.close_calls = 0 + + async def close(self): + self.close_calls += 1 + if self.error is not None: + raise self.error + + +def service(expiry_runtime): + result = object.__new__(OpenVikingService) + result._expiry_runtime = expiry_runtime + result._session_service = SessionService() + return result + + +@pytest.mark.asyncio +async def test_core_retains_runtime_and_adapter_when_timer_work_has_not_stopped(): + shutdown_error = StoreOperationError("shard_not_stopped", "callback still running") + expiry_runtime = Runtime(error=shutdown_error) + core = service(expiry_runtime) + original_adapter = core._session_service.adapter + with pytest.raises(StoreOperationError) as error: + await core._stop_expiry() + assert error.value is shutdown_error + assert expiry_runtime.close_calls == 1 + assert core._expiry_runtime is expiry_runtime + assert core._session_service.adapter is original_adapter + assert core._session_service.adapter_updates == [] + expiry_runtime.error = None + await core._stop_expiry() + assert expiry_runtime.close_calls == 2 + assert core._expiry_runtime is None + assert core._session_service.adapter is None + assert core._session_service.adapter_updates == [None] + + +@pytest.mark.asyncio +async def test_core_clears_runtime_and_detaches_adapter_after_successful_close(): + expiry_runtime = Runtime() + core = service(expiry_runtime) + await core._stop_expiry() + assert expiry_runtime.close_calls == 1 + assert core._expiry_runtime is None + assert core._session_service.adapter is None + assert core._session_service.adapter_updates == [None] diff --git a/tests/scheduling/expiry/test_session_timeout_service.py b/tests/scheduling/expiry/test_session_timeout_service.py new file mode 100644 index 0000000000..589a4f11c4 --- /dev/null +++ b/tests/scheduling/expiry/test_session_timeout_service.py @@ -0,0 +1,637 @@ +# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. +# SPDX-License-Identifier: AGPL-3.0 +"""Service-level idle checks and best-effort acceptance metadata recording.""" + +import asyncio +import copy +import json +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from openviking.core.namespace import canonical_session_uri +from openviking.scheduling.expiry.config import ExpiryConfig +from openviking.service.session_service import SessionService +from openviking.session import Session +from openviking.session.session import SessionMeta +from openviking.utils.time_utils import parse_iso_datetime +from openviking_cli.exceptions import NotFoundError +from openviking_cli.utils.config.memory_config import SessionAutoCommitConfig + + +class MetadataFS: + """Minimal storage with observable locking and controllable concurrent writes.""" + + def __init__(self, ctx): + self.session_uri = canonical_session_uri(ctx, "s") + self.meta_uri = f"{self.session_uri}/.meta.json" + self.meta = { + "auto_commit_policy": {"idle_timeout_seconds": 60, "keep_recent_count": 12}, + "last_message_at": "2026-01-01T00:00:00Z", + "last_idle_commit_accepted_at": "", + "message_count": 1, + "commit_count": 3, + "updated_at": "2026-01-01T00:00:00Z", + } + self.exists = True + self.locked = False + self.events = [] + self.writes = [] + self.on_acquire = None + self.read_error = None + self.write_error = None + self.raw_meta = None + self._async_agfs = SimpleNamespace( + pathlock_acquire_exact=self.acquire, + pathlock_release=self.release, + ) + + def _uri_to_path(self, uri, *, ctx): + assert uri == self.session_uri + return "/local/session/s" + + async def acquire(self, path, **kwargs): + assert path == "/local/session/s" + assert not self.locked + self.locked = True + self.events.append("acquire") + if self.on_acquire is not None: + self.on_acquire() + return "lease" + + async def release(self, lease): + assert lease == "lease" + assert self.locked + self.events.append("release") + self.locked = False + + async def stat(self, uri, *, ctx, **kwargs): + assert uri == self.session_uri + self.events.append("stat") + if not self.exists: + raise NotFoundError("s", "session") + return {"isDir": True} + + async def read_file(self, uri, *, ctx, **kwargs): + assert uri == self.meta_uri + assert self.locked, "Acceptance metadata must be re-read under the session lock" + self.events.append("read") + if not self.exists: + raise NotFoundError("s", "session") + if self.read_error is not None: + raise self.read_error + return self.raw_meta if self.raw_meta is not None else json.dumps(self.meta) + + async def write_file(self, uri, content, *, ctx, **kwargs): + assert uri == self.meta_uri + assert self.locked + assert kwargs.get("lease_ref") == "lease", "Metadata writes must reuse the held lease" + assert self.exists, "Recording acceptance must not recreate a deleted session" + self.events.append("write") + if self.write_error is not None: + raise self.write_error + self.meta = json.loads(content) + self.writes.append(copy.deepcopy(self.meta)) + + +@pytest.fixture +def idle_session(): + return SimpleNamespace( + session_id="s", + messages=[object()], + meta=SessionMeta( + session_id="s", + auto_commit_policy={"idle_timeout_seconds": 60, "keep_recent_count": 12}, + last_message_at="2026-01-01T00:00:00Z", + message_count=1, + keep_recent_count=12, + ), + commit_async=AsyncMock(spec=Session.commit_async.__get__(object(), Session)), + ) + + +@pytest.fixture +def timeout_service(idle_session, request_context, monkeypatch): + class FrozenDatetime(datetime): + @classmethod + def now(cls, tz=None): + value = cls(2026, 1, 1, 0, 2, tzinfo=timezone.utc) + return value.astimezone(tz) if tz is not None else value.replace(tzinfo=None) + + monkeypatch.setattr("openviking.service.session_service.datetime", FrozenDatetime) + fs = MetadataFS(request_context) + service = SessionService(viking_fs=fs) + service.get = AsyncMock(spec=service.get, return_value=idle_session) + return service, fs + + +@pytest.fixture +def now(): + return datetime(2026, 1, 1, 0, 2, tzinfo=timezone.utc) + + +@pytest.mark.parametrize("policy", [None, {"idle_timeout_seconds": 0}]) +def test_timeout_check_disabled_policy_is_pure(idle_session, now, policy): + idle_session.meta.auto_commit_policy = policy + idle_session.meta.last_message_at = "" + before = copy.deepcopy(idle_session.meta.to_dict()) + + result = SessionService._check_timeout_commit(idle_session, now) + + assert result == {"status": "skipped", "reason": "idle_disabled"} + assert idle_session.meta.to_dict() == before + idle_session.commit_async.assert_not_called() + + +def test_timeout_check_empty_messages_precedes_cooldown_and_activity(idle_session, now): + idle_session.messages = [] + idle_session.meta.message_count = 99 + idle_session.meta.last_idle_commit_accepted_at = now.isoformat() + idle_session.meta.last_message_at = "" + + assert SessionService._check_timeout_commit(idle_session, now) == { + "status": "skipped", + "reason": "no_messages", + } + + +def test_timeout_check_uses_messages_not_keep_window_or_pending_tokens(idle_session, now): + idle_session.meta.message_count = 0 + idle_session.meta.pending_tokens = 0 + idle_session.meta.keep_recent_count = 100 + + assert SessionService._check_timeout_commit(idle_session, now) is None + + +@pytest.mark.parametrize("offset_ms", [-1, 0, 1]) +@pytest.mark.parametrize( + "last_message_at", + ["2026-01-01T00:01:00Z", "2026-01-01T00:01:00", "2026-01-01T08:01:00+08:00"], +) +def test_timeout_check_activity_boundary(idle_session, now, offset_ms, last_message_at): + idle_session.meta.last_message_at = last_message_at + before = copy.deepcopy(idle_session.meta.to_dict()) + result = SessionService._check_timeout_commit( + idle_session, now + timedelta(milliseconds=offset_ms) + ) + + if offset_ms < 0: + assert result == { + "status": "deferred", + "reason": "idle_activity", + "timeout_seconds": 60, + "due_at_ms": int(now.timestamp() * 1000), + } + else: + assert result is None + assert idle_session.meta.to_dict() == before + + +@pytest.mark.parametrize("offset_ms", [-1, 0, 1]) +def test_timeout_check_cooldown_boundary(idle_session, now, offset_ms): + idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:01:00Z" + checked_at = now + timedelta(milliseconds=offset_ms) + + result = SessionService._check_timeout_commit(idle_session, checked_at) + + if offset_ms < 0: + assert result == { + "status": "deferred", + "reason": "idle_cooldown", + "timeout_seconds": 60, + "due_at_ms": int(checked_at.timestamp() * 1000) + 60_000, + } + else: + assert result is None + + +def test_timeout_check_cooldown_uses_current_timeout_and_precedes_activity(idle_session, now): + idle_session.meta.auto_commit_policy = {"idle_timeout_seconds": 120} + idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:00:30Z" + idle_session.meta.last_message_at = "2026-01-01T00:01:59Z" + idle_session.meta.last_auto_commit_at = "2000-01-01T00:00:00Z" + + assert SessionService._check_timeout_commit(idle_session, now) == { + "status": "deferred", + "reason": "idle_cooldown", + "timeout_seconds": 120, + "due_at_ms": int(now.timestamp() * 1000) + 120_000, + } + + +def test_timeout_check_legacy_auto_commit_time_does_not_start_idle_cooldown(idle_session, now): + idle_session.meta.last_auto_commit_at = now.isoformat() + + assert SessionService._check_timeout_commit(idle_session, now) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("working_memory_enabled", [None, False, True]) +async def test_timeout_commit_delegates_original_commit_and_records_after_acceptance( + timeout_service, idle_session, request_context, monkeypatch, working_memory_enabled +): + service, fs = timeout_service + idle_session.meta.memory_policy = ( + None + if working_memory_enabled is None + else {"working_memory": {"enabled": working_memory_enabled}} + ) + result = {"status": "accepted", "archived": True, "task_id": "task", "archive_uri": "archive"} + accepted_at = "2026-01-01T00:03:00.123Z" + + async def commit(**kwargs): + fs.events.append("commit") + return result + + clock_calls = 0 + + def acceptance_clock(): + nonlocal clock_calls + if clock_calls == 0: + assert fs.events == ["commit"], "Acceptance timestamp is sampled after commit returns" + clock_calls += 1 + return accepted_at + + idle_session.commit_async.side_effect = commit + monkeypatch.setattr( + "openviking.service.session_service.get_current_timestamp", acceptance_clock + ) + + actual = await service.commit_on_timeout("s", request_context) + + assert actual is result + service.get.assert_awaited_once_with("s", request_context, auto_create=False) + idle_session.commit_async.assert_awaited_once_with( + keep_recent_count=0, + persist_keep_recent_count=False, + record_auto_commit_success=True, + ) + assert fs.meta["last_idle_commit_accepted_at"] == accepted_at + assert len(fs.writes) == 1 + assert fs.events.index("commit") < fs.events.index("acquire") < fs.events.index("read") + assert fs.events[-1] == "release" + + +@pytest.mark.asyncio +async def test_timeout_commit_does_not_recheck_activity_after_outer_check( + timeout_service, idle_session, request_context +): + service, fs = timeout_service + result = {"status": "accepted", "archived": True} + + async def append_then_original_commit(**kwargs): + idle_session.messages.append(object()) + idle_session.meta.last_message_at = datetime.now(timezone.utc).isoformat() + return result + + idle_session.commit_async.side_effect = append_then_original_commit + + assert await service.commit_on_timeout("s", request_context) is result + assert len(idle_session.messages) == 2 + idle_session.commit_async.assert_awaited_once_with( + keep_recent_count=0, + persist_keep_recent_count=False, + record_auto_commit_success=True, + ) + assert len(fs.writes) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reason", ["idle_disabled", "no_messages"]) +async def test_timeout_commit_outer_skip_does_not_commit_or_record( + timeout_service, idle_session, request_context, reason +): + service, fs = timeout_service + if reason == "idle_disabled": + idle_session.meta.auto_commit_policy = None + else: + idle_session.messages = [] + + assert await service.commit_on_timeout("s", request_context) == { + "status": "skipped", + "reason": reason, + } + idle_session.commit_async.assert_not_awaited() + assert fs.events == [] + assert fs.writes == [] + + +@pytest.mark.asyncio +async def test_timeout_commit_skipped_original_commit_does_not_record( + timeout_service, idle_session, request_context +): + service, fs = timeout_service + result = {"status": "skipped", "reason": "no_messages", "archived": False} + idle_session.commit_async.return_value = result + + assert await service.commit_on_timeout("s", request_context) is result + idle_session.commit_async.assert_awaited_once() + assert fs.events == [] + assert fs.writes == [] + + +@pytest.mark.asyncio +async def test_timeout_commit_not_found_loading_is_skipped(timeout_service, request_context): + service, fs = timeout_service + service.get.side_effect = NotFoundError("s", "session") + + assert await service.commit_on_timeout("s", request_context) == { + "status": "skipped", + "reason": "not_found", + } + service.get.assert_awaited_once_with("s", request_context, auto_create=False) + assert fs.events == [] + + +@pytest.mark.asyncio +async def test_timeout_commit_other_load_errors_propagate(timeout_service, request_context): + service, fs = timeout_service + error = RuntimeError("metadata storage unavailable") + service.get.side_effect = error + + with pytest.raises(RuntimeError, match="metadata storage unavailable") as caught: + await service.commit_on_timeout("s", request_context) + + assert caught.value is error + assert fs.events == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("not_found", [False, True]) +async def test_timeout_commit_commit_errors_propagate_without_recording( + timeout_service, idle_session, request_context, not_found +): + service, fs = timeout_service + error = NotFoundError("archive", "archive") if not_found else RuntimeError("commit failed") + idle_session.commit_async.side_effect = error + + with pytest.raises(type(error)) as caught: + await service.commit_on_timeout("s", request_context) + + assert caught.value is error + idle_session.commit_async.assert_awaited_once() + assert fs.events == [] + + +@pytest.fixture +def service_logs(caplog): + from openviking.service.session_service import logger + + logger.addHandler(caplog.handler) + try: + yield caplog + finally: + logger.removeHandler(caplog.handler) + + +@pytest.mark.asyncio +async def test_timeout_commit_metadata_write_failure_still_returns_accepted( + timeout_service, idle_session, request_context, service_logs +): + service, fs = timeout_service + fs.write_error = RuntimeError("acceptance metadata write failed") + result = {"status": "accepted", "archived": True, "task_id": "task"} + idle_session.commit_async.return_value = result + + assert await service.commit_on_timeout("s", request_context) is result + assert "write" in fs.events + assert fs.events[-1] == "release" + assert fs.writes == [] + assert "Failed to record idle commit time" in service_logs.text + + +@pytest.mark.asyncio +async def test_record_idle_time_rereads_under_lock_and_preserves_concurrent_fields( + timeout_service, request_context +): + service, fs = timeout_service + concurrent = { + **fs.meta, + "auto_commit_policy": {"idle_timeout_seconds": 180, "keep_recent_count": 77}, + "last_message_at": "2026-01-01T00:02:59Z", + "message_count": 9, + "commit_count": 4, + "pending_tokens": 4321, + "future_extension": {"opaque": [1, 2, 3]}, + } + fs.on_acquire = lambda: setattr(fs, "meta", copy.deepcopy(concurrent)) + accepted_at = "2026-01-01T00:03:00.123Z" + + await service._record_idle_commit_time("s", request_context, accepted_at) + + assert fs.meta["last_idle_commit_accepted_at"] == accepted_at + assert parse_iso_datetime(fs.meta["updated_at"]) >= parse_iso_datetime(concurrent["updated_at"]) + assert { + key: value + for key, value in fs.meta.items() + if key not in {"last_idle_commit_accepted_at", "updated_at"} + } == { + key: value + for key, value in concurrent.items() + if key not in {"last_idle_commit_accepted_at", "updated_at"} + } + assert len(fs.writes) == 1 + assert fs.events.index("acquire") < fs.events.index("read") < fs.events.index("write") + assert fs.events[-1] == "release" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "existing_at", + ["2026-01-01T00:04:00Z", "2026-01-01T08:04:00+08:00"], +) +async def test_record_idle_time_never_moves_existing_acceptance_backwards( + timeout_service, request_context, existing_at +): + service, fs = timeout_service + fs.meta["last_idle_commit_accepted_at"] = existing_at + + await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") + + assert parse_iso_datetime(fs.meta["last_idle_commit_accepted_at"]) == parse_iso_datetime( + existing_at + ) + assert fs.events[-1] == "release" + + +@pytest.mark.asyncio +async def test_record_idle_time_deleted_session_is_not_recreated(timeout_service, request_context): + service, fs = timeout_service + fs.on_acquire = lambda: setattr(fs, "exists", False) + + await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") + + assert fs.exists is False + assert fs.writes == [] + assert fs.events[-1] == "release" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["read_error", "invalid_json", "not_a_dict", "write_error"]) +async def test_record_idle_time_metadata_failures_are_logged_best_effort( + timeout_service, request_context, service_logs, failure +): + service, fs = timeout_service + if failure == "read_error": + fs.read_error = RuntimeError("metadata read failed") + elif failure == "invalid_json": + fs.raw_meta = "{invalid" + elif failure == "not_a_dict": + fs.raw_meta = "[]" + else: + fs.write_error = RuntimeError("metadata write failed") + + await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") + + assert fs.writes == [] + assert fs.events[-1] == "release" + assert "Failed to record idle commit time" in service_logs.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reason", ["idle_activity", "idle_cooldown"]) +async def test_timeout_commit_deferred_does_not_commit_or_record( + timeout_service, idle_session, request_context, reason +): + service, fs = timeout_service + if reason == "idle_activity": + idle_session.meta.last_message_at = "2026-01-01T00:01:59Z" + expected_due_ms = 1767225779000 + else: + idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:01:59Z" + expected_due_ms = 1767225780000 + + assert await service.commit_on_timeout("s", request_context) == { + "status": "deferred", + "reason": reason, + "due_at_ms": expected_due_ms, + "timeout_seconds": 60, + } + idle_session.commit_async.assert_not_awaited() + assert fs.events == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("value", ["", "invalid timestamp"]) +async def test_timeout_commit_invalid_activity_time_fails_closed( + timeout_service, idle_session, request_context, value +): + service, fs = timeout_service + idle_session.meta.last_message_at = value + + with pytest.raises((RuntimeError, ValueError)): + await service.commit_on_timeout("s", request_context) + + idle_session.commit_async.assert_not_awaited() + assert fs.events == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ["read", "write"]) +async def test_record_idle_time_cancellation_propagates_and_releases_lock( + timeout_service, request_context, operation +): + service, fs = timeout_service + cancellation = asyncio.CancelledError() + setattr(fs, f"{operation}_error", cancellation) + + with pytest.raises(asyncio.CancelledError) as caught: + await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") + + assert caught.value is cancellation + assert fs.events[-1] == "release" + assert fs.locked is False + assert fs.writes == [] + + +@pytest.mark.asyncio +async def test_record_idle_time_lock_failure_is_logged_best_effort( + timeout_service, request_context, service_logs +): + service, fs = timeout_service + acquire = AsyncMock(spec=fs.acquire, side_effect=RuntimeError("lock unavailable")) + fs._async_agfs.pathlock_acquire_exact = acquire + + await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") + + acquire.assert_awaited_once() + assert fs.events == [] + assert fs.writes == [] + assert "Failed to record idle commit time" in service_logs.text + + +@pytest.mark.asyncio +async def test_record_idle_time_replaces_invalid_previous_acceptance( + timeout_service, request_context, service_logs +): + service, fs = timeout_service + fs.meta["last_idle_commit_accepted_at"] = "invalid timestamp" + accepted_at = "2026-01-01T00:03:00Z" + + await service._record_idle_commit_time("s", request_context, accepted_at) + + assert fs.meta["last_idle_commit_accepted_at"] == accepted_at + assert len(fs.writes) == 1 + assert fs.events[-1] == "release" + assert "Replacing invalid idle commit time" in service_logs.text + + +@pytest.fixture +def enable_default_legacy_backend(monkeypatch): + from openviking.service import core + + original_get_config = core.get_openviking_config + + def enabled_config(): + config = original_get_config() + config.memory.session_auto_commit = SessionAutoCommitConfig(enabled=True) + return config + + monkeypatch.setattr(core, "get_openviking_config", enabled_config) + + +@pytest.fixture +def legacy_service(enable_default_legacy_backend, service): + return service + + +@pytest.mark.asyncio +async def test_default_legacy_backend_starts_scheduler_without_expiry(legacy_service): + assert SessionAutoCommitConfig().idle_backend == "legacy" + assert legacy_service._session_auto_commit_scheduler is not None + assert legacy_service._session_auto_commit_scheduler._running is True + assert legacy_service._expiry_runtime is None + assert legacy_service.sessions._session_timer_adapter is None + + +@pytest.fixture +def enable_timer_without_business_work(monkeypatch): + from openviking.service import core + + original_get_config = core.get_openviking_config + + def enabled_config(): + config = original_get_config() + config.memory.session_auto_commit = SessionAutoCommitConfig(enabled=False) + config.expiry = ExpiryConfig( + service_name="unused", + reporting_enabled=True, + redis={"endpoints": ["127.0.0.1:1"]}, + ) + return config + + monkeypatch.setattr(core, "get_openviking_config", enabled_config) + + +@pytest.fixture +def timer_without_business_work_service(enable_timer_without_business_work, service): + return service + + +@pytest.mark.asyncio +async def test_enabled_timer_without_business_work_does_not_start_runtime( + timer_without_business_work_service, +): + assert timer_without_business_work_service._expiry_runtime is None diff --git a/tests/scheduling/expiry/test_session_timer.py b/tests/scheduling/expiry/test_session_timer.py new file mode 100644 index 0000000000..da63431b42 --- /dev/null +++ b/tests/scheduling/expiry/test_session_timer.py @@ -0,0 +1,223 @@ +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from openviking.scheduling.expiry import ( + CallbackRequest, + CallbackStatus, + ExpiryMode, + ItemIdentity, +) +from openviking.service.session_timer import ( + SessionTimeoutHandler, + SessionTimerAdapter, + effective_idle_timeout, +) +from openviking.session.session import SessionTimerSnapshot + + +class Timer: + def __init__(self): + self.reports = [] + self.unregistered = [] + + async def report(self, report): + self.reports.append(report) + return SimpleNamespace(accepted_due_at_ms=report.due_at_ms) + + async def unregister(self, identity): + self.unregistered.append(identity) + + +def test_adapter_builds_due_from_last_message_and_current_policy(): + timer = Timer() + adapter = SessionTimerAdapter(timer) + snapshot = SessionTimerSnapshot( + "a", + "u", + "s", + "2026-01-01T00:00:00+00:00", + {"idle_timeout_seconds": 60}, + ) + report = adapter.build_report(snapshot) + assert report is not None + assert report.due_at_ms == 1767225660000 + assert report.timeout_seconds == 60 + + +def test_disabled_policy_produces_no_report(): + adapter = SessionTimerAdapter(Timer()) + snapshot = SessionTimerSnapshot("a", "u", "s", "2026-01-01T00:00:00Z", None) + assert adapter.build_report(snapshot) is None + assert effective_idle_timeout({"idle_timeout_seconds": 0}) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reason", ["idle_cooldown", "idle_activity"]) +async def test_handler_reports_deferred_before_completing_old_event(reason): + timer = Timer() + + class Sessions: + async def commit_on_timeout(self, session_id, ctx): + return { + "status": "deferred", + "reason": reason, + "due_at_ms": 999, + "timeout_seconds": 30, + } + + identity = ItemIdentity("session", "a", "u", "s") + result = await SessionTimeoutHandler(Sessions(), timer).handle(CallbackRequest(identity)) + assert result.status is CallbackStatus.COMPLETED + assert timer.reports[0].due_at_ms == 999 + + +@pytest.mark.asyncio +async def test_handler_maps_not_found_to_completed(): + class Sessions: + async def commit_on_timeout(self, session_id, ctx): + return {"status": "skipped", "reason": "not_found"} + + result = await SessionTimeoutHandler(Sessions(), Timer()).handle( + CallbackRequest(ItemIdentity("session", "a", "u", "s")) + ) + assert result.status is CallbackStatus.COMPLETED + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reason", ["idle_cooldown", "idle_activity"]) +async def test_handler_deferred_report_failure_propagates(reason): + class Sessions: + async def commit_on_timeout(self, session_id, ctx): + return { + "status": "deferred", + "reason": reason, + "due_at_ms": 999, + "timeout_seconds": 30, + } + + timer = Timer() + error = RuntimeError("reschedule unavailable") + timer.report = AsyncMock(spec=timer.report, side_effect=error) + identity = ItemIdentity("session", "a", "u", "s") + + with pytest.raises(RuntimeError, match="reschedule unavailable") as caught: + await SessionTimeoutHandler(Sessions(), timer).handle(CallbackRequest(identity)) + + assert caught.value is error + timer.report.assert_awaited_once() + report = timer.report.await_args.args[0] + assert report.identity == identity + assert report.due_at_ms == 999 + assert report.timeout_seconds == 30 + assert report.mode is ExpiryMode.IDLE_ONESHOT + + +@pytest.mark.asyncio +async def test_handler_reads_source_session_from_identity_without_metadata(): + calls = [] + + class Sessions: + async def commit_on_timeout(self, session_id, ctx): + calls.append((session_id, ctx)) + return {"status": "accepted", "task_id": "task"} + + timer = Timer() + identity = ItemIdentity("session", "account", "user", "s") + + result = await SessionTimeoutHandler(Sessions(), timer).handle(CallbackRequest(identity)) + + assert result.status is CallbackStatus.COMPLETED + assert len(calls) == 1 + session_id, ctx = calls[0] + assert session_id == "s" + assert ctx.account_id == "account" + assert ctx.user.user_id == "user" + assert timer.reports == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reason", ["idle_disabled", "no_messages"]) +async def test_handler_terminal_skips_do_not_reschedule(reason): + class Sessions: + async def commit_on_timeout(self, session_id, ctx): + return {"status": "skipped", "reason": reason} + + timer = Timer() + result = await SessionTimeoutHandler(Sessions(), timer).handle( + CallbackRequest(ItemIdentity("session", "a", "u", "s")) + ) + + assert result.status is CallbackStatus.COMPLETED + assert timer.reports == [] + + +@pytest.mark.asyncio +async def test_handler_commit_failure_propagates(): + class Sessions: + async def commit_on_timeout(self, session_id, ctx): + raise RuntimeError("source commit failed") + + timer = Timer() + with pytest.raises(RuntimeError, match="source commit failed"): + await SessionTimeoutHandler(Sessions(), timer).handle( + CallbackRequest(ItemIdentity("session", "a", "u", "s")) + ) + assert timer.reports == [] + + +@pytest.mark.asyncio +async def test_adapter_append_reports_current_snapshot_identity_and_policy(): + timer = Timer() + snapshot = SessionTimerSnapshot( + "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 120} + ) + + await SessionTimerAdapter(timer).report_after_append(snapshot) + + assert len(timer.reports) == 1 + report = timer.reports[0] + assert report.identity == ItemIdentity("session", "a", "u", "s") + assert report.due_at_ms == 1767225720000 + assert report.timeout_seconds == 120 + assert report.mode is ExpiryMode.IDLE_ONESHOT + + +@pytest.fixture +def adapter_logs(caplog): + from openviking.service.session_timer import logger + + logger.addHandler(caplog.handler) + try: + yield caplog + finally: + logger.removeHandler(caplog.handler) + + +@pytest.mark.asyncio +async def test_adapter_append_report_failure_is_best_effort(adapter_logs): + timer = Timer() + timer.report = AsyncMock(spec=timer.report, side_effect=RuntimeError("redis unavailable")) + snapshot = SessionTimerSnapshot( + "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 60} + ) + + await SessionTimerAdapter(timer).report_after_append(snapshot) + + timer.report.assert_awaited_once() + assert "report_failed" in adapter_logs.text + + +@pytest.mark.asyncio +async def test_adapter_append_report_cancellation_propagates(): + timer = Timer() + timer.report = AsyncMock(spec=timer.report, side_effect=asyncio.CancelledError()) + snapshot = SessionTimerSnapshot( + "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 60} + ) + + with pytest.raises(asyncio.CancelledError): + await SessionTimerAdapter(timer).report_after_append(snapshot) + timer.report.assert_awaited_once() diff --git a/tests/session/test_session_commit.py b/tests/session/test_session_commit.py index 4f5c1454d7..946f7b4498 100644 --- a/tests/session/test_session_commit.py +++ b/tests/session/test_session_commit.py @@ -34,9 +34,101 @@ async def _marker_exists(session, archive_uri: str, name: str) -> bool: return False +class _RecordingTimerHook: + def __init__(self) -> None: + self.snapshots = [] + self.failures = [] + + async def report_after_append(self, snapshot) -> None: + self.snapshots.append(snapshot) + + def record_report_failure(self, reason, exc=None) -> None: + self.failures.append((reason, exc)) + + class TestCommit: """Test commit""" + async def test_append_reports_repaired_meta_after_authoritative_reload_failure( + self, session_with_messages: Session, monkeypatch + ): + hook = _RecordingTimerHook() + session_with_messages._timer_hook = hook + session_with_messages.meta.auto_commit_policy = {"idle_timeout_seconds": 60} + await session_with_messages._save_meta() + original_read_file = session_with_messages._viking_fs.read_file + meta_uri = f"{session_with_messages.uri}/.meta.json" + failed = False + + async def fail_meta_reload_once(uri, *args, **kwargs): + nonlocal failed + if uri == meta_uri and not failed: + failed = True + raise RuntimeError("synthetic metadata reload failure") + return await original_read_file(uri, *args, **kwargs) + + monkeypatch.setattr(session_with_messages._viking_fs, "read_file", fail_meta_reload_once) + + await session_with_messages.add_message_async("user", [TextPart("latest activity")]) + + assert failed is True + assert len(hook.snapshots) == 1 + assert hook.snapshots[0].last_message_at == session_with_messages.meta.last_message_at + assert hook.snapshots[0].auto_commit_policy == {"idle_timeout_seconds": 60} + assert hook.failures == [] + + async def test_service_timeout_archives_append_after_check_without_idle_phase1_extensions( + self, session_with_messages: Session, service, request_context, monkeypatch + ): + session = session_with_messages + session.meta.auto_commit_policy = {"idle_timeout_seconds": 60, "keep_recent_count": 2} + session.meta.keep_recent_count = 2 + session.meta.last_message_at = "2000-01-01T00:00:00Z" + await session._save_meta() + original_commit = Session.commit_async + commits = [] + extract = AsyncMock( + spec=session._session_compressor.extract_long_term_memories, + return_value=[], + ) + monkeypatch.setattr(session._session_compressor, "extract_long_term_memories", extract) + + async def append_before_commit(loaded_session, **kwargs): + commits.append(kwargs) + await loaded_session.add_message_async("user", [TextPart("append after idle check")]) + return await original_commit(loaded_session, **kwargs) + + monkeypatch.setattr(Session, "commit_async", append_before_commit) + + result = await service.sessions.commit_on_timeout(session.session_id, request_context) + + assert result["status"] == "accepted" + assert commits == [ + { + "keep_recent_count": 0, + "persist_keep_recent_count": False, + "record_auto_commit_success": True, + } + ] + archived = await session._viking_fs.read_file( + f"{result['archive_uri']}/messages.jsonl", ctx=request_context + ) + archived_messages = [json.loads(line) for line in archived.split("\n") if line.strip()] + assert len(archived_messages) == 5 + assert archived_messages[-1]["parts"][0]["text"] == "append after idle check" + persisted = await service.sessions.get(session.session_id, request_context) + assert persisted.messages == [] + assert persisted.meta.keep_recent_count == 2 + assert persisted.meta.last_idle_commit_accepted_at + phase1 = await persisted._read_phase1_meta(result["archive_uri"]) + assert phase1["status"] == "ready" + assert "idle_timeout" not in phase1 + assert "last_idle_commit_accepted_at" not in phase1 + assert "last_idle_commit_archive" not in persisted.meta.to_dict() + assert (await _wait_for_task(result["task_id"]))["status"] == "completed" + extract.assert_awaited_once() + assert len(extract.await_args.kwargs["messages"]) == 5 + async def test_commit_preserves_unicode_separators_and_accepts_later_messages( self, wm_session_with_messages: Session ): diff --git a/uv.lock b/uv.lock index e8577c9931..60c2613541 100644 --- a/uv.lock +++ b/uv.lock @@ -3617,6 +3617,9 @@ ocr = [ opengauss = [ { name = "psycopg2-binary" }, ] +expiry = [ + { name = "redis" }, +] test = [ { name = "boto3" }, { name = "datasets" }, @@ -3732,6 +3735,7 @@ requires-dist = [ { name = "ragas", marker = "extra == 'test'", specifier = ">=0.1.0" }, { name = "rapidfuzz", specifier = ">=3.14,<4" }, { name = "readability-lxml", marker = "extra == 'bot'", specifier = ">=0.8.0" }, + { name = "redis", marker = "extra == 'expiry'", specifier = ">=5,<6" }, { name = "requests", specifier = ">=2.33.0" }, { name = "rich", marker = "extra == 'bot'", specifier = ">=13.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, @@ -3771,7 +3775,7 @@ requires-dist = [ { name = "wheel", marker = "extra == 'build'" }, { name = "xxhash", specifier = ">=3.0.0" }, ] -provides-extras = ["context-gateway", "context-gateway-fast", "test", "auth", "dev", "doc", "eval", "gemini", "gemini-async", "ocr", "build", "bot", "benchmark", "local-embed", "opengauss"] +provides-extras = ["expiry", "context-gateway", "context-gateway-fast", "test", "auth", "dev", "doc", "eval", "gemini", "gemini-async", "ocr", "build", "bot", "benchmark", "local-embed", "opengauss"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.2" }] @@ -5440,6 +5444,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/75/2cc58965097e351415af420be81c4665cf80da52a17ef43c01ffbe2caf91/readability_lxml-0.8.4.1-py3-none-any.whl", hash = "sha256:874c0cea22c3bf2b78c7f8df831bfaad3c0a89b7301d45a188db581652b4b465", size = 19912, upload-time = "2025-05-03T21:11:43.993Z" }, ] +[[package]] +name = "redis" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, + { name = "pyjwt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/cf/128b1b6d7086200c9f387bd4be9b2572a30b90745ef078bd8b235042dc9f/redis-5.3.1.tar.gz", hash = "sha256:ca49577a531ea64039b5a36db3d6cd1a0c7a60c34124d46924a45b956e8cf14c", size = 4626200, upload-time = "2025-07-25T08:06:27.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/26/5c5fa0e83c3621db835cfc1f1d789b37e7fa99ed54423b5f519beb931aa7/redis-5.3.1-py3-none-any.whl", hash = "sha256:dc1909bd24669cc31b5f67a039700b16ec30571096c5f1f0d9d2324bff31af97", size = 272833, upload-time = "2025-07-25T08:06:26.317Z" }, +] + [[package]] name = "referencing" version = "0.37.0" From d59662fbc2ec33f347d7c60b6f7729046e094f7c Mon Sep 17 00:00:00 2001 From: lujiakun Date: Wed, 14 Oct 2026 11:18:11 +0800 Subject: [PATCH 2/5] feat: integrate Redis session expiry lifecycle --- docs/en/api/05-sessions.md | 18 +- docs/en/guides/01-configuration.md | 58 +- docs/zh/api/05-sessions.md | 18 +- ...-idle-auto-commit-scheduler-code-design.md | 2795 +++++++++++++++++ ...n-idle-auto-commit-scheduler-vibecoding.md | 453 +++ docs/zh/guides/01-configuration.md | 58 +- openviking/scheduling/expiry/api.py | 1 + openviking/scheduling/expiry/config.py | 83 +- openviking/scheduling/expiry/consumer.py | 87 +- openviking/scheduling/expiry/keys.py | 18 +- openviking/scheduling/expiry/redis_store.py | 58 +- openviking/scheduling/expiry/registry.py | 5 + openviking/scheduling/expiry/runtime.py | 167 +- openviking/scheduling/expiry/service.py | 11 +- openviking/scheduling/expiry/shard_runner.py | 48 +- openviking/server/bootstrap.py | 221 +- openviking/server/config.py | 11 +- openviking/server/routers/sessions.py | 12 +- openviking/service/core.py | 28 +- .../{session_timer.py => session_expiry.py} | 44 +- openviking/service/session_service.py | 35 +- openviking/session/auto_commit_policy.py | 21 +- openviking/session/session.py | 28 +- openviking_cli/expiry_bootstrap.py | 91 +- .../utils/config/open_viking_config.py | 19 +- pyproject.toml | 1 - tests/scheduling/expiry/test_config.py | 157 +- tests/scheduling/expiry/test_consumer.py | 104 +- .../expiry/test_expiry_bootstrap.py | 156 + .../expiry/test_facade_and_registry.py | 57 +- tests/scheduling/expiry/test_leases.py | 38 +- .../scheduling/expiry/test_models_and_keys.py | 16 +- tests/scheduling/expiry/test_redis_loop.py | 44 + tests/scheduling/expiry/test_redis_store.py | 281 +- tests/scheduling/expiry/test_runtime.py | 284 +- ...ession_timer.py => test_session_expiry.py} | 115 +- .../expiry/test_session_timeout_service.py | 18 +- .../test_agent_evolution_global_setting.py | 47 +- tests/server/test_api_sessions.py | 69 + tests/server/test_bootstrap.py | 14 +- tests/session/test_session_commit.py | 14 +- tests/unit/session/test_auto_commit_policy.py | 20 + .../test_server_bootstrap_expiry_publisher.py | 200 ++ 43 files changed, 5523 insertions(+), 500 deletions(-) create mode 100644 docs/zh/design/session-idle-auto-commit-scheduler-code-design.md create mode 100644 docs/zh/design/session-idle-auto-commit-scheduler-vibecoding.md rename openviking/service/{session_timer.py => session_expiry.py} (79%) create mode 100644 tests/scheduling/expiry/test_expiry_bootstrap.py rename tests/scheduling/expiry/{test_session_timer.py => test_session_expiry.py} (62%) create mode 100644 tests/unit/test_server_bootstrap_expiry_publisher.py diff --git a/docs/en/api/05-sessions.md b/docs/en/api/05-sessions.md index c779abd2e1..27edb74e03 100644 --- a/docs/en/api/05-sessions.md +++ b/docs/en/api/05-sessions.md @@ -48,15 +48,15 @@ Create a new session. Sessions are containers for conversations, storing message `auto_commit_policy` fields (all optional; omitted fields fall back to the defaults when a policy is present): -| Field | Type | Default | Max | Description | -|-------|------|---------|-----|-------------| -| `pending_token_threshold` | int | 150000 | 1000000 | When uncommitted pending tokens exceed this value (strictly greater-than), an auto commit is triggered after a message write. | -| `message_count_threshold` | int | 100 | 1000 | When the uncommitted live message count exceeds this value (strictly greater-than), an auto commit is triggered after a message write. | -| `idle_timeout_seconds` | int | 86400 | 604800 | After this many idle seconds, a session with uncommitted content becomes eligible for the server-side idle scheduler. An idle-timeout commit archives the full backlog and ignores `keep_recent_count`. | -| `keep_recent_count` | int | 0 | 500 | Number of recent live messages to keep (not archived) on a threshold-triggered auto commit. Idle-timeout commits ignore this and commit everything. | -| `min_commit_interval_seconds` | int | 0 | 604800 | Minimum seconds between two automatic commits (throttle). | - -All fields have a minimum of `0` and are clamped into `[0, max]`. Unknown keys are rejected with `InvalidArgumentError`. Idle commits also require `memory.session_auto_commit.enabled` on the server; setting `idle_timeout_seconds` alone does not start the scanner. +| Field | Type | Default | Min | Max | Description | +|-------|------|---------|-----|-----|-------------| +| `pending_token_threshold` | int | 150000 | 0 | 1000000 | When uncommitted pending tokens exceed this value (strictly greater-than), an auto commit is triggered after a message write. | +| `message_count_threshold` | int | 100 | 0 | 1000 | When the uncommitted live message count exceeds this value (strictly greater-than), an auto commit is triggered after a message write. | +| `idle_timeout_seconds` | int | 86400 | 0 (off) or 300 | 604800 | After this many idle seconds, a session with uncommitted content becomes eligible for the server-side idle scheduler. An idle-timeout commit archives the full backlog and ignores `keep_recent_count`. | +| `keep_recent_count` | int | 0 | 0 | 500 | Number of recent live messages to keep (not archived) on a threshold-triggered auto commit. Idle-timeout commits ignore this and commit everything. | +| `min_commit_interval_seconds` | int | 0 | 0 | 604800 | Minimum seconds between two automatic commits (throttle). | + +New `idle_timeout_seconds` inputs must be `0` (off) or at least `300`; values above `604800` are clamped to that maximum. For stored-data compatibility, existing positive values below 300 continue to be read unchanged. Other fields have a minimum of `0` and are clamped to their ranges. Unknown keys are rejected with `InvalidArgumentError`. Idle commits also require `memory.session_auto_commit.enabled` on the server; setting `idle_timeout_seconds` alone does not start the scanner. #### 3. Usage Examples diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md index 220e411990..3a467abcf4 100644 --- a/docs/en/guides/01-configuration.md +++ b/docs/en/guides/01-configuration.md @@ -1732,6 +1732,7 @@ For memory-related settings, add a `memory` section in `ov.conf`: "memory": { "session_auto_commit": { "enabled": false, + "idle_backend": "legacy", "check_interval_seconds": 600.0, "scan_rate_limit_files_per_second": 2.0 } @@ -1742,6 +1743,7 @@ For memory-related settings, add a `memory` section in `ov.conf`: | Parameter | Type | Description | Default | |-----------|------|-------------|---------| | `enabled` | bool | Master switch for automatic session commits. When enabled, newly created sessions without an explicit `auto_commit_policy` get a default policy, and the idle-timeout background scheduler is started. When disabled, neither happens | `false` | +| `idle_backend` | string | Idle-timeout discovery backend: `legacy` scans session metadata; `redis` uses distributed expiry scheduling | `legacy` | | `check_interval_seconds` | float | Minimum interval between two idle scan rounds in seconds. Must be greater than `0` | `600.0` | | `scan_rate_limit_files_per_second` | float | Maximum number of session `.meta.json` files read per second during an idle scan. Must be greater than `0`. Bounds background storage IO when the sessions tree is very large | `2.0` | @@ -1752,22 +1754,56 @@ Notes: - When `enabled=false`, sessions created without an explicit or `server.user_config_defaults.auto_commit_policy` policy keep auto commit disabled and return `auto_commit_policy: null`; `SessionAutoCommitScheduler` is not started. - When `enabled=true`: - Sessions without an explicit or deployment-default policy get the built-in policy below. - - `SessionAutoCommitScheduler` starts a scan round immediately, then waits at least `check_interval_seconds` before the next round. If a round already took longer than that interval, the next round starts immediately. - - Within a round, session `.meta.json` files are read serially at no more than `scan_rate_limit_files_per_second` to avoid IO spikes. - - It does not perform a dedicated startup recovery sweep; idle detection happens only on periodic scans. + - With `idle_backend="legacy"`, `SessionAutoCommitScheduler` starts a scan round immediately, then waits at least `check_interval_seconds` before the next round. Session `.meta.json` files are read serially at no more than `scan_rate_limit_files_per_second`. + - With `idle_backend="redis"`, the server enables expiry reporting, publishing, and callback consumption together using the top-level `expiry` configuration. - Token- and message-count auto commit run inline after message writes, do not depend on the scheduler, but still require the session to carry an `auto_commit_policy`. +#### Distributed Expiry Configuration + +All API, publisher, and callback instances that share expiry state must use the same stable `service_name`. It identifies the deployed document service, not an individual pod. + +```json +{ + "expiry": { + "service_name": "openviking-documents", + "publisher_separate_process": true, + "shard_count": 128, + "session_consumer_concurrency": 8, + "redis": { + "mode": "cluster", + "endpoints": ["redis://redis-proxy.example.com:6379"], + "username": "openviking", + "password_env": "OPENVIKING_EXPIRY_REDIS_PASSWORD" + } + } +} +``` + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `service_name` | string | Stable document service name shared by all instances using the same expiry data | `""` | +| `publisher_separate_process` | bool | Let `openviking-server` supervise publisher in a dedicated child process; when `false`, publisher runs in each API process | `true` | +| `shard_count` | int | Number of logical expiry shards; all instances sharing a namespace must use the same value, in the range `1..512` | `128` | +| `session_consumer_concurrency` | int | Concurrent Session expiry callbacks per API process; must be greater than `0` | `8` | +| `redis.mode` | string | Redis deployment mode: `standalone` or `cluster` | `standalone` | +| `redis.endpoints` | string array | Redis endpoints; a cluster may use one proxy endpoint | `[]` | +| `redis.username` | string | Redis ACL username | `""` | +| `redis.password_env` | string | Environment variable containing the Redis password; startup fails if it is not set | `""` | +| `redis.password` | string | Legacy plaintext password; prefer `password_env`; mutually exclusive with it | `""` | + +`service_name` and Redis endpoints are required when Redis idle auto commit is enabled. Credentials must use the dedicated fields above and cannot be embedded in endpoint URIs. With the default `publisher_separate_process=true`, `openviking-server` starts one internal publisher before serving traffic, stops it during shutdown, and exits if that child fails unexpectedly. This publisher is not exposed as a public CLI command. + ##### Per-session Auto Commit Policy -When a session carries an `auto_commit_policy`, any field you omit falls back to the built-in default below. Sessions without a stored policy keep auto commit disabled. Values are clamped into `[0, max]`, and unknown keys are rejected with `InvalidArgumentError`. See [Sessions API](../api/05-sessions.md#create-session) for how to set and view it. +When a session carries an `auto_commit_policy`, any field you omit falls back to the built-in default below. Sessions without a stored policy keep auto commit disabled. Except for the idle-timeout write minimum, values are clamped into `[0, max]`; unknown keys are rejected with `InvalidArgumentError`. See [Sessions API](../api/05-sessions.md#create-session) for how to set and view it. -| Field | Type | Default | Max | Description | -|-------|------|---------|-----|-------------| -| `pending_token_threshold` | int | 150000 | 1000000 | When uncommitted pending tokens exceed this value (strictly greater-than), an auto commit is triggered after a message write. | -| `message_count_threshold` | int | 100 | 1000 | When the uncommitted live message count exceeds this value (strictly greater-than), an auto commit is triggered after a message write. | -| `idle_timeout_seconds` | int | 86400 | 604800 | After this many idle seconds, a session with uncommitted content becomes eligible for the server-side idle scheduler. Idle-timeout commits archive the full backlog and ignore `keep_recent_count`. | -| `keep_recent_count` | int | 0 | 500 | Number of recent live messages to keep (not archived) on a threshold-triggered auto commit. Idle-timeout commits ignore this and commit everything. | -| `min_commit_interval_seconds` | int | 0 | 604800 | Minimum seconds between two automatic commits (throttle). | +| Field | Type | Default | Min | Max | Description | +|-------|------|---------|-----|-----|-------------| +| `pending_token_threshold` | int | 150000 | 0 | 1000000 | When uncommitted pending tokens exceed this value (strictly greater-than), an auto commit is triggered after a message write. | +| `message_count_threshold` | int | 100 | 0 | 1000 | When the uncommitted live message count exceeds this value (strictly greater-than), an auto commit is triggered after a message write. | +| `idle_timeout_seconds` | int | 86400 | 0 (off) or 300 | 604800 | New positive inputs have a five-minute minimum; existing stored positive values below 300 continue to be read unchanged. Idle-timeout commits archive the full backlog and ignore `keep_recent_count`. | +| `keep_recent_count` | int | 0 | 0 | 500 | Number of recent live messages to keep (not archived) on a threshold-triggered auto commit. Idle-timeout commits ignore this and commit everything. | +| `min_commit_interval_seconds` | int | 0 | 0 | 604800 | Minimum seconds between two automatic commits (throttle). | Code entry: `openviking/session/auto_commit_policy.py:AutoCommitPolicy`. diff --git a/docs/zh/api/05-sessions.md b/docs/zh/api/05-sessions.md index 2607865a51..196c25afb3 100644 --- a/docs/zh/api/05-sessions.md +++ b/docs/zh/api/05-sessions.md @@ -48,15 +48,15 @@ Session API 按认证用户作用域访问会话，并返回 canonical user sess `auto_commit_policy` 字段（均为可选；存在 policy 时，未传字段回退到默认值）： -| 字段 | 类型 | 默认值 | 上限 | 说明 | -|------|------|--------|------|------| -| `pending_token_threshold` | int | 150000 | 1000000 | 当未提交的 pending token 超过该值（严格大于）时，会在消息写入后触发一次自动 commit。 | -| `message_count_threshold` | int | 100 | 1000 | 当未提交的 live message 数量超过该值（严格大于）时，会在消息写入后触发一次自动 commit。 | -| `idle_timeout_seconds` | int | 86400 | 604800 | 有未提交内容的 session 在空闲这么多秒后，进入服务端 idle scheduler 的处理范围。idle 触发的 commit 会归档全部积压消息，并忽略 `keep_recent_count`。 | -| `keep_recent_count` | int | 0 | 500 | 阈值触发的自动 commit 后保留（不归档）的最近 live message 数量。idle 超时触发的 commit 会忽略该值并归档所有消息。 | -| `min_commit_interval_seconds` | int | 0 | 604800 | 两次自动 commit 之间的最小间隔秒数（节流）。 | - -所有字段最小值为 `0`，会被 clamp 到 `[0, 上限]`。未知字段会以 `InvalidArgumentError` 拒绝。idle 超时还要求服务端开启 `memory.session_auto_commit.enabled`；仅设置 `idle_timeout_seconds` 不会启动扫描器。 +| 字段 | 类型 | 默认值 | 最小值 | 上限 | 说明 | +|------|------|--------|--------|------|------| +| `pending_token_threshold` | int | 150000 | 0 | 1000000 | 当未提交的 pending token 超过该值（严格大于）时，会在消息写入后触发一次自动 commit。 | +| `message_count_threshold` | int | 100 | 0 | 1000 | 当未提交的 live message 数量超过该值（严格大于）时，会在消息写入后触发一次自动 commit。 | +| `idle_timeout_seconds` | int | 86400 | 0（关闭）或 300 | 604800 | 有未提交内容的 session 在空闲这么多秒后，进入服务端 idle scheduler 的处理范围。idle 触发的 commit 会归档全部积压消息，并忽略 `keep_recent_count`。 | +| `keep_recent_count` | int | 0 | 0 | 500 | 阈值触发的自动 commit 后保留（不归档）的最近 live message 数量。idle 超时触发的 commit 会忽略该值并归档所有消息。 | +| `min_commit_interval_seconds` | int | 0 | 0 | 604800 | 两次自动 commit 之间的最小间隔秒数（节流）。 | + +新传入的 `idle_timeout_seconds` 必须为 `0`（关闭）或不小于 `300`，超过 `604800` 时会被 clamp 到该上限；为兼容存量数据，已经保存的小于 300 的正值仍按原值读取。其他字段最小值为 `0`，并会被 clamp 到各自范围。未知字段会以 `InvalidArgumentError` 拒绝。idle 超时还要求服务端开启 `memory.session_auto_commit.enabled`；仅设置 `idle_timeout_seconds` 不会启动扫描器。 #### 3. 使用示例 diff --git a/docs/zh/design/session-idle-auto-commit-scheduler-code-design.md b/docs/zh/design/session-idle-auto-commit-scheduler-code-design.md new file mode 100644 index 0000000000..f09367f522 --- /dev/null +++ b/docs/zh/design/session-idle-auto-commit-scheduler-code-design.md @@ -0,0 +1,2795 @@ +# Session Redis 计时调度：逐文件代码设计 + +本文配套 [VibeCoding 实施文档](session-idle-auto-commit-scheduler-vibecoding.md)，把实现方案展开为可审阅的代码骨架。业务仍只使用三个接口；骨架展示签名和关键控制流，不代替可运行实现或行为验收。 + +本轮以用户修正为准：报告 R→M，无伴随状态或资源更新版本；耗尽仅日志；逐 shard 临时抢占，TTL 120 秒、20 秒续租，失败停止旧任务。冷却未满足经统一 report 重新计时。元数据仍可缺失，跨槽未决请求仍有残余风险，不能用取消任务宣称完全隔离旧写入。 + +代码基线为 `7d77e3d68fea82070a0b700a9acd64c40e7f4a9a`，本轮更新时间为2026-10-11。Redis optional、默认legacy不变，Redis协议与业务三个接口沿用已确认设计。最新服务层方案覆盖旧idle专用提交/恢复设计：外层加载和纯检查通过即调用原 `Session.commit_async`，不做持锁最终idle检查，期间新消息允许一起提交。accepted后服务层锁内重读最新meta，仅best-effort patch接受时间与updated_at；失败仍accepted、删除不重建。没有idle归档指针或可修复缓存，不承诺严格空闲或零重复。原通用提交的ready、QueueFS、恢复与WM契约继续保留。 + +## 1. 阅读方式、依赖和文件目录 + +- 新文件列出计划中的全部类、数据字段、公开接口及主要私有函数，包括 Lua 入口。 +- 已有文件只列本次新增或需要修改的函数；未涉及的已有函数继续保留，不在此复制整个内核。 +- `...` 表示实现体待编码，但参数、返回值、异常、锁/线程边界由注释约定。关键交接流程给出具体控制流。 +- 片段中的既有类型沿用项目 import；新增类型来自本文指定模块。本文做语法和接口一致性检查，不宣称这些骨架已经可以运行。 +- 字段和函数名以本附录为具体建议；如与主文档的简略伪代码不同，以这里展开的签名为准，业务约束仍以主文档为准。 + +```text +openviking/scheduling/expiry/ + __init__.py # 只导出 api.py 的业务契约，不导出实现类 + api.py # 三个业务接口、业务 DTO、处理器协议及注册选项 + models.py # 调度/消费内部数据结构、校验和纯计算函数 + config.py # 轻量配置定义，独立进程不经过聚合配置包 + keys.py # 身份编码、hash、key 和注册项寻址 + metrics.py # timer 指标的轻量包装 + service.py # 私有 facade 实现，只由 runtime 构造 + registry.py # 内部处理器注册表及不可变注册快照 + redis_store.py # Redis 数据操作；所有网络 I/O 经专属 loop + publisher.py # 分页扫描、分钟构建和发布 + consumer.py # 已注册桶发现、claim、业务回调、条件 ACK + leases.py # 仅 try_acquire/renew/release：Redis token 操作 + shard_runner.py # 单 shard 执行、后台 heartbeat、撤销/有界 join/pending 强引用 + runtime.py # 构造共享 runner、Redis loop 与有界启停 + lua/ + _common.lua + ensure_metadata.lua + upsert_resource.lua + source_time.lua + read_source_page.lua + publish_minute.lua + read_registry_page.lua + claim.lua + ack.lua + acquire_owner.lua + renew_owner.lua + release_owner.lua +openviking/service/session_expiry.py # session 业务适配 +openviking/session/session.py # append reporter/接受时间字段，原通用commit不拆分 +openviking/service/session_service.py # 加载、纯检查、原commit调用及best-effort局部patch +openviking/service/core.py # 启停顺序与旧 backend 选择 +openviking_cli/utils/config/expiry_config.py +openviking_cli/utils/config/open_viking_config.py +openviking_cli/utils/config/memory_config.py +openviking_cli/expiry_bootstrap.py +pyproject.toml +``` + +模块分层从底向上为 `api → models/keys → store/registry/service → leases → shard_runner → publisher/consumer → runtime`，上层引用下层，api 不反向 import 实现模块。Coordinator 只封装 token 操作；runner 只负责一段单 shard 工作的租约执行生命周期；遍历 shard 和调度节奏由具体 worker 自己持有，不引入通用 worker 框架。业务模块通过包入口只依赖 ExpiryScheduler 协议和业务 DTO，启动组装层才依赖 runtime。session 适配另外依赖自身业务的 SessionService。Session 本身只持有自己定义的 reporter 协议，不 import session_expiry。runtime 不 import SessionService，内部 publisher 子进程不加载业务 handler。 + +两个 loop 的责任明确分开：Redis 客户端、连接池和原始 Redis 命令只在 Redis I/O loop；业务 handler、session commit、scheduler/consumer 的控制流在它们的运行 loop（API 中为 API loop）。report 可以从不同调用 loop 发起，store 负责桥接网络 I/O，不能将整个 session commit 丢进 Redis loop。 + +## 2. api.py 与 models.py：业务契约和内部模型分开 + +### 2.1 api.py：唯一稳定的业务契约 + +业务只接收 ExpiryScheduler 实例，不构造实现类。下面的三个方法是全部业务操作；没有启动、freeze、Redis client、队列领取或 ACK 接口。 + +```python +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Protocol + + +SESSION_ITEM_TYPE = "session" +SUPPORTED_ITEM_TYPES: frozenset[str] = frozenset({SESSION_ITEM_TYPE}) + + +class ExpiryMode(str, Enum): + IDLE_ONESHOT = "idle_oneshot" + FIXED = "fixed" # 本期预留；尚不支持时明确报错。 + + +class CallbackStatus(str, Enum): + COMPLETED = "completed" + RETRY = "retry" + + +class ExpiryError(RuntimeError): + """公开错误只含业务可理解的错误码和是否可重试，不提供内部执行凭据。""" + + def __init__(self, code: str, message: str, *, retryable: bool = False) -> None: + """参数/注册错误不可重试；暂时不可用由调用者执行自己的失败策略。""" + super().__init__(message) + self.code = code + self.retryable = retryable + + +@dataclass(frozen=True) +class ItemIdentity: + item_type: str + account_id: str + user_id: str + local_id: str + + +@dataclass(frozen=True) +class ExpiryRegistration: + identity: ItemIdentity + due_at_ms: int + timeout_seconds: int + mode: ExpiryMode = ExpiryMode.IDLE_ONESHOT + + +@dataclass(frozen=True) +class ExpiryRegistrationResult: + accepted_due_at_ms: int # 服务接受的到期时间；不是实际入队时间，不返回内部执行凭据。 + + +@dataclass(frozen=True) +class CallbackRequest: + identity: ItemIdentity # 业务用此身份回源；不携带元数据缓存是否命中的标志。 + + +@dataclass(frozen=True) +class CallbackResult: + status: CallbackStatus + reason: str + + +@dataclass(frozen=True) +class HandlerOptions: + concurrency: int = 4 # 通用默认值；各业务在启动注册时可指定，运行中不可动态更改。 + max_attempts: int = 3 # 包含首次；session 采用 3。相同类型各回调副本必须一致。 + + +class ExpiryHandler(Protocol): + async def handle(self, request: CallbackRequest) -> CallbackResult: + """只执行业务操作；返回完成或重试。队列重领、超时和清理由框架负责。""" + ... + + +class ExpiryScheduler(Protocol): + async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: + """写入或更新计时；业务提供身份、到期时间和 timeout，等待本次操作结果。""" + ... + + async def unregister(self, identity: ItemIdentity) -> None: + """注销当前计时；重复调用幂等，已交付处理的任务按业务当前状态完成。""" + ... + + def register_handler(self, item_type: str, handler: ExpiryHandler, *, + options: HandlerOptions | None = None) -> None: + """业务模块启动时注册类型及处理器；重复注册或运行中修改明确报错。""" + ... +``` + +HandlerOptions 表达业务处理能力，不暴露队列实现。首期只支持 session，`SUPPORTED_ITEM_TYPES` 是上报、Handler 注册和调度恢复共用的代码清单。report 不做请求顺序比较，接口不再返回重复或陈旧状态。 + +### 2.2 models.py：不向业务导出的内部类型 + +这些类型仅供框架内部使用，包入口不会导出 Claim、BucketRef、OwnerToken、游标或 Redis 状态。内部模块明确从 api.py 引用公开 DTO，不再把公开/私有模型混放。 + +```python +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Awaitable, Callable, Literal, Protocol, TypeVar +from .api import ItemIdentity, ExpiryRegistration, ExpiryError, ExpiryMode + +T = TypeVar("T") +MINUTE_MS = 60_000 +MIN_METADATA_TTL_MS = 24 * 60 * 60 * 1000 +SCORE_TIME_BASE = 1 << 40 +MAX_ATTEMPT = 255 +MAX_EXACT_INTEGER = (1 << 53) - 1 + + +class StoreOperationError(ExpiryError): + """内部存储错误；执行结果是否未知只供调度/重试恢复使用。""" + + def __init__(self, code: str, message: str, *, retryable: bool = False, + outcome_unknown: bool = False) -> None: + """保留内部执行状态；facade 会映射为公开错误，不直接透传此类型。""" + super().__init__(code, message, retryable=retryable) + self.outcome_unknown = outcome_unknown + + +class OwnerRole(str, Enum): + SCHEDULER = "scheduler" + CALLBACK = "callback" + + +@dataclass(frozen=True) +class ItemMetadata: + identity: ItemIdentity + + +@dataclass(frozen=True) +class NamespaceSpec: + shard_count: int + hash_name: str + hash_seed: int + identity_encoding: str # 固定实际规则名称：urlsafe_base64，无独立版本号。 + + +@dataclass(frozen=True) +class OwnerToken: + role: OwnerRole + shard_id: int + token: str # 每次成功领取的随机 token；不复用上次进程 token。 + expires_at_ms: int # 两个角色均有 TTL；仅用于观测，本地截止使用 monotonic。 + + +@dataclass +class ShardLease: + owner: OwnerToken + lost: asyncio.Event # ownership 已不可信；一旦置位，后续迟到成功也不可复活。 + valid_until_monotonic: float + revoked: asyncio.Event = field(default_factory=asyncio.Event) # 本地执行已撤销，不等于失锁。 + + def ensure_active(self) -> None: + """动作前检查 ownership 截止和本地撤销；不能撤回已发外部请求。""" + if self.lost.is_set() or time.monotonic() >= self.valid_until_monotonic: + self.lost.set() + raise StoreOperationError("owner_lost", "Shard lease is no longer valid") + if self.revoked.is_set(): + raise StoreOperationError("owner_lost", "Shard lease execution is revoked") + + +@dataclass(frozen=True) +class ShardWindow: + progress_ms: int # C：最后发布的分钟上界。 + upper_ms: int # U：当前源 TIME 去掉秒和毫秒；只是本轮局部快照。 + redis_now_ms: int + + +@dataclass(frozen=True) +class SourceCursor: + due_at_ms: int + item_id: str # score 相同按 Redis member 的二进制字节顺序续读。 + + +@dataclass(frozen=True) +class SourceItem: + item_id: str + due_at_ms: int + + +@dataclass(frozen=True) +class SourcePage: + items: tuple[SourceItem, ...] + next_cursor: SourceCursor | None # None 表示本次分钟范围已读完。 + + +@dataclass(frozen=True) +class BucketRef: + item_type: str + shard_id: int + minute_ms: int # 窗口 (minute_ms - 60000, minute_ms] 的结束边界。 + + +@dataclass(frozen=True) +class RegistryCursor: + minute_ms: int + member: str + + +@dataclass(frozen=True) +class RegistryPage: + buckets: tuple[BucketRef, ...] + next_cursor: RegistryCursor | None + + +@dataclass(frozen=True) +class PublishResult: + status: Literal["published", "already_published"] + progress_ms: int + + +@dataclass(frozen=True) +class Claim: + bucket: BucketRef + item_id: str + claim_score: int + + +@dataclass(frozen=True) +class ClaimBatch: + claims: tuple[Claim, ...] + next_retry_layer: int # 消费者下次从该层继续，避免每次只检查低 attempt。 + layers_scanned: int + exhausted: tuple[Claim, ...] # 最终机会已超时，尚未删除；供 consumer 写日志后条件 ACK。 + + +# 内部模型不承载 session policy、消息内容、event_id 或 group_id。 + + +class AsyncExecutor(Protocol): + async def run(self, operation: Callable[[], Awaitable[T]]) -> T: + """在固定 I/O loop 创建并执行 coroutine，调用方立即等待。 + + operation 必须是工厂，不能把在别的 loop 创建的 Redis Future 传进来。 + 取消/超时不能被解释为服务端已回滚；写入结果未知由 store 分类。 + """ + ... + + +def validate_item_type(item_type: str) -> None: + """只校验 [a-z][a-z0-9_]{0,63} 格式；不认识 session/watch 等具体业务类型。""" + ... + + +def validate_identity(identity: ItemIdentity) -> None: + """校验类型名及非空字符串身份，具体组件由 keys.py 可逆编码。""" + ... + + +def validate_report(task: ExpiryRegistration) -> None: + """校验类型名称格式、非空身份、正 timeout、毫秒整数范围。 + + 类型是否已注册由 facade/内部目录判断，这里没有业务类型白名单。 + FIXED -> unsupported_mode；坏身份 -> invalid_identity；参数错误不可重试。 + bool 不当作 int 接受；所有值必须处于 Redis Lua 的精确整数范围。 + 使用 metadata_ttl_ms 校验两倍 timeout 的毫秒计算；TTL 不由业务单独传入。 + """ + ... + + +def minute_start_ms(timestamp_ms: int) -> int: + """Unix 毫秒归一到分钟起点。""" + return timestamp_ms // MINUTE_MS * MINUTE_MS + + +def scan_cutoff_ms(now_ms: int) -> int: + """扫描包含式上界：源 TIME 向下取整到当前分钟，不减 1ms、不预取下一分钟。""" + return minute_start_ms(now_ms) + + +def bucket_minute_ms(due_at_ms: int) -> int: + """due 保留原精度；返回包含它的结束分钟，整分钟 due 属于该分钟窗口。""" + if type(due_at_ms) is not int or not 0 <= due_at_ms <= MAX_EXACT_INTEGER: + raise ExpiryError("invalid_report", "Due time is outside the supported range") + minute = ((due_at_ms + MINUTE_MS - 1) // MINUTE_MS) * MINUTE_MS + if minute > MAX_EXACT_INTEGER: + raise ExpiryError("invalid_report", "Due time is outside the supported range") + return minute + + +def metadata_ttl_ms(timeout_seconds: int) -> int: + """M 创建与续期统一使用 max(两倍 timeout,24小时)，返回 PEXPIRE 的毫秒值。""" + if type(timeout_seconds) is not int or not 0 < timeout_seconds <= MAX_EXACT_INTEGER // 2000: + raise ExpiryError("invalid_report", "Timeout is outside the supported range") + return max(timeout_seconds * 2000, MIN_METADATA_TTL_MS) + + +def encode_claim_score(attempt: int, started_at_seconds: int) -> int: + """与 Lua 算术完全一致；非法范围报错，不静默截断。""" + if type(attempt) is not int or not 0 <= attempt <= MAX_ATTEMPT: + raise ValueError("attempt out of range") + if type(started_at_seconds) is not int or not 0 < started_at_seconds < SCORE_TIME_BASE: + raise ValueError("claim timestamp out of range") + return attempt * SCORE_TIME_BASE + started_at_seconds + + +def decode_claim_score(score: int) -> tuple[int, int]: + """仅解码已领取记录；0 是 ready 哨兵，由调用者先处理。""" + if type(score) is not int or not 0 < score < (MAX_ATTEMPT + 1) * SCORE_TIME_BASE: + raise ValueError("invalid claim score") + attempt, started_at = divmod(score, SCORE_TIME_BASE) + if started_at == 0: + raise ValueError("invalid claim timestamp") + return attempt, started_at +``` + +公开错误码包括 invalid_identity、invalid_report、unsupported_mode、unregistered_type、duplicate_handler、invalid_handler、registration_closed、reporting_disabled、not_started、timer_unavailable。namespace、owner、progress、clock 及结果未知状态属于内部恢复错误，由 facade 映射后返回业务；业务 NotFound 由业务 handler 转成 COMPLETED。 + + +## 3. keys.py：编码和寻址 + +```python +from dataclasses import dataclass +from .api import ItemIdentity +from .models import BucketRef, OwnerRole, validate_identity + +HASH_SEED = 0 +IDENTITY_ENCODING = "urlsafe_base64" + + +def encode_component(value: str) -> str: + """UTF-8 -> URL-safe Base64，无 '=' 填充；身份字段不能为空。""" + ... + + +def decode_component(value: str) -> str: + """严格解码并重编码校验规范形式，拒绝别名、坏 UTF-8、非法字符。""" + ... + + +def encode_item_id(identity: ItemIdentity) -> str: + """类型名 + 三个编码组件；不能将用户原始花括号放进 Redis key。""" + validate_identity(identity) + return ":".join((identity.item_type, encode_component(identity.account_id), + encode_component(identity.user_id), encode_component(identity.local_id))) + + +def decode_item_id(item_id: str) -> ItemIdentity: + """严格四段，校验类型名格式；不查业务白名单，用于框架内部精确路由。""" + ... + + +def shard_for(item_id: str, shard_count: int) -> int: + """xxhash.xxh64(item_id.encode('utf-8'), seed=HASH_SEED).intdigest() % N。""" + ... + + +@dataclass(frozen=True) +class ExpiryKeys: + service_key: str # 构造时先 encode_component(service_name)，禁止直接传原始字符串。 + + def namespace(self) -> str: + """协议 manifest HASH；只保存固定的分片、哈希和身份编码配置。""" + return f"expiry:namespace:{self.service_key}" + + def metadata(self, item_id: str) -> str: + """M：单 item 身份缓存。""" + return f"expiry:metadata:{self.service_key}:{item_id}" + + def source(self, shard_id: int) -> str: + """R：不加真实 hash tag。""" + return f"expiry:publisher:source:{self.service_key}:{shard_id}" + + + + def control_tag(self, shard_id: int) -> str: + """返回带花括号的真实 tag，避免不同逻辑内核控制 key 扎堆。""" + return f"{{{self.service_key}:{shard_id}}}" + + def progress(self, shard_id: int) -> str: + """C：与 G/调度 owner 同槽。""" + return f"expiry:publisher:progress:{self.control_tag(shard_id)}" + + def registry(self, shard_id: int) -> str: + """G：只登记已经写完并发布的 B。""" + return f"expiry:consumer:registry:{self.control_tag(shard_id)}" + + def owner(self, role: OwnerRole, shard_id: int, *, item_type: str | None = None) -> str: + """不同角色用不同 key，scheduler 与 callback 不互抢。""" + ... + + def bucket(self, ref: BucketRef) -> str: + """B：无需与 G 同槽。""" + return (f"expiry:consumer:bucket:{self.service_key}:{ref.item_type}:" + f"{ref.shard_id}:{ref.minute_ms}") + + + + def registry_member(self, ref: BucketRef) -> str: + """序列化注册项，不带 namespace，因为 G 已按 service_name 隔离。""" + return f"{ref.item_type}:{ref.shard_id}:{ref.minute_ms}" + + def parse_registry_member(self, value: str, *, expected_shard: int) -> BucketRef: + """校验 type、shard 和分钟对齐，拒绝跨 shard/损坏注册项。""" + ... +``` + +## 4. metrics.py：可观测性接口 + +```python +from collections.abc import Callable, Mapping +from .models import Claim + + +class ExpiryMetrics: + """包装内核指标 sink；内部 publisher 进程可使用同一 sink 或结构化日志。 + + 标签限于固定 operation/item_type/outcome；不以 item_id/account_id 作标签。 + sink 失败不能影响业务结果，批量任务不逐项写成功日志。 + """ + + def __init__(self, sink: Callable[[str, float, Mapping[str, str]], None] | None = None): + """绑定可选指标 sink；不连接 Redis 或初始化业务服务。""" + self._sink = sink + + def count(self, name: str, value: int = 1, *, item_type: str = "", + outcome: str = "") -> None: + """计数：report/claim/ACK/回源/失败等；内部隔离 sink 异常。""" + ... + + def observe(self, name: str, value: float, *, item_type: str = "", + outcome: str = "") -> None: + """耗时、页大小、retry 层数等分布。单位在指标名中固定。""" + ... + + def log_exhausted(self, claim: "Claim", *, max_attempts: int, + reason: str, outcome_unknown: bool) -> None: + """输出结构化耗尽日志：身份/窗口/claim/机会数/原因，不含消息正文。 + + 日志进入现有日志设施，不写 Redis。失败向调用者抛错，不能声称已记录。 + 日志与随后 ACK 不是事务，允许重复或崩溃时丢失；不伪造恰好一次。 + """ + ... + + def gauge(self, name: str, value: float, *, item_type: str = "") -> None: + """当前 backlog、inflight、owner 数等；不可逐资源生成系列。""" + ... +``` + +## 5. redis_store.py：完整存储接口 + +这个类集中管理 Redis 数据操作，scheduler/consumer 不直接取得 Redis 客户端。以下所有公开 async 方法都通过 `_io.run(...)` 执行原始客户端操作；标记 `_on_loop` 的方法只能从该 I/O loop 调用。逻辑流程可以跨多次 I/O，不伪装成跨 slot 事务。 + +```python +from __future__ import annotations + +from typing import Any, Literal, Sequence +from .keys import ExpiryKeys +from .metrics import ExpiryMetrics +from .api import ExpiryRegistration, ExpiryRegistrationResult +from .models import ( + AsyncExecutor, BucketRef, Claim, ClaimBatch, NamespaceSpec, OwnerRole, + ItemMetadata, OwnerToken, PublishResult, RegistryCursor, RegistryPage, + ShardWindow, SourceCursor, SourcePage, StoreOperationError, metadata_ttl_ms, +) +from .config import ExpiryRedisConfig + +RetryPolicy = Literal["read", "idempotent", "resolve_by_caller"] + + +class RedisExpiryStore: + def __init__(self, config: ExpiryRedisConfig, keys: ExpiryKeys, + io: AsyncExecutor, metrics: ExpiryMetrics) -> None: + """仅保存配置；此处不创建绑定调用方 loop 的 Redis 连接。""" + self._config, self._keys, self._io, self._metrics = config, keys, io, metrics + self._client: Any = None # 仅 _on_loop 方法访问。 + self._scripts: dict[str, str] = {} + self._ready = False + + async def open(self) -> None: + """在 I/O loop 创建 standalone/cluster 客户端，PING 并检查所需命令。""" + await self._io.run(self._open_on_loop) + + async def close(self) -> None: + """先拒绝新请求，再在 I/O loop 关闭连接；loop 此时仍须存活。""" + await self._io.run(self._close_on_loop) + + async def ensure_namespace(self, spec: NamespaceSpec) -> None: + """新 namespace: INITIALIZING -> 幂等初始化 N 个 C=0 -> READY。 + + 所有 reporter 等待 READY。中断初始化只补缺项、不覆盖已有 C。 + READY namespace 的缺 C 是损坏，不按当前时间自动重置。 + 实际N/hash/编码配置不匹配 -> namespace_mismatch；没有独立格式版本字段。 + """ + ... + + async def ensure_metadata(self, item_id: str, task: ExpiryRegistration) -> None: + """内部计算 TTL 后调用 ensure_metadata.lua；创建和续期使用同一个毫秒值。 + + ttl_ms=max(2*timeout_seconds*1000,86_400_000)，无独立客户端 EXISTS。 + """ + identity = task.identity + ttl_ms = metadata_ttl_ms(task.timeout_seconds) + await self._eval( + "ensure_metadata", [self._keys.metadata(item_id)], + [identity.item_type, identity.local_id, identity.account_id, identity.user_id, ttl_ms], + retry="idempotent", + ) + + async def read_metadata(self, item_id: str) -> ItemMetadata | None: + """None 只表示 Redis key 不存在；网络失败抛错，身份冲突也抛错。""" + ... + + async def upsert_resource(self, shard_id: int, item_id: str, + task: ExpiryRegistration) -> ExpiryRegistrationResult: + """仅 report 调用：单 R Lua 用 TIME 计算 max(requested_due,now+1) 后 ZADD。 + + 不读 C，不写伴随状态，不做业务顺序比较；同 item_id 允许后到请求覆盖。 + 网络未知结果不自动无限重放，业务是否再次 report 按失败策略决定。 + """ + ... + + async def unregister_resource(self, shard_id: int, item_id: str) -> None: + """单 key ZREM R 后 DEL M；部分失败报错，不扫描 B，重复删除幂等。""" + ... + + async def read_progress(self, shard_id: int) -> int: + """读 C；READY namespace 中不存在则抛 namespace_corrupt。""" + ... + + async def read_scan_window(self, shard_id: int) -> ShardWindow: + """读 C，再通过 source_time.lua(R) 取得源 TIME，U=去掉秒和毫秒。 + + 只读，不更新 R 或 C。U=C 直接跳过；U SourcePage: + """调用原子分页脚本；after 为稳定 score/member 游标，不是 OFFSET。""" + ... + + async def reset_unpublished_minute(self, owner: OwnerToken, *, minute_ms: int, + expected_progress_ms: int, + item_types: Sequence[str]) -> None: + """先校验 owner/C，本地 lease 有效且 C 尚未覆盖，再 UNLINK 已知 type 的 B。 + + 仅允许 minute_ms > C；已发布分钟拒绝清空。 + 校验 C 与跨槽 UNLINK 不是事务；租约过期后的旧请求仍是主稿 10.3 的限制。 + """ + ... + + async def stage_items(self, owner: OwnerToken, bucket: BucketRef, + item_ids: Sequence[str]) -> None: + """未注册 B 上的有界 ZADD 0；只由该 shard 独占 scheduler 调用。 + + 返回意味着本批明确成功；超时返回 result_unknown，不在后台盲目重放。 + 整个分钟所有 stage 调用都结束后，才能 publish。 + """ + ... + + async def publish_minute(self, owner: OwnerToken, *, expected_progress_ms: int, + minute_ms: int, buckets: Sequence[BucketRef]) -> PublishResult: + """同槽 Lua 注册所有 B 并推进 C；owner/C 错误在任何写入前返回。 + + minute_ms>expected_progress_ms，允许跨过扫描已确认的空分钟；发布后 C=minute_ms。 + buckets 必须非空，空分钟跳过；源扫描完成由 scheduler 保证。 + """ + ... + + async def cleanup_source(self, shard_id: int, through_ms: int) -> int: + """ZREMRANGEBYSCORE R -inf through_ms；返回删除数，单次耗时要监控。 + + 只接受已确认 C；不执行任何 ZADD。是否换成有界批删需压测后另行决定。 + """ + ... + + async def read_registry_page(self, shard_id: int, *, after: RegistryCursor | None, + limit: int) -> RegistryPage: + """G 只包含已发布分钟，按 score/member 稳定分页；不全量 ZRANGE。 + + 页末成员可能被摘除，使用与资源分页相同的原子定位策略。 + """ + ... + + async def claim(self, bucket: BucketRef, *, limit: int, max_attempts: int, + processing_timeout_seconds: int, + retry_layer: int, retry_layers_per_poll: int) -> ClaimBatch: + """单 B Lua：有界领取，最后机会超时项只返回 exhausted，不删除或重领。 + + 每 claim 占一次机会；session 查询0/1/2层，超时检查不可被新项饿死。 + exhausted 由 consumer 日志后条件 ACK，无 Redis 失败存储。 + 响应未知不自动重放 claim，等超时按计数恢复。 + """ + ... + + async def ack(self, claim: Claim) -> bool: + """相同 score 才删除；False 表示已删除或被新领取，不再调用业务补偿。 + + 使用 retry=idempotent 在请求预算内重试同一条件 ACK，不能重跑 handler。 + """ + ... + + + + + + async def remove_registration_if_empty(self, bucket: BucketRef) -> bool: + """读 B 为空后摘 G；依赖发布后不再生产，允许遗留空注册项。""" + ... + + async def try_acquire_owner(self, role: OwnerRole, shard_id: int, token: str, + ttl_seconds: int) -> OwnerToken | None: + """SET NX PX，两个角色均使用120秒TTL；忙返回None。 + + 同token仅用于确认本次领取响应，不延长已失败lease；下一次领取必须新token。 + """ + ... + + async def renew_owner(self, owner: OwnerToken, ttl_seconds: int) -> OwnerToken | None: + """比较 token 后 PEXPIRE；缺失/不匹配返回None，异常即本次lease失败。 + + 不能重新创建锁，不能在丢锁后用同token恢复处理；两种角色均适用。 + """ + ... + + async def assert_owner(self, owner: OwnerToken) -> None: + """读取 token 验证；失败抛 owner_lost。非跨槽 B 写 fencing。""" + ... + + async def release_owner(self, owner: OwnerToken) -> bool: + """Lua 比较 token 后删除，不清除后继 owner。""" + ... + + async def _open_on_loop(self) -> None: + """延迟 import redis.asyncio；缺 extra 时给可操作的依赖错误。""" + ... + + async def _close_on_loop(self) -> None: + """关闭两个 Redis 模式下的客户端；清空缓存引用。""" + ... + + async def _initialize_shard(self, shard_id: int) -> None: + """仅 namespace INITIALIZING 时 SET C 0 NX，已有值不覆盖。 + + 全部初始化完成后置READY；report须等待READY，不按当前时间跳过历史源数据。 + """ + ... + + async def _command(self, name: str, *args: object, + retry: RetryPolicy = "read") -> Any: + """将客户端命令封装为 coroutine 工厂，经 _io.run 发送并分类错误。""" + ... + + async def _eval(self, script: str, keys: Sequence[str], args: Sequence[object], + *, retry: RetryPolicy) -> Any: + """非 I/O loop 入口；只调度 _eval_on_loop，业务锁不带入这里。""" + return await self._io.run(lambda: self._eval_on_loop(script, keys, args, retry=retry)) + + async def _eval_on_loop(self, script: str, keys: Sequence[str], args: Sequence[object], + *, retry: RetryPolicy) -> Any: + """EVALSHA，NOSCRIPT 则对实际目标 slot EVAL；MOVED/ASK 遵从客户端。 + + 网络超时不对 resolve_by_caller 自动重试；返回 result_unknown。 + Lua 输入 key 显式列在 KEYS；跨 slot 参数在发送前拒绝。 + """ + ... + + def _read_script(self, name: str) -> str: + """读取 lua/_common.lua + lua/name.lua 并拼接，进程内缓存源码/SHA。""" + ... + + def _translate_redis_error(self, exc: Exception, *, retry: RetryPolicy) -> StoreOperationError: + """区分协议错误、可重试读错误、服务端执行结果未知；保留原异常链。""" + ... +``` + +不提供 `get_client()`，也不提供通用“跨 slot pipeline 事务”。`stage_items/reset_unpublished_minute` 是结果未知时需要上层协调的操作；条件 ACK 与读取有各自可重试语义；report 无版本去重，不自动声称重复请求无影响。 + +## 6. registry.py、service.py 与 __init__.py：三个业务方法，内部自行管理 + +### 6.1 registry.py：内部处理器注册表 + +```python +from dataclasses import dataclass +from types import MappingProxyType +from typing import Mapping +from .api import HandlerOptions, ExpiryHandler, ExpiryError +from .models import validate_item_type + + +@dataclass(frozen=True) +class HandlerBinding: + handler: ExpiryHandler + options: HandlerOptions + + +class HandlerRegistry: + """只在框架内部使用；处理器和并发设置不写入 Redis。""" + + def __init__(self) -> None: + """创建空注册表，不硬编码任何业务类型。""" + self._bindings: dict[str, HandlerBinding] = {} + self._frozen = False + + def register(self, item_type: str, handler: ExpiryHandler, options: HandlerOptions) -> None: + """校验类型名、异步 handle、正整数并发及 max_attempts∈[1,256]（bool 非法）。 + + 坏 handler/options -> invalid_handler；重复类型 -> duplicate_handler； + freeze 后修改 -> registration_closed；均使用公开 ExpiryError。 + """ + ... + + def freeze(self) -> Mapping[str, HandlerBinding]: + """runtime 在开始消费前冻结，返回只读注册快照；不能从业务接口调用。""" + ... +``` + +### 6.2 service.py：不导出的 facade 实现 + +```python +from dataclasses import dataclass, field +from .api import ( + HandlerOptions, ItemIdentity, ExpiryHandler, ExpiryRegistration, ExpiryRegistrationResult, ExpiryError, +) +from .keys import encode_item_id, shard_for +from .metrics import ExpiryMetrics +from .models import StoreOperationError, validate_report +from .redis_store import RedisExpiryStore +from .registry import HandlerRegistry + + +@dataclass +class ExpiryRuntimeState: + """框架私有生命周期状态，runtime 持有；业务对象不提供状态修改入口。""" + storage_ready: bool = False + accepting_calls: bool = False + + +class _ExpirySchedulerFacade: + """实现 ExpiryScheduler；只有 runtime 构造，业务通过依赖注入得到协议引用。""" + + def __init__(self, store: RedisExpiryStore, registry: HandlerRegistry, + state: ExpiryRuntimeState, *, shard_count: int, + reporting_enabled: bool, metrics: ExpiryMetrics) -> None: + """保存内部依赖；业务不负责构造 store、keys、registry 或 state。""" + self._store, self._registry, self._state = store, registry, state + self._metrics, self._shard_count = metrics, shard_count + self._reporting_enabled = reporting_enabled + + async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: + """执行一次计时写入，将内部存储错误映射为业务错误。""" + self._require_ready() + if not self._reporting_enabled: + raise ExpiryError("reporting_disabled", "Timer reporting is disabled") + validate_report(task) + self._ensure_supported_type(task.identity.item_type) + try: + item_id = encode_item_id(task.identity) + shard_id = shard_for(item_id, self._shard_count) + result = await self._store.upsert_resource(shard_id, item_id, task) + await self._store.ensure_metadata(item_id, task) + except StoreOperationError as exc: + raise self._business_error(exc) from exc + self._metrics.count("timer_report", item_type=task.identity.item_type, + outcome="written") + return result + + async def unregister(self, identity: ItemIdentity) -> None: + """删除受支持类型的当前计时；不要求本进程仍有该类型 handler。""" + self._require_ready() + self._ensure_supported_type(identity.item_type) + item_id = encode_item_id(identity) + try: + await self._store.unregister_resource(shard_for(item_id, self._shard_count), item_id) + except StoreOperationError as exc: + raise self._business_error(exc) from exc + + def register_handler(self, item_type: str, handler: ExpiryHandler, *, + options: HandlerOptions | None = None) -> None: + """同步本地注册，允许在 storage 启动前调用；运行后由内部 registry 拒绝修改。""" + self._registry.register(item_type, handler, options or HandlerOptions()) + + @staticmethod + def _ensure_supported_type(item_type: str) -> None: + """按 SUPPORTED_ITEM_TYPES 校验；未知类型在任何 Redis 写入前失败。""" + ... + + def _require_ready(self) -> None: + """未初始化或关闭中抛 not_started；就绪状态只能由内部生命周期管理。""" + if not (self._state.storage_ready and self._state.accepting_calls): + raise ExpiryError("not_started", "Expiry scheduling is not ready") + + def _business_error(self, exc: StoreOperationError) -> ExpiryError: + """参数/身份错误保留公开错误码；内部不可用映射 timer_unavailable 并记内部指标。 + + 不向业务返回 owner、claim score、Lua 错误格式或 outcome_unknown 字段。 + """ + ... +``` + +### 6.3 __init__.py：限定业务可依赖的导出 + +```python +from .api import ( + CallbackRequest, CallbackResult, CallbackStatus, HandlerOptions, + ItemIdentity, ExpiryHandler, ExpiryRegistration, ExpiryScheduler, + ExpiryRegistrationResult, SESSION_ITEM_TYPE, ExpiryError, ExpiryMode, +) + +__all__ = [ + "ExpiryScheduler", "SESSION_ITEM_TYPE", "ItemIdentity", "ExpiryRegistration", "ExpiryRegistrationResult", + "ExpiryMode", "ExpiryError", "HandlerOptions", "ExpiryHandler", + "CallbackRequest", "CallbackResult", "CallbackStatus", +] +``` + +包入口不导出 facade、store、registry、runtime、Redis 配置、Claim 或 BucketRef。Python 模块路径不构成安全隔离，但业务模块只依赖此公开导出，内部类型不作为兼容接口。 + +### 6.4 独立调度器如何得到类型信息 + +`api.py` 中的 `SUPPORTED_ITEM_TYPES` 是类型信息的唯一来源。业务 `register_handler()` 只允许注册清单内类型；report 在写 Redis 前按同一清单校验。独立调度器直接引用清单，不需要加载业务 Handler，也不访问 Redis 类型目录；处理器对象和 HandlerOptions 只留在本地进程。 + +调度器先取得本轮源时间上界，再使用 `SUPPORTED_ITEM_TYPES` 定位并清理未发布分钟的临时 B。注册表 G 仍是唯一的“桶可以消费”的标志。类型下线前必须先排空 source、bucket 和 processing 数据，再从代码清单删除。 + +回调实例应加载其负责业务的注册模块；遇到自己没有 handler 的历史/新类型，保留任务并报警，不静默 ACK。首期不做运行中替换处理器或调整并发；新增业务在进程启动阶段完成注册。 + +当前 callback owner 按 shard 分配，同一 namespace 的回调副本应加载同一组业务处理器。缺少 handler 的 owner 不会自动把该类型让给其他副本；新增类型应先部署对应回调注册，再开放业务上报。按类型部署不同 worker 属于另一种路由模型，不在首期隐式支持。 + + +## 7. publisher.py：持锁清空到期积压，空范围跳过 + +```python +from __future__ import annotations + +import asyncio +import time +from collections import defaultdict +from typing import Sequence +from .keys import decode_item_id +from .metrics import ExpiryMetrics +from .models import ( + MINUTE_MS, BucketRef, OwnerRole, PublishResult, + StoreOperationError, bucket_minute_ms, +) +from .redis_store import RedisExpiryStore +from .config import ExpiryConfig +from .shard_runner import ShardExecution, ShardRunner + + +async def _wait_until_next_round(stop: asyncio.Event, delay: float) -> None: + """无租约时的轮间等待；stop可唤醒，不负责续租或失锁监视。""" + try: + await asyncio.wait_for(stop.wait(), delay) + except TimeoutError: + pass + + +class ExpiryPublisher: + def __init__(self, store: RedisExpiryStore, runner: ShardRunner, + config: ExpiryConfig, metrics: ExpiryMetrics) -> None: + """worker拥有shard遍历与分钟节奏，runner只执行一次单shard工作。""" + self._store, self._runner, self._config, self._metrics = store, runner, config, metrics + + async def run(self, stop: asyncio.Event) -> None: + """每轮递增遍历全部shard；shard_not_stopped必须上抛，不能继续抢锁。""" + while not stop.is_set(): + for shard_id in range(self._config.shard_count): + if stop.is_set(): + break + try: + await self._runner.run_if_acquired( + OwnerRole.SCHEDULER, shard_id, self.process_shard, stop, + ) + except StoreOperationError as exc: + if exc.code == "shard_not_stopped": + raise + self._metrics.count("timer_shard_round", outcome="failed") + except Exception: + self._metrics.count("timer_shard_round", outcome="failed") + if not stop.is_set(): + delay = max(0.05, 60.0 - time.time() % 60.0) + await _wait_until_next_round(stop, delay) + + async def process_shard(self, lease: ShardExecution, stop: asyncio.Event) -> None: + """持续处理全部到期积压，不因页数/分钟数预算交出锁。 + + 每步检查lease；取消后退出旧函数，新领取从C重建。 + _process_minute返回None表示重读后该分钟已空，本地扫描下界前进但不写C。 + """ + while not stop.is_set(): + lease.ensure_active() + window = await self._store.read_scan_window(lease.owner.shard_id) + progress = window.progress_ms + lease.ensure_active() + await self._store.cleanup_source(lease.owner.shard_id, progress) + if window.upper_ms <= progress: + return + after_ms = progress + published_any = False + while after_ms < window.upper_ms and not stop.is_set(): + lease.ensure_active() + page = await self._store.read_source_page( + lease.owner.shard_id, lower_exclusive_ms=after_ms, + upper_inclusive_ms=window.upper_ms, after=None, limit=1, + ) + if not page.items: + break + minute = bucket_minute_ms(page.items[0].due_at_ms) + result = await self._process_minute(lease, progress_ms=progress, minute_ms=minute) + if result is not None: + progress = result + published_any = True + after_ms = minute + if not published_any: + return + # 确认追平当前上界后再读一次新上界，不持锁忙等尚未到期资源。 + + async def _process_minute(self, lease: ShardExecution, *, progress_ms: int, + minute_ms: int) -> int | None: + """分页读完整个非空分钟 -> 写完B -> 原子发布G/C -> 清理R。""" + lease.ensure_active() + await self._store.reset_unpublished_minute( + lease.owner, minute_ms=minute_ms, expected_progress_ms=progress_ms, + item_types=tuple(sorted(SUPPORTED_ITEM_TYPES)), + ) + cursor = None + types_with_data: set[str] = set() + while True: + lease.ensure_active() + page = await self._store.read_source_page( + lease.owner.shard_id, lower_exclusive_ms=max(progress_ms, minute_ms - MINUTE_MS), + upper_inclusive_ms=minute_ms, after=cursor, limit=self._config.scan_batch_size, + ) + by_type: dict[str, list[str]] = defaultdict(list) + for item in page.items: + identity = decode_item_id(item.item_id) + if identity.item_type not in SUPPORTED_ITEM_TYPES: + raise StoreOperationError("unregistered_type", "Expiry item type is not supported") + by_type[identity.item_type].append(item.item_id) + for item_type, item_ids in by_type.items(): + lease.ensure_active() + ref = BucketRef(item_type, lease.owner.shard_id, minute_ms) + await self._store.stage_items(lease.owner, ref, item_ids) + types_with_data.add(item_type) + cursor = page.next_cursor + if cursor is None: + break + if not types_with_data: + return None + lease.ensure_active() + refs = tuple(BucketRef(t, lease.owner.shard_id, minute_ms) for t in sorted(types_with_data)) + result = await self._publish_completed_minute( + lease, expected_progress_ms=progress_ms, minute_ms=minute_ms, buckets=refs, + ) + lease.ensure_active() + await self._store.cleanup_source(lease.owner.shard_id, result.progress_ms) + return result.progress_ms + + async def _publish_completed_minute(self, lease: ShardExecution, *, expected_progress_ms: int, + minute_ms: int, buckets: Sequence[BucketRef]) -> PublishResult: + """同一有效lease中保留不可变发布参数，超时查C或重试同一publish。 + + C未前进不代表旧请求不会执行，不能重置B。每次I/O前检查lease。 + lease失败立即退出，不在后续新lease中恢复本函数；未决请求风险见主稿10.3。 + """ + ... +``` + +分页是内存/命令预算，持锁期间可持续处理任意多个分钟。source_time只读，R的score只有report更新；调度的R变更仅为按已确认C清理。 + +## 8. consumer.py：清空积压、三次机会、仅日志终结 + +```python +from __future__ import annotations + +import asyncio +from collections import deque +from dataclasses import dataclass, field +from typing import Mapping +from .api import CallbackRequest, CallbackStatus, ExpiryError +from .keys import decode_item_id +from .metrics import ExpiryMetrics +from .models import BucketRef, Claim, OwnerRole, StoreOperationError, decode_claim_score +from .registry import HandlerBinding +from .redis_store import RedisExpiryStore +from .config import ExpiryConfig +from .shard_runner import ShardExecution, ShardRunner + + +@dataclass +class _TypeState: + limit: int + reserved: int = 0 + buckets: deque[BucketRef] = field(default_factory=deque) + queued: set[BucketRef] = field(default_factory=set) + tasks: set[asyncio.Task[None]] = field(default_factory=set) + + +class ExpiryConsumer: + def __init__(self, store: RedisExpiryStore, runner: ShardRunner, + bindings: Mapping[str, HandlerBinding], config: ExpiryConfig, + metrics: ExpiryMetrics) -> None: + """类型并发不跨shard叠加；旧tasks未退出时不得重置共享state。""" + self._store, self._runner, self._bindings = store, runner, bindings + self._config, self._metrics = config, metrics + self._types = {t: _TypeState(b.options.concurrency) for t, b in bindings.items()} + self._retry_layers: dict[BucketRef, int] = {} + + async def run(self, stop: asyncio.Event) -> None: + """worker自己逐个竞争全部shard；轮后按callback_poll_seconds等待。""" + while not stop.is_set(): + for shard_id in range(self._config.shard_count): + if stop.is_set(): + break + try: + await self._runner.run_if_acquired( + OwnerRole.CALLBACK, shard_id, self.process_shard, stop, + ) + except StoreOperationError as exc: + if exc.code == "shard_not_stopped": + raise + self._metrics.count("timer_shard_round", outcome="failed") + except Exception: + self._metrics.count("timer_shard_round", outcome="failed") + if not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), self._config.callback_poll_seconds) + except TimeoutError: + pass + + async def process_shard(self, lease: ShardExecution, stop: asyncio.Event) -> None: + """消费业务流程不自建lost等待task；runner监视失锁/stop/grace并撤销子任务。""" + self._reset_shard_state() + try: + while not stop.is_set(): + lease.ensure_active() + found = await self._discover(lease) + for item_type in self._types: + await self._fill_slots(lease, item_type) + tasks = {task for state in self._types.values() for task in state.tasks} + has_queued = any(state.buckets for state in self._types.values()) + if not found and not has_queued and not tasks: + return + await lease.wait(self._config.callback_poll_seconds, stop) + # 正常stop等待已启动callbacks；外层runner仍续租并监视lost/有效截止， + # 到grace截止会取消process/children，故此处不是独立无界关闭路径。 + tasks = {task for state in self._types.values() for task in state.tasks} + if tasks: + await asyncio.wait(tasks) + lease.ensure_active() + finally: + if not any(not task.done() for state in self._types.values() for task in state.tasks): + self._reset_shard_state() + + def _reset_shard_state(self) -> None: + """仅在旧tasks全部终止后清队列/计数；每次单shard执行清retry层缓存。""" + for state in self._types.values(): + if any(not task.done() for task in state.tasks): + raise StoreOperationError("shard_not_stopped", "Previous callbacks are still running") + state.tasks.clear() + state.buckets.clear() + state.queued.clear() + state.reserved = 0 + self._retry_layers.clear() + + async def _discover(self, lease: ShardExecution) -> bool: + """lease检查后从G稳定分页，返回是否仍有可处理桶；每次发现有缓存预算。 + + 一页满不等于处理完；持续推进游标、绕回确认空。没有handler的类型不领取。 + """ + ... + + async def _fill_slots(self, lease: ShardExecution, item_type: str) -> None: + """计算空闲槽并claim；claim前后检查execution，retry层仅本次shard运行使用。""" + state = self._types[item_type] + available = state.limit - state.reserved + if available <= 0 or not state.buckets: + return + bucket = state.buckets.popleft() + state.queued.discard(bucket) + lease.ensure_active() + binding = self._bindings[item_type] + batch = await self._store.claim( + bucket, limit=available, max_attempts=binding.options.max_attempts, + processing_timeout_seconds=self._config.processing_timeout_seconds, + retry_layer=self._retry_layers.get(bucket, 0), + retry_layers_per_poll=self._config.retry_layers_per_poll, + ) + lease.ensure_active() + self._retry_layers[bucket] = batch.next_retry_layer + for claim in batch.exhausted: + await self._log_exhausted_and_ack(lease, claim, "processing_timeout") + for claim in batch.claims: + # 传工厂，不提前创建coroutine；runner强引用跟踪每个_execute子task。 + task = lease.spawn(lambda claim=claim: self._execute(lease, claim)) + state.reserved += 1 + state.tasks.add(task) + task.add_done_callback(state.tasks.discard) + lease.ensure_active() + removed = await self._store.remove_registration_if_empty(bucket) + if not removed and bucket not in state.queued: + state.queued.add(bucket) + state.buckets.append(bucket) + elif removed: + self._retry_layers.pop(bucket, None) + + async def _execute(self, lease: ShardExecution, claim: Claim) -> None: + """最多调用一次handler；COMPLETED后只重试同一ACK，不变成业务失败。""" + item_type = claim.bucket.item_type + completed = False + try: + lease.ensure_active() + request = await self._resolve_request(claim) + lease.ensure_active() + binding = self._bindings[item_type] + result = await asyncio.wait_for(binding.handler.handle(request), + timeout=self._config.processing_timeout_seconds) + lease.ensure_active() + if result.status is CallbackStatus.COMPLETED: + completed = True + await self._store.ack(claim) + lease.ensure_active() + await self._store.remove_registration_if_empty(claim.bucket) + else: + await self._finish_failed_attempt(lease, claim, "callback_retry") + except asyncio.CancelledError: + raise + except StoreOperationError as exc: + if exc.code == "owner_lost" or exc.outcome_unknown: + raise # 让runner保留失锁/未知结果语义，不吞掉后主动释放。 + self._record_callback_error(claim, exc) + if not completed: + lease.ensure_active() + await self._finish_failed_attempt(lease, claim, "callback_exception") + except Exception as exc: + self._record_callback_error(claim, exc) + if not completed: + lease.ensure_active() + await self._finish_failed_attempt(lease, claim, "callback_exception") + finally: + self._release_slot(item_type) + + async def _finish_failed_attempt(self, lease: ShardExecution, claim: Claim, reason: str) -> None: + """未耗尽留B等待超时；已耗尽先日志再条件ACK。lease失效不再执行清理。""" + attempt, _ = decode_claim_score(claim.claim_score) + if attempt + 1 >= self._bindings[claim.bucket.item_type].options.max_attempts: + await self._log_exhausted_and_ack(lease, claim, reason) + + async def _log_exhausted_and_ack(self, lease: ShardExecution, claim: Claim, reason: str) -> None: + """先输出耗尽日志，再ACK相同claim；错误保留B，之后可重复日志和清理。 + + 日志无Redis副本，与ACK不是事务。lease_expired/异常均保守标记结果未知。 + """ + try: + lease.ensure_active() + self._metrics.log_exhausted( + claim, max_attempts=self._bindings[claim.bucket.item_type].options.max_attempts, + reason=reason, outcome_unknown=True, + ) + lease.ensure_active() + await self._store.ack(claim) + lease.ensure_active() + await self._store.remove_registration_if_empty(claim.bucket) + except StoreOperationError as exc: + if exc.code == "owner_lost" or exc.outcome_unknown: + raise + self._record_callback_error(claim, exc) + except Exception as exc: + self._record_callback_error(claim, exc) + + async def _resolve_request(self, claim: Claim) -> CallbackRequest: + """M缺失可完整身份回源；M网络错误抛异常，身份冲突不可跨用户猜测。""" + identity = decode_item_id(claim.item_id) + if identity.item_type != claim.bucket.item_type: + raise ExpiryError("invalid_identity", "Bucket type differs from item ID") + metadata = await self._store.read_metadata(claim.item_id) + if metadata is not None and metadata.identity != identity: + raise ExpiryError("invalid_identity", "Metadata differs from item ID") + self._metrics.count("timer_metadata_lookup", item_type=identity.item_type, + outcome="miss" if metadata is None else "hit") + return CallbackRequest(identity) + + async def drain(self, timeout_seconds: float) -> None: + """兼容辅助入口，不替代runner关闭：有界观察，超时cancel后报未退出，不无界join。""" + tasks = {task for state in self._types.values() for task in state.tasks} + if not tasks: + return + _, pending = await asyncio.wait(tasks, timeout=max(0.0, timeout_seconds)) + for task in pending: + task.cancel() + if pending: + raise StoreOperationError("shard_not_stopped", "Callbacks did not drain within budget") + + def _release_slot(self, item_type: str) -> None: + """同一运行loop释放一次预留，任务完成/取消均执行。""" + ... + + def _record_callback_error(self, claim: Claim, exc: Exception) -> None: + """低基数错误指标和限频日志；不保存失败到Redis，不直接清成员。""" + ... + + # 持租约的poll等待使用ShardExecution.wait；consumer不维护单独lost task。 +``` + +任务claim超时与shard租约超时是两种机制；首期不续租单任务，条件ACK保护超时重领。第三次超时只日志清理，不再发起业务调用。 + +## 9. leases.py、shard_runner.py 与 runtime.py：token、执行和生命周期 + +### 9.1 leases.py：Coordinator 仅三种 token 操作 + +```python +from __future__ import annotations + +import asyncio +import time +import uuid +from .config import ExpiryConfig +from .metrics import ExpiryMetrics +from .models import OwnerRole, ShardLease, StoreOperationError +from .redis_store import RedisExpiryStore + + +class ShardLeaseCoordinator: + def __init__(self, store: RedisExpiryStore, config: ExpiryConfig, + metrics: ExpiryMetrics) -> None: + """只封装Redis token操作；不遍历shard，不创建process/heartbeat或等待轮间节奏。""" + self._store, self._config, self._metrics = store, config, metrics + + async def try_acquire(self, role: OwnerRole, shard_id: int) -> ShardLease | None: + """新随机token；截止=request开始monotonic+TTL，返回前检查未过期。""" + started = time.monotonic() + token = uuid.uuid4().hex + owner = await self._store.try_acquire_owner( + role, shard_id, token, self._config.owner_ttl_seconds, + ) + if owner is None: + return None + lease = ShardLease(owner, asyncio.Event(), started + self._config.owner_ttl_seconds) + lease.ensure_active() + return lease + + async def renew(self, lease: ShardLease) -> bool: + """任一失锁/异常/未知/取消或迟到成功使ownership永久不可信，旧lease不可复活。""" + try: + lease.ensure_active() + except StoreOperationError: + return False + started = time.monotonic() + previous_deadline = lease.valid_until_monotonic + try: + renewed = await self._store.renew_owner(lease.owner, self._config.owner_ttl_seconds) + except asyncio.CancelledError: + lease.lost.set() + raise + except Exception: + lease.lost.set() + return False + task = asyncio.current_task() + if (renewed is None or lease.lost.is_set() + or (task is not None and task.cancelling()) + or time.monotonic() >= previous_deadline): + lease.lost.set() + return False + if lease.revoked.is_set(): + return False + lease.owner = renewed + lease.valid_until_monotonic = started + self._config.owner_ttl_seconds + return True + + async def release(self, lease: ShardLease) -> None: + """runner确认本地工作/续租均终止且ownership可信后才调用；revoked本身不禁止释放。""" + if time.monotonic() >= lease.valid_until_monotonic: + lease.lost.set() + if lease.lost.is_set(): + return + try: + await self._store.release_owner(lease.owner) + except asyncio.CancelledError: + lease.lost.set() + raise + except Exception as exc: + if isinstance(exc, StoreOperationError) and exc.outcome_unknown: + lease.lost.set() + self._metrics.count("timer_owner_release", outcome="failed") +``` + +### 9.2 shard_runner.py：一次单 shard 执行，不替 worker 排轮次 + +```python +from __future__ import annotations + +import asyncio +import time +from typing import Awaitable, Callable +from .config import ExpiryConfig +from .leases import ShardLeaseCoordinator +from .metrics import ExpiryMetrics +from .models import OwnerRole, OwnerToken, ShardLease, StoreOperationError + +ShardProcessor = Callable[["ShardExecution", asyncio.Event], Awaitable[None]] + + +class ShardExecution: + """内部可撤销执行句柄；业务processor只见检查点、owner和受跟踪的spawn/wait。""" + + def __init__(self, lease: ShardLease) -> None: + self._lease = lease + self._children: set[asyncio.Task[None]] = set() + self._waiters: set[asyncio.Task[object]] = set() + self._error: BaseException | None = None + self._outcome_unknown = False + + @property + def owner(self) -> OwnerToken: + return self._lease.owner + + @property + def shard_id(self) -> int: + return self.owner.shard_id + + def ensure_active(self) -> None: + task = asyncio.current_task() + if task is not None and task.cancelling(): + # handler吞掉取消后也不能执行ACK；先同步标记unknown，不能等done callback。 + self._outcome_unknown = True + raise asyncio.CancelledError + self._lease.ensure_active() + + def spawn(self, operation: Callable[[], Awaitable[None]]) -> asyncio.Task[None]: + """先检查再创建task；传coroutine工厂，避免撤销后遗留未await的coroutine。""" + self.ensure_active() + + async def guarded() -> None: + self.ensure_active() + await operation() + + task = asyncio.create_task(guarded()) + self._children.add(task) + task.add_done_callback(self._child_done) + return task + + async def wait(self, delay_seconds: float, stop: asyncio.Event) -> None: + """等待poll/stop/lost/revoked；两侧检查，timeout不超过本地lease截止。 + + stop/lost/revoked的辅助waiters保持强引用，finally cancel；runner有界收集。 + """ + ... + + def _waiter_done(self, task: asyncio.Task[object]) -> None: + """摘除准确task并读取异常，不能泄漏辅助等待。""" + ... + + def _child_done(self, task: asyncio.Task[None]) -> None: + """摘除task并记录错误；cancelled/cancelling或outcome_unknown均保守标记unknown。""" + ... + + def _revoke(self) -> None: + self._lease.revoked.set() # 只撤销本地执行，不伪造ownership失锁。 + + +def _consume_exception(task: asyncio.Task[object]) -> None: + if not task.cancelled(): + task.exception() + + +class ShardRunner: + def __init__(self, leases: ShardLeaseCoordinator, config: ExpiryConfig, + metrics: ExpiryMetrics) -> None: + self._leases, self._config, self._metrics = leases, config, metrics + self._active: set[OwnerRole] = set() + self._pending: dict[OwnerRole, set[asyncio.Task[object]]] = {} + + @property + def has_pending(self) -> bool: + return any(not task.done() for tasks in self._pending.values() for task in tasks) + + async def wait_pending(self, timeout: float) -> bool: + """仅有界观察强引用保留的工作，不取消；返回是否全部终止。""" + tasks = {task for tasks in self._pending.values() for task in tasks if not task.done()} + if tasks: + await asyncio.wait(tasks, timeout=max(0.0, timeout)) + return not self.has_pending + + async def run_if_acquired(self, role: OwnerRole, shard_id: int, + process: ShardProcessor, stop: asyncio.Event) -> bool: + """同role已有active或pending时抛shard_not_stopped；stop/busy返回False。 + + acquire放入独立task并shield：调用者cancel后cancel请求task并有界_settle， + 不让抑制取消的Redis实现拖住join；未知领取只留TTL，不补release。 + 成功后创建全新ShardExecution并_run；返回True表示取得过租约，不代表业务成功。 + """ + ... + + async def _run(self, role: OwnerRole, execution: ShardExecution, + process: ShardProcessor, stop: asyncio.Event) -> None: + """创建processor/heartbeat/stop/lost tasks并独立监视有效截止。 + + 正常stop启动shutdown_timeout_seconds grace，heartbeat继续；grace期间仍监视 + lost/heartbeat/lease截止，不只等processor。失锁、grace超时、外层取消均forced。 + finally先revoke，再cancel processor/heartbeat/children/waiters，并有界_settle。 + 若renew请求正进行，同步置lost后再cancel，抑制取消后的成功不能复活lease。 + processor已返回却仍有children属于生命周期违例，forced且不得主动release。 + 所有task终止后读取processor/children异常，再决定release，最后向worker传播错误。 + 普通业务异常但已终止、无forced/unknown且ownership可信时可比较token释放； + forced/失锁/未知结果保守TTL。release本身也用asyncio.wait有界观察/保留pending。 + """ + ... + + async def _heartbeat(self, lease: ShardLease, renewing: asyncio.Event) -> None: + """后台每20秒调用Coordinator.renew；renewing标记请求在途，失败结束。""" + while True: + await asyncio.sleep(self._config.owner_renew_interval_seconds) + renewing.set() + try: + if not await self._leases.renew(lease): + return + finally: + renewing.clear() + + async def _settle(self, role: OwnerRole, tasks: set[asyncio.Task]) -> None: + """cancel后的join用asyncio.wait(timeout=request_budget_seconds)，不等取消确认。 + + await之前把未终止task放_pending强引用（包括acquire/release/heartbeat）， + done callback读取异常并只移除自己持有的准确集合，不清后轮同role的state。 + 预算后仍pending抛shard_not_stopped；再次取消cleanup也不能丢失强引用。 + """ + ... + + def _pending_done(self, role: OwnerRole, retained: set[asyncio.Task[object]], + task: asyncio.Task[object]) -> None: + """旧task真实退出才移除；保留集合identity检查避免误删后轮记录。""" + ... + + @staticmethod + def _not_stopped(role: OwnerRole) -> StoreOperationError: + return StoreOperationError("shard_not_stopped", f"The {role.value} shard still has active local work") +``` + +runner 的 pending 限制是**同角色的全部 shard**，不是只禁止重领原 shard；另一角色的生命周期独立。`asyncio.wait` 仅保证事件循环能运行时等待有界，不强杀协程；未合作取消的工作仍被强引用保留，worker必须停止同role轮次，依赖不能先关闭。只有正常终止、ownership仍可信且没有未知结果才可主动释放；普通业务异常本身不是“结果未知”，错误仍须传播。 + +这不改变Redis/Lua/protocol，不撤回已发Redis写入，不构成跨槽fencing；当前B不按token隔离的残余风险继续按主稿10.3保留。Session外层timeout检查和原commit设计不因runner重构而改变。 + +### 9.3 runtime.py：共享 runner 的组装与有界关闭 + +Redis I/O 的终止凭据是 `_RedisRequest.finished`，不是调用方 wrapper 被取消，也不是 `run_coroutine_threadsafe` 返回的 Future 状态。请求提交前登记强引用，Redis-loop task 及其原始 operation 真正退出后才完成 finished 并摘除登记；请求取消或超时后仍保留生命周期桥接。runner 的有界观察和 executor 的 quiesce/join 分别保护业务工作与远端 I/O，不能用 role task 的退出代替 Redis-loop 的真实退出。 + +关闭顺序为 `stop_roles → io.quiesce → io.run_cleanup(store.close) → io.close`。quiesce 拒绝新外部请求，等待已登记请求真实 done，预算耗尽抛 `shard_not_stopped`，不提前关 store。cleanup 在 Redis loop 运行 raw parent，不先占用 `_gate`；store.close 内部嵌套 `_io.run` 可在 quiesce 后执行并正常关闭 client，避免 max_inflight=1 时递归等待自身持有的 gate。io.close 再次 quiesce 后仅请求一次 loop.stop，以剩余预算 join 线程；线程仍活着则保留 loop/thread/gate 引用，后续 close 可重试，但 start 不能创建第二线程。store cleanup 成功即置 `_storage_closed=True`；若后续线程 join 超时，仍保留 `_storage_started/storage_ready`，重试 close 跳过已成功的 cleanup，只重新观察/等待线程退出，不向已 stop_requested 的 loop 再提交 store.close。只有线程真实退出后清 I/O 引用和 runtime 的 `_storage_started/storage_ready`；start_storage 成功重置 `_storage_closed=False`。core 保留 runtime/adapter 并传播未退出异常，阻止 QueueFS/AGFS 提前关闭。 + +这里的预算是调用方的有界观察，不是强杀保证：`run` 取消后的 join、`_execute` 的取消收尾和线程内部清理可能等待非合作协程；其生命周期必须仍可由 pending/requests/thread 引用观察。此机制不撤回服务端已收到的 Redis 写入，也不提供跨槽 fencing。Session 外层 timeout 与其余 core startup/optional backend 不变。 + +```python +from __future__ import annotations + +import asyncio +import concurrent.futures +import threading +import time +from dataclasses import dataclass, field +from typing import Awaitable, Callable, TypeVar +from .api import ExpiryScheduler +from .config import ExpiryConfig +from .consumer import ExpiryConsumer +from .keys import ExpiryKeys, encode_component +from .leases import ShardLeaseCoordinator +from .metrics import ExpiryMetrics +from .models import StoreOperationError +from .redis_store import RedisExpiryStore +from .publisher import ExpiryPublisher +from .registry import HandlerRegistry +from .service import _ExpirySchedulerFacade, ExpiryRuntimeState +from .shard_runner import ShardRunner + +T = TypeVar("T") + + +@dataclass(eq=False) +class _RedisRequest: + finished: concurrent.futures.Future = field(default_factory=concurrent.futures.Future) + cancelled: threading.Event = field(default_factory=threading.Event) + task: asyncio.Task | None = None # 仅Redis loop读写；finished只在实际task退出后完成。 + + +class RedisLoopExecutor: + """专属后台线程 + asyncio loop；跟踪真实Redis任务，不运行session commit。""" + + def __init__(self, *, max_inflight: int, request_budget_seconds: float) -> None: + self._max_inflight = max_inflight + self._request_budget_seconds = request_budget_seconds + self._loop: asyncio.AbstractEventLoop | None = None + self._thread: threading.Thread | None = None + self._gate: asyncio.Semaphore | None = None + self._accepting = False + self._ready = threading.Event() + self._requests: set[_RedisRequest] = set() + self._requests_lock = threading.Lock() + self._stop_requested = False + + async def start(self) -> None: + """已accepting时幂等；旧thread引用仍在则shard_not_stopped，不创建第二线程。 + + 清ready/stop_requested，启动线程并等待ready；gate仅在Redis loop创建。 + """ + ... + + async def run(self, operation: Callable[[], Awaitable[T]]) -> T: + """调用方取消后仍等待真实finished；多次取消不能切断wrapper的生命周期桥接。""" + loop = self._loop + if loop is not None and asyncio.get_running_loop() is loop: + # cleanup parent已登记，嵌套client关闭允许在quiesce之后执行。 + return await self._execute(operation) + if not self._accepting or loop is None: + raise StoreOperationError("not_started", "Redis expiry I/O loop is not running") + request = self._submit(operation) + finished = asyncio.wrap_future(request.finished) + try: + return await asyncio.shield(finished) + except asyncio.CancelledError: + self._cancel(request) + await self._join(finished) + raise + + async def quiesce(self, timeout_seconds: float) -> None: + """先拒绝新外部请求，以asyncio.wait有界等待登记请求的真实finished。 + + 包括尚未开始的请求；done读取异常，pending安装异常消费回调但不丢强引用。 + 未退出抛shard_not_stopped，不关闭client/loop/thread。 + """ + ... + + async def run_cleanup(self, operation: Callable[[], Awaitable[T]], + timeout_seconds: float) -> T: + """quiesce后提交tracked raw parent；不经_execute先占gate，避免nested run递归。 + + cleanup自身使用显式timeout而非普通请求预算；超时/取消发一次cancel， + 保留request并消费后续异常，超时报shard_not_stopped，不把取消当退出。 + """ + ... + + async def close(self, timeout_seconds: float) -> None: + deadline = time.monotonic() + max(0.0, timeout_seconds) + await self.quiesce(timeout_seconds) + loop, thread = self._loop, self._thread + if loop is not None and not self._stop_requested: + self._stop_requested = True + loop.call_soon_threadsafe(loop.stop) + if thread is not None: + await asyncio.to_thread(thread.join, max(0.0, deadline - time.monotonic())) + if thread.is_alive(): + raise self._not_stopped() # 保留引用，重试不重复stop；start不能新建线程。 + self._loop = None + self._thread = None + self._gate = None + + def _submit(self, operation: Callable[[], Awaitable[T]], *, + cleanup: bool = False) -> _RedisRequest: + """requests先登记后call_soon_threadsafe创建task，关闭须看到尚未开始的请求。 + + 普通请求task运行_execute；cleanup task直接await operation，不先占gate。 + task的done callback调用_request_done；预先取消的request启动时立即cancel。 + 无loop或已stop_requested拒绝；调度失败撤销登记。 + """ + ... + + def _cancel(self, request: _RedisRequest) -> None: + """threading.Event确保只向远端task发一次cancel；先取消后启动也不会漏掉。""" + ... + + def _request_done(self, request: _RedisRequest, task: asyncio.Task) -> None: + """真实task终止才将result/error/CancelledError写finished，然后摘除requests。""" + ... + + @staticmethod + def _consume_exception(future: asyncio.Future) -> None: + if not future.cancelled(): + future.exception() + + @staticmethod + async def _join(future: asyncio.Future) -> None: + while not future.done(): + try: + await asyncio.shield(future) + except (asyncio.CancelledError, Exception): + pass # 即使调用方再次cancel也保留等待；不重复cancel原始operation。 + if not future.cancelled(): + future.exception() + + @staticmethod + def _not_stopped() -> StoreOperationError: + return StoreOperationError( + "shard_not_stopped", "Redis expiry work or its I/O thread has not terminated", + ) + + def _thread_main(self) -> None: + """创建loop/gate并发布ready，run_forever结束后cancel/gather残余任务并关loop。 + + 线程内清理不冒充有界强杀：close在外层有界join，非合作清理仍alive则保留引用。 + """ + ... + + async def _execute(self, operation: Callable[[], Awaitable[T]]) -> T: + """gate内创建原始operation task，总请求预算含gate等待。 + + 超时cancel原始task后_join至真实done再报redis_timeout/outcome_unknown； + finally也join尚未退出的operation，防止外层task先done而远端工作仍存活。 + 预算触发取消并非硬终止，亦不证明服务端Redis写入撤销。 + """ + ... + + +@dataclass(frozen=True) +class ExpiryRuntimeRoles: + report: bool = False + publish: bool = False + consume: bool = False + + +class ExpiryRuntime: + def __init__(self, config: ExpiryConfig, *, roles: ExpiryRuntimeRoles, + metrics: ExpiryMetrics | None = None) -> None: + """构造 executor/store/service；不初始化 AGFS 或模型。""" + self._config = config + self._metrics = metrics or ExpiryMetrics() + self._keys = ExpiryKeys(encode_component(config.service_name)) + self._io = RedisLoopExecutor( + max_inflight=config.redis.max_inflight_requests, + request_budget_seconds=config.redis.request_budget_seconds, + ) + self._store = RedisExpiryStore(config.redis, self._keys, self._io, self._metrics) + self._registry = HandlerRegistry() + self._state = ExpiryRuntimeState() + self._facade = _ExpirySchedulerFacade( + self._store, self._registry, self._state, shard_count=config.shard_count, + reporting_enabled=roles.report, metrics=self._metrics, + ) + self._leases = ShardLeaseCoordinator(self._store, config, self._metrics) + self._shard_runner = ShardRunner(self._leases, config, self._metrics) + self._publisher: ExpiryPublisher | None = None + self._consumer: ExpiryConsumer | None = None + self._tasks: dict[str, asyncio.Task[None]] = {} + self._stop = asyncio.Event() + self._storage_started = False + self._storage_closed = False # store cleanup成功与I/O线程退出分别记录。 + + @property + def scheduler(self) -> ExpiryScheduler: + """仅供启动组装层取得协议引用，再注入业务；不把 runtime 交给业务。""" + return self._facade + + async def start_storage(self) -> None: + """依次 executor.start -> store.open -> ensure_namespace，置内部 storage_ready。 + + 成功时置_storage_started/storage_ready，并重置_storage_closed=False。 + 此时只开放本地 handler 注册，report 等 start_roles 完成才可用。 + 任一步失败按逆序释放已建资源，不启动 consumer。 + """ + ... + + async def start_roles(self) -> None: + """冻结本地 registry 后启动组装层选择的 publisher/consumer。 + + 同一_shard_runner注入scheduler/consumer，不把Coordinator交给worker。 + API 须先完成业务注册和 QueueFS 启动，再调用这里。 + 独立 publisher 没有本地 handler，直接使用 SUPPORTED_ITEM_TYPES。 + """ + bindings = self._registry.freeze() + self._stop.clear() + if self._roles.publish: + self._publisher = ExpiryPublisher( + self._store, self._shard_runner, self._config, self._metrics, + ) + self._spawn_role("publisher", self._publisher.run(self._stop)) + if self._roles.consume: + self._consumer = ExpiryConsumer( + self._store, self._shard_runner, bindings, self._config, self._metrics, + ) + self._spawn_role("callback", self._consumer.run(self._stop)) + self._state.accepting_calls = True + + async def wait(self) -> None: + """独立进程等待角色任务结束；任一角色非预期退出向 CLI 传播异常。""" + ... + + async def stop_roles(self) -> None: + """先拒绝新调用/set stop；grace后cancel，所有join均用asyncio.wait有界观察。 + + 保留未终止role task和runner pending；仅真实done才读异常并移除引用。 + 未退出则抛shard_not_stopped，保持store/I/O/业务依赖打开，允许后续再次观察。 + """ + self._state.accepting_calls = False + self._stop.set() + tasks = set(self._tasks.values()) + if tasks: + _, pending = await asyncio.wait(tasks, timeout=self._config.shutdown_timeout_seconds) + for task in pending: + task.cancel() + if pending: + await asyncio.wait(pending, timeout=self._config.redis.request_budget_seconds) + for name, task in list(self._tasks.items()): + if task.done(): + if not task.cancelled(): + task.exception() + self._tasks.pop(name, None) + settled = await self._shard_runner.wait_pending(self._config.redis.request_budget_seconds) + if self._tasks or not settled: + raise StoreOperationError("shard_not_stopped", "Expiry work is still running; dependencies remain open") + + async def close(self) -> None: + """依次停止role、等待真实I/O、同loop清理store、停止并join线程。 + + stop_roles/quiesce未退出不进入依赖清理；cleanup之后finally尝试io.close， + io.close若报未退出则保留storage标志，不能提前清状态或继续关业务依赖。 + 各阶段分别使用配置预算，非整个close共享一个总截止。 + """ + await self.stop_roles() + if not self._storage_started: + return + await self._io.quiesce(self._config.shutdown_timeout_seconds) + try: + if not self._storage_closed: + await self._io.run_cleanup(self._store.close, self._config.shutdown_timeout_seconds) + self._storage_closed = True + finally: + await self._io.close(self._config.shutdown_timeout_seconds) + self._storage_started = False + self._state.storage_ready = False + + def _spawn_role(self, name: str, operation: Awaitable[None]) -> None: + """保存 Task 强引用与退出结果；角色异常要可见，不能静默失去调度循环。""" + ... +``` + +## 10. lua/：精简后的全部脚本契约 + +仅保留12个Lua文件（含共享前缀）。不再有源状态初始化、源关闭水位、注销伴随状态或失败归档脚本。所有脚本在写前校验参数与key类型；脚本执行异常不保证回滚。 + +### 10.1 _common.lua + +```lua +local BASE = 1099511627776 +local MAX_INT = 9007199254740991 + +local function decimal(n) + -- 时间和claim score使用精确十进制字符串。 + return string.format('%.0f', n) +end + +local function now_ms() + -- 返回当前执行节点TIME，不能用另一slot节点或客户端时间替代源准入时间。 + local t = redis.call('TIME') + return tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000) +end + +local function integer_arg(value, min_value, max_value) + -- 严格校验规范十进制、整数和范围，不接受NaN/inf；错误发生在任何写入前。 +end + +local function expect_type(key, allowed) + -- TYPE必须为none或指定类型，否则写前报错。 +end + +local function require_owner(key, token) + -- GET必须等于token且PTTL>0，否则owner_lost；不重新创建或续活失效锁。 +end + +local function page_after(key, lower, upper, last_score, last_member, limit) + -- 稳定score/member分页：原游标仍在则rank快路径，否则rank二分定位。 + -- 有界读取limit+1，返回items及next，不使用客户端OFFSET，不扫描全部同score。 +end +``` + +### 10.2 ensure_metadata.lua + +```lua +-- KEYS = M +-- ARGV = item_type, local_id, account_id, user_id, ttl_ms +-- RETURN = 'created' | 'refreshed' +-- ttl_ms=max(2*timeout_seconds*1000,86400000)，由store统一计算。 +-- 先校验HASH/none、全部身份和ttl范围；无schema字段。 +-- 不存在则HSET身份；存在逐字段校验，冲突在任何写入前报错。 +-- 最后PEXPIRE ttl_ms；只有前一步R明确成功后才调用本脚本。 +``` + +### 10.3 upsert_resource.lua + +```lua +-- KEYS = R；ARGV = item_id, requested_due_ms +-- RETURN = decimal(accepted_due_ms) +-- 仅report调用；不存在V，不读C，没有revision/CAS/请求顺序比较。 +expect_type(KEYS[1], 'zset') +local requested = integer_arg(ARGV[2], 0, MAX_INT) +local current_time = now_ms() +local accepted = math.max(requested, current_time + 1) +-- 写前检查item_id非空；向上对齐后的桶分钟必须仍在精确整数范围内。 +local max_supported_due = math.floor(MAX_INT / 60000) * 60000 +if accepted > max_supported_due then return redis.error_reply('invalid_report') end +redis.call('ZADD', KEYS[1], decimal(accepted), ARGV[1]) +return decimal(accepted) +``` + +这里校正的是写入时的未来到期时间，不能识别业务先后顺序。重试报告可能覆盖并发更新；接口不返回DUPLICATE/STALE。R已经成功但M失败时不补偿删除R。 + +### 10.4 source_time.lua + +```lua +-- KEYS = R （用于路由到R所在Redis slot）；无ARGV。 +-- RETURN = decimal(source_now_ms) +-- 本脚本只读，不修改R，也不创建任何伴随key。 +return decimal(now_ms()) +``` + +store读C并使用该时间构造U=floor(T/60000)*60000。U=C或无数据则跳过；不执行“关闭窗口”写入。 + +### 10.5 read_source_page.lua + +```lua +-- KEYS = R +-- ARGV = lower_exclusive, upper_inclusive, after_score_or_empty, after_member, limit +-- RETURN = cjson {items=[[item,score_string],...], next=[score_string,item] | false} +-- 校验后调用page_after；整个范围为空返回空items，不修改C。 +-- scheduler先limit=1定位下一个非空分钟，再分页读完整分钟，避免逐空分钟循环。 +``` + +### 10.6 publish_minute.lua + +```lua +-- KEYS = G, C, scheduler_owner （同槽） +-- ARGV = owner_token, expected_C, minute_ms, member_count, registry_member... +-- RETURN = {'published'|'already_published', decimal(C)} +-- 先验证key类型、C存在、分钟对齐、全部注册member与shard/minute一致。 +-- 空分钟不调用，本脚本要求member_count>=1。 +require_owner(KEYS[3], ARGV[1]) +local expected = integer_arg(ARGV[2], 0, MAX_INT) +local minute = integer_arg(ARGV[3], 0, MAX_INT) +local count = integer_arg(ARGV[4], 1, 1024) +local current = tonumber(redis.call('GET', KEYS[2])) +if current >= minute then + return {'already_published', decimal(current)} +end +if current ~= expected or minute <= current then + return redis.error_reply('progress_conflict') +end +-- 中间空分钟已由scheduler扫描证实；只要求单调，不要求minute=current+60000。 +for i = 1, count do + redis.call('ZADD', KEYS[1], decimal(minute), ARGV[4+i]) +end +redis.call('SET', KEYS[2], decimal(minute)) +return {'published', decimal(minute)} +``` + +owner比较与发布原子，但不能约束其他slot上已经发出的B写入；保留主文档10.3的限制。 + +### 10.7 read_registry_page.lua + +```lua +-- KEYS = G；ARGV = after_minute_or_empty, after_member, limit +-- RETURN = cjson {items=[[registry_member,minute_string],...], next=[minute,member] | false} +-- 使用page_after；G只包含发布完成的桶，无需额外用客户端minute=max_attempts：仅加入exhausted，保持原score，不删除、不新增Redis记录。 +-- score=0新项优先，但至少为已有可重试项保留一个槽；claims总数<=limit。 +-- consumer对exhausted先日志再ack.lua，不调用handler；总检查/写入<=2*limit。 + +local function next_score(old_score, now_seconds, max_attempts) + -- now_seconds在外部通过范围检查。 + if old_score == 0 then return now_seconds end + local attempt = math.floor(old_score / BASE) + if attempt + 1 >= max_attempts then error('attempts_exhausted') end + local next_time = math.max(now_seconds, old_score % BASE + 1) + if next_time >= BASE then error('invalid_claim_time') end + return (attempt + 1) * BASE + next_time +end +``` + +### 10.9 ack.lua + +```lua +-- KEYS = B；ARGV = item_id, exact_claim_score；RETURN = 0 | 1。 +-- 成功回调ACK和耗尽日志后的清理共用；不删除R/M。 +local expected = integer_arg(ARGV[2], 1, 256 * BASE - 1) +local current = redis.call('ZSCORE', KEYS[1], ARGV[1]) +if current and tonumber(current) == expected then + return redis.call('ZREM', KEYS[1], ARGV[1]) +end +return 0 +``` + +### 10.10 acquire_owner.lua + +```lua +-- KEYS = owner；ARGV = token, ttl_ms（本期120000） +-- RETURN = {'owned', expires_at_ms} | {'busy', 0} +-- SET NX PX；同token只确认本次请求已取得锁，不因此延长lease。 +-- 新token未取得则busy；返回TIME/PTTL供观测，本地使用请求起点+TTL保守截止。 +``` + +### 10.11 renew_owner.lua + +```lua +-- KEYS = owner；ARGV = token, ttl_ms（120000）；RETURN = expires_at_ms | 0。 +-- GET==token且PTTL>0才PEXPIRE；不创建缺失锁，不延长其他token的锁。 +-- 任意不匹配/异常/超时在调用方立即使当前lease失效，不等到下一个20秒周期。 +``` + +### 10.12 release_owner.lua + +```lua +-- KEYS = owner；ARGV = token；RETURN = 0 | 1。 +-- 本地续租task已经停止并await后调用，防止释放与自己的续租交错。 +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +``` + +## 11. session_expiry.py：业务适配，不把 policy 塞进调度层 + +```python +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING +from openviking.server.identity import RequestContext, Role +from openviking.session.auto_commit_policy import AutoCommitPolicy +from openviking.session.session import SessionExpirySnapshot +from openviking.scheduling.expiry import ( + CallbackRequest, CallbackResult, CallbackStatus, HandlerOptions, ItemIdentity, + ExpiryRegistration, ExpiryScheduler, ExpiryMode, +) +from openviking_cli.session.user_id import UserIdentifier + +if TYPE_CHECKING: + from openviking.service.session_service import SessionService + + +def install_session_expiry( + scheduler: ExpiryScheduler, sessions: SessionService, *, concurrency: int = 8 +) -> None: + """业务模块自己的启动注册函数；core 只调用业务初始化，不认识处理器细节。""" + scheduler.register_handler("session", SessionTimeoutHandler(sessions, scheduler), + options=HandlerOptions(concurrency=concurrency, max_attempts=3)) + sessions.set_session_expiry_adapter(SessionExpiryAdapter(scheduler)) + + +def effective_idle_timeout(policy: dict | None) -> int | None: + """None 表示禁用；不能将 None 传给 from_dict 后意外启用默认 policy。""" + if policy is None: + return None + timeout = AutoCommitPolicy.from_dict(policy).idle_timeout_seconds + return timeout if timeout > 0 else None + + +def timestamp_ms(value: str) -> int: + """沿用内核时间格式解析规则，统一 UTC 毫秒；错误显式上报。""" + ... + + +class SessionExpiryAdapter: + """实现本业务的 SessionExpiryReporter；承接已成功持久化的消息快照。""" + + def __init__(self, scheduler: ExpiryScheduler) -> None: + """只依赖到期调度协议；不依赖 store、runtime 或框架内部指标类型。""" + self._scheduler = scheduler + + def build_report(self, snapshot: SessionExpirySnapshot) -> ExpiryRegistration | None: + """在业务层计算 due；禁用时返回 None，不回改已有历史 score。""" + timeout = effective_idle_timeout(snapshot.auto_commit_policy) + if timeout is None: + return None + return ExpiryRegistration( + identity=ItemIdentity("session", snapshot.account_id, + snapshot.user_id, snapshot.session_id), + due_at_ms=timestamp_ms(snapshot.last_message_at) + timeout * 1000, + timeout_seconds=timeout, + mode=ExpiryMode.IDLE_ONESHOT, + ) + + async def report_after_append(self, snapshot: SessionExpirySnapshot) -> None: + """消息已落盘，报告最终失败记录指标，不伪装消息回滚。 + + 立即 await，没有内存去抖；CancelledError 不被 except Exception 吞掉。 + """ + try: + report = self.build_report(snapshot) + if report is not None: + await self._scheduler.report(report) + except Exception as exc: + self.record_report_failure("report_failed", exc) + + async def unregister_after_delete(self, identity: ItemIdentity) -> None: + """业务删除已成功后尽力注销；失败记录日志/指标，不反向恢复业务数据。""" + ... + + def record_report_failure(self, reason: str, exc: Exception | None = None) -> None: + """使用业务自身的日志/指标记录无法报告计时；不接触框架内部队列或指标对象。""" + ... + + +class SessionTimeoutHandler: + def __init__(self, sessions: SessionService, scheduler: ExpiryScheduler) -> None: + """持有业务服务和公开scheduler协议；不持有Redis实现，冷却重投也走report。""" + self._sessions, self._scheduler = sessions, scheduler + + async def handle(self, request: CallbackRequest) -> CallbackResult: + """只按业务身份构造 ctx 并提交，不感知元数据缓存、调度桶或消费凭据。""" + ctx = self._make_context(request.identity) + # SessionService.commit_on_timeout 自行把明确业务 NotFound 转 skipped。 + # 存储/网络异常继续抛给 consumer，保留 Redis 成员。 + result = await self._sessions.commit_on_timeout(request.identity.local_id, ctx) + if result.get("status") == "deferred": + await self._report_deferred(request.identity, result) + return CallbackResult(CallbackStatus.COMPLETED, "idle_rescheduled") + return self._to_callback_result(result) + + async def _report_deferred(self, identity: ItemIdentity, result: dict) -> None: + """使用服务层纯检查生成的新due报告，不持有Session提交锁，不取旧score或旧timeout。 + + 必须await R→M成功后才返回。异常传播为旧事件RETRY，不吞错假装已重投。 + 只接受 idle_cooldown 和 idle_activity；disabled/无消息不生成deferred； + 重投不更新last_idle_commit_accepted_at。 + """ + if result.get("reason") not in {"idle_cooldown", "idle_activity"}: + raise ValueError("Unexpected deferred reason") + await self._scheduler.report(ExpiryRegistration( + identity=identity, due_at_ms=result["due_at_ms"], + timeout_seconds=result["timeout_seconds"], mode=ExpiryMode.IDLE_ONESHOT, + )) + + def _make_context(self, identity: ItemIdentity) -> RequestContext: + """使用资源自身 account/user，沿用业务用户权限，不提升到 root。""" + return RequestContext( + user=UserIdentifier(account_id=identity.account_id, user_id=identity.user_id), + role=Role.USER, + ) + + def _to_callback_result(self, result: dict) -> CallbackResult: + """accepted 或明确 skipped -> COMPLETED；未知返回结构 -> RETRY/抛协议错误。 + + 不把HTTP 200或任意truthy字典当作accepted；原commit API返回accepted即成功，不等后台任务。 + """ + status = result.get("status") + if status == "accepted": + return CallbackResult(CallbackStatus.COMPLETED, "accepted") + if status == "skipped" and result.get("reason") in { + "not_found", "idle_disabled", "no_messages", + }: + return CallbackResult(CallbackStatus.COMPLETED, result["reason"]) + return CallbackResult(CallbackStatus.RETRY, "unexpected_commit_result") +``` + +`effective_idle_timeout` 只用于上报适配器；超时判断由SessionService复用自身的AutoCommitPolicy与时间工具完成。Session提交内部不做idle判断、不掌握deferred或Redis，也不反向import此service文件。 + +## 12. session.py：新增时间字段与追加 reporter，保留原通用提交 + +### 12.1 新增字段和内部类型 + +```python +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + + +@dataclass(frozen=True) +class SessionExpirySnapshot: + account_id: str + user_id: str + session_id: str + last_message_at: str + auto_commit_policy: dict[str, Any] | None + # policy 必须深拷贝；后续 Session 对象更新不得改变这次报告。 + + +class SessionExpiryReporter(Protocol): + async def report_after_append(self, snapshot: SessionExpirySnapshot) -> None: + """持久化成功、释放 session lock 后立即等待；reporter 自己处理上报失败策略。""" + ... + + def record_report_failure(self, reason: str, exc: Exception | None = None) -> None: + """消息仍然成功，但无法构造可信计时快照时调用。""" + ... +``` + +在现有 `SessionMeta` 追加字段，不新建第二份 metadata 文件： + +```python +@dataclass +class SessionMeta: + # 现有全部字段原样保留；这里只展示新增字段和要修改的方法。 + last_idle_commit_accepted_at: str = "" # 服务层best-effort记录，不是ready原子记录或修复缓存。 + + def to_dict(self) -> dict[str, Any]: + """只加入接受时间字段，保留现有全部字段与输出兼容性，不新增idle归档指针。""" + ... + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> SessionMeta: + """缺字段使用空串；时间解析沿用内核工具，不从归档查证或恢复此字段。""" + ... +``` + +### 12.2 Session 新增/修改的完整方法目录 + +以下 class 是现有类的**增量视图**，不是替换整个类。构造函数保留原参数，仅在末尾加可选 reporter： + +```python +class Session: + def __init__( + self, viking_fs: "VikingFS", vikingdb_manager: "VikingDBManager | None" = None, + session_compressor: "SessionCompressor | None" = None, + user: "UserIdentifier | None" = None, ctx: "RequestContext | None" = None, + session_id: str | None = None, session_uri: str | None = None, + auto_commit_threshold: int = 8000, + tool_output_externalization_config: "ToolOutputExternalizationConfig | None" = None, + agent_evolution_enabled: bool = True, usage_reporter: "UsageReporter | None" = None, + agent_evolution_enabled_provider: "Callable[[], bool | Awaitable[bool]] | None" = None, + memory_policy_provider: "MemoryPolicyProvider | None" = None, + vlm_resolver: "VLMResolver | None" = None, + expiry_reporter: SessionExpiryReporter | None = None, + ) -> None: + """原初始化保留，仅增加 self._expiry_reporter = expiry_reporter。""" + ... + + async def _append_messages_authoritatively(self, messages: "list[Message]") -> None: + """修改现有方法：锁内追加并保存业务meta，锁外立即 await reporter。 + + 原写消息失败时不报告；无 VikingFS 的原内存分支不宣称持久化成功。 + 元信息读取失败时维持既有消息处理规则，但不伪造可信的policy/时间快照。 + """ + ... + + def _build_expiry_snapshot_locked(self) -> SessionExpirySnapshot: + """新增：只在最新消息/meta 写入成功后调用；深拷贝 policy 和身份。 + + 仅深拷贝当前时间、policy和身份，不分配资源更新版本。 + """ + ... + + # commit/commit_async及原通用提交内部实现保持不变，不为timer新增或拆分提交函数。 + # 服务层直接调用原commit_async；Session内部没有idle判断、deferred、Redis调用。 + # 原返回结构、参数校验、锁、ready、QueueFS及恢复能力继续存在。 + # 各既有meta更新路径须保留接受时间字段，但不生成或恢复该字段。 +``` + +### 12.3 追加函数内部放置位置 + +下面是已有 `_append_messages_authoritatively()` 中的插入结构，省略的行保留原写消息逻辑，不是另新增一个消息入口： + +```python +# 已持有 session path lock，且最新 meta 读取状态已知。 +# 原有：reload -> _apply_appended_messages_to_state -> append/write messages.jsonl。 +snapshot = None +snapshot_error = None +metadata_reload_error = None +await self._save_meta() +if self._expiry_reporter is not None: + try: + # save成功后，self._meta已是本次路径锁内刚持久化的权威状态。 + snapshot = self._build_expiry_snapshot_locked() + except Exception as exc: + snapshot_error = exc +# 原 finally 释放 session path lock。 +# 以下放到 try/finally 外，只在持久化成功后执行： +if self._expiry_reporter is not None: + if snapshot is not None: + await self._expiry_reporter.report_after_append(snapshot) + else: + self._expiry_reporter.record_report_failure("snapshot_build_failed", snapshot_error) +``` + +实现时处理“消息文件已写成，但 meta 写失败”的既有业务结果，不额外声称它们是事务。追加前 meta 重读失败不再单独阻止 report：只要随后 `_save_meta()` 成功，就必须用刚持久化的状态构造 snapshot；只有 snapshot 本身无法构造时才不 report。`update_config()` 已在 session lock 内重读 meta，序列化时保留新增的接受时间字段；`update_event_search_tags()` 继续委托它，无需新建计时更新逻辑。 + +### 12.4 原通用 commit、QueueFS 与 Working Memory 契约 + +以下是保留清单，不新增一套提交实现： + +| 位置/函数 | 必须保留的原行为 | +|---|---| +| Session.commit / commit_async | 原公开签名、参数校验、锁和返回结构不变；enable_working_memory的None/True/False完整透传 | +| 原通用ready与归档恢复 | 原能力和兼容协议不删除、不添加idle标记、接受时间或额外严格判断 | +| MemoryPolicy.from_dict / to_dict | enable_working_memory缺省False；序列化保留working_memory.enabled的真实布尔值 | +| SessionCommitMsg.to_dict / from_dict | 新消息memory_policy_version=1；旧队列沿用既有WM默认值兼容，显式False不覆盖 | +| Session.resume_queued_commit / _run_memory_extraction | 使用队列策略及原后台任务流程，不因timer重新解析当前用户策略 | +| Session._write_done_file | enable_working_memory参数和同名.done字段继续保留 | +| ArchiveStore.normalize_done / read_done / scan_states | 保留旧working_memory_enabled兼容；WM关闭且无摘要仍可正常完成 | + +`memory_policy_version` 和原归档格式版本是既有持久业务协议，与Redis M.schema_version不同，不删除、不复制到R/M/B。无需因本方案修改memory_policy.py、archive_store.py或session_commit_msg.py。 + +idle服务层调用不覆盖WM，原始归档、QueueFS及符合policy的长期记忆提取继续存在。原commit的accepted及skipped字段（包括`effective_enable_working_memory`）原样返回；deferred只在SessionService/外层handler之间使用，不扩展Session公开commit返回语义。timer以原API返回accepted为成功，不等后台任务，不用ready恢复idle接受记录。 + +## 13. session_service.py：外层纯检查、原提交与 best-effort 接受记录 + +以下是服务层增量骨架；使用现有factory/get、路径锁、身份和时间工具，不将检查迁入Session。`get(..., auto_create=False)`必须加载最新session/meta，明确不存在才转skipped，其他读取错误不能退回旧内存状态。 + +```python +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional +from openviking.core.namespace import canonical_session_uri +from openviking.server.identity import RequestContext +from openviking.session import Session +from openviking.session.archive_store import is_storage_not_found +from openviking.session.auto_commit_policy import AutoCommitPolicy +from openviking.utils.time_utils import get_current_timestamp, parse_iso_datetime +from openviking.scheduling.expiry import ExpiryScheduler + +if TYPE_CHECKING: + from .session_expiry import SessionExpiryAdapter + +# 复用现有logger与meta锁超时配置，不新增另一套时间解析或锁策略。 +logger = logging.getLogger(__name__) +_SESSION_META_LOCK_TIMEOUT_SECONDS = 30.0 + + +class SessionService: + # __init__ 原签名不变，初始化 self._session_expiry_adapter = None。 + + def configure_expiry(self, scheduler: "ExpiryScheduler") -> None: + """业务初始化入口：由 session 模块注册自己的 handler 和适配器。""" + from .session_expiry import install_session_expiry + install_session_expiry(scheduler, self) + + def set_session_expiry_adapter(self, adapter: "SessionExpiryAdapter | None") -> None: + """新增：初始化阶段注入；关闭时摘除，不扫描已经创建的 Session 对象。""" + self._session_expiry_adapter = adapter + + def session(self, ctx: RequestContext, session_id: str | None = None, *, + session_uri: str | None = None) -> Session: + """原 factory 参数/行为保留；构造 Session 时增加 expiry_reporter=adapter。 + + 原 memory_policy_provider、usage_reporter、vlm_resolver 等参数全部继续传递。 + """ + ... + + async def commit_on_timeout(self, session_id: str, ctx: RequestContext) -> dict[str, Any]: + """加载最新状态、检查一次，直接复用原commit；不自动创建session。 + + 不在Session提交锁内最终复检idle，期间追加的新消息允许一起提交。 + 原API返回accepted即成功，不等待后台任务；返回结构原样保留。 + 入口加载/检查/commit异常传播，只有明确NotFound转skipped。 + """ + try: + session = await self.get(session_id, ctx, auto_create=False) + except Exception as exc: + if not is_storage_not_found(exc): + raise + return {"status": "skipped", "reason": "not_found"} + decision = self._check_timeout_commit(session, datetime.now(timezone.utc)) + if decision is not None: + return decision # skipped/deferred不记录接受时间，不调用commit。 + result = await session.commit_async( + keep_recent_count=0, + persist_keep_recent_count=False, + record_auto_commit_success=True, + ) + if result.get("status") == "accepted": + await self._record_idle_commit_time(session_id, ctx, get_current_timestamp()) + # 原生命周期/归档指标保留；不把记录失败改成业务RETRY。 + return result + + @staticmethod + def _check_timeout_commit(session: Session, now: datetime) -> Optional[dict]: + """纯检查：policy/timeout -> no_messages -> cooldown -> last_message_due。 + + 只读最新加载状态，不取提交锁，不读归档，不写meta/Redis。 + None表示直接调用原commit；deferred只由服务层/外层handler掌握。 + 时间解析使用parse_iso_datetime；缺失last_message_at或解析异常向外抛。 + """ + policy_data = session.meta.auto_commit_policy + if policy_data is None: + return {"status": "skipped", "reason": "idle_disabled"} + timeout = AutoCommitPolicy.from_dict(policy_data).idle_timeout_seconds + if timeout <= 0: + return {"status": "skipped", "reason": "idle_disabled"} + if not session.messages: + return {"status": "skipped", "reason": "no_messages"} + if now.tzinfo is None: + now = now.replace(tzinfo=timezone.utc) + now_ms = int(now.timestamp() * 1000) + accepted_at = session.meta.last_idle_commit_accepted_at + if accepted_at: + accepted = parse_iso_datetime(accepted_at) + if accepted.tzinfo is None: + accepted = accepted.replace(tzinfo=timezone.utc) + if (now - accepted).total_seconds() < timeout: + return { + "status": "deferred", "reason": "idle_cooldown", + "timeout_seconds": timeout, "due_at_ms": now_ms + timeout * 1000, + } + if not session.meta.last_message_at: + raise RuntimeError("Session last_message_at is missing") + last_message = parse_iso_datetime(session.meta.last_message_at) + if last_message.tzinfo is None: + last_message = last_message.replace(tzinfo=timezone.utc) + activity_due_ms = int(last_message.timestamp() * 1000) + timeout * 1000 + if now_ms < activity_due_ms: + return { + "status": "deferred", "reason": "idle_activity", + "timeout_seconds": timeout, "due_at_ms": activity_due_ms, + } + return None + + async def _record_idle_commit_time( + self, session_id: str, ctx: RequestContext, accepted_at: str, + ) -> None: + """accepted之后的best-effort记录，失败日志后仍返回accepted。 + + 服务层取session root exact路径锁，重读最新raw meta字典，只patch两字段； + 保留并发append/config更新及未知字段，不调用旧Session._save_meta。 + 新旧有效接受时间只取max，旧时间不回写；不读idle归档或修复缓存。 + meta不存在时跳过，不创建目录/文件，不重建已删除session。 + helper内部读取/锁/写失败只记录日志；取消沿用既有异常语义。 + """ + try: + self._ensure_initialized() + fs = self._viking_fs + session_uri = canonical_session_uri(ctx, session_id) + path = fs._uri_to_path(session_uri, ctx=ctx) + lease = await fs._async_agfs.pathlock_acquire_exact( + path, timeout_secs=_SESSION_META_LOCK_TIMEOUT_SECONDS, + ) + try: + # 在与既有delete协作的session锁内直接读取；缺失不能以旧meta补建。 + metadata = json.loads(await fs.read_file(f"{session_uri}/.meta.json", ctx=ctx)) + if not isinstance(metadata, dict): + raise ValueError("Session metadata must be a JSON object") + current = parse_iso_datetime(accepted_at) + if current.tzinfo is None: + current = current.replace(tzinfo=timezone.utc) + previous = metadata.get("last_idle_commit_accepted_at") + if previous: + try: + previous_dt = parse_iso_datetime(previous) + if previous_dt.tzinfo is None: + previous_dt = previous_dt.replace(tzinfo=timezone.utc) + except (TypeError, ValueError): + logger.warning("Replacing invalid idle commit time for %s", session_id) + else: + if previous_dt >= current: + return # max合并：已有时间更晚/相同，不写旧值。 + metadata["last_idle_commit_accepted_at"] = accepted_at + metadata["updated_at"] = get_current_timestamp() + await fs.write_file( + f"{session_uri}/.meta.json", json.dumps(metadata, ensure_ascii=False), + ctx=ctx, lease_ref=lease, + ) + finally: + await fs._async_agfs.pathlock_release(lease) + except Exception: + logger.warning("Failed to record idle commit time for %s", session_id, exc_info=True) + + async def delete(self, session_id: str, ctx: RequestContext) -> bool: + """原业务删除成功后，await adapter.unregister_after_delete(identity)。 + + identity 取 ctx 的 account/user 和传入 session_id；不依赖可能已过期的 M。 + 原删除失败则不注销；不搜寻已投递 B；注销失败由适配器记录。 + """ + ... +``` + +`create/get/update_config/commit/commit_async/maybe_schedule_auto_commit/run_auto_commit` 的最新外部签名保留。SessionService.commit/commit_async已有的 `enable_working_memory: Optional[bool] = None` 不删除；继续以 `is not None` 筛选后传给Session，False必须保留。HTTP CommitRequest的StrictBool校验和router透传不变，新增idle handler不提供一次性WM覆盖。只有 factory 注入影响它们返回的 Session；Redis idle 回调不依赖旧自动提交的 has_running 扫描。 + +## 14. 配置文件：字段、校验和兼容入口 + +### 14.1 scheduling/expiry/config.py：轻量配置定义（新增拆分） + +实际读取发现 `openviking_cli.utils.config.__init__` 会聚合导入总配置、模型和解析器配置。配置类定义放入这个轻量文件，CLI 配置目录只保留重导出入口。store/publisher/consumer/runtime 均从 `.config` 导入；内部 publisher 启动模块仅在校验完整 `ov.conf` 时加载聚合配置。 + +```python +from __future__ import annotations + +import os +from typing import Literal +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class ExpiryRedisConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + mode: Literal["standalone", "cluster"] = "standalone" + endpoints: list[str] = Field(default_factory=list) + username: str = "" + password_env: str = "" + password: str = Field(default="", repr=False) + tls: bool = False + connect_timeout_seconds: float = Field(default=1.0, gt=0) + socket_timeout_seconds: float = Field(default=2.0, gt=0) + request_budget_seconds: float = Field(default=5.0, gt=0) + max_connections: int = Field(default=64, ge=1) + max_inflight_requests: int = Field(default=128, ge=1) + + @model_validator(mode="after") + def validate_connection(self) -> ExpiryRedisConfig: + """校验 endpoint 格式、独立凭证字段和 standalone 地址数量。 + + endpoints 为空允许 Redis idle backend 未启用；启用时由聚合配置拒绝。 + endpoint 不允许携带凭证、path、query 或 fragment。 + 不尝试网络连接，不读密码值，不在配置对象构建时启动客户端。 + """ + ... + + def resolve_password(self) -> str | None: + """优先按 password_env 读取；变量不存在时失败，不静默降级为无密码。""" + ... + + +class ExpiryConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + service_name: str = "" + publisher_separate_process: bool = True + shard_count: int = Field(default=128, ge=1, le=512) + session_consumer_concurrency: int = Field(default=8, ge=1) + precision_seconds: Literal[60] = 60 + owner_ttl_seconds: Literal[120] = 120 + owner_renew_interval_seconds: Literal[20] = 20 + scan_batch_size: int = Field(default=128, ge=1, le=4096) + cleanup_batch_size: int = Field(default=128, ge=1, le=4096) + registry_page_size: int = Field(default=128, ge=1, le=4096) + processing_timeout_seconds: int = Field(default=120, ge=1) + callback_poll_seconds: float = Field(default=0.5, gt=0) + shutdown_timeout_seconds: float = Field(default=30.0, gt=0) + redis: ExpiryRedisConfig = Field(default_factory=ExpiryRedisConfig) + + def validate_for_runtime(self) -> None: + """Redis idle backend 启用时要求 service_name/endpoints 并校验预算。 + + publisher 与 consumer 自动遍历0..N-1，每次持有一个shard；无静态候选清单。 + TTL固定120秒、续租20秒；任务处理超时独立配置。 + """ + ... +``` + +TTL 120秒和20秒续租来自用户约定，其余数值为压测起点。持续积压会长期持有当前shard，不承诺按资源量或实例严格均衡。 + +### 14.2 openviking_cli/utils/config/expiry_config.py + +```python +# 兼容内核配置目录的入口；类型定义只保留一份，避免默认值/校验分叉。 +from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig + +__all__ = ["ExpiryConfig", "ExpiryRedisConfig"] +``` + +### 14.3 memory_config.py + +```python +class SessionAutoCommitConfig(BaseModel): + # 原 enabled/check_interval_seconds/scan_rate_limit_files_per_second 保留。 + idle_backend: Literal["legacy", "redis"] = "legacy" + # enabled 的 description 同步修改：默认 policy 开关和所选 idle backend 的启用条件。 + # 不再写成一定启动 AGFS 扫描器。没有新增方法。 +``` + +### 14.4 open_viking_config.py + +```python +class OpenVikingConfig(BaseModel): + # 原全部字段保留。 + expiry: ExpiryConfig = Field(default_factory=ExpiryConfig) + + @model_validator(mode="after") + def _validate_expiry_backend(self) -> OpenVikingConfig: + """Redis idle backend 启用时校验完整 expiry 运行配置。""" + idle = self.memory.session_auto_commit + if idle.enabled and idle.idle_backend == "redis": + self.expiry.validate_for_runtime() + return self + + @classmethod + def from_dict(cls, config: dict[str, Any]) -> OpenVikingConfig: + """原加载逻辑保留，新增字段走 Pydantic 嵌套解析。 + + 现有实现先构造 instance，再单独设置 memory：因此 memory 应用完成后， + 必须再次显式调用 instance._check_expiry_backend()，不能只依赖首次构造的 validator。 + """ + ... + + def to_dict(self) -> dict[str, Any]: + """原 model_dump 已自动包含 expiry；保留签名，增加往返检查。""" + ... +``` + +既有配置仍默认 legacy。选择 Redis backend 后，report、publisher、consumer 作为一个完整能力启用，不再提供可独立组合的用户角色开关。 + +## 15. core.py：初始化位置必须放对 + +```python +class OpenVikingService: + # __init__ 原参数保留；初始化 _expiry_runtime = None。 + + async def initialize(self) -> None: + """在原初始化流程中插入： + + 1. SessionService 依赖/policy 已设置后，判断是否启用 Redis expiry。 + 2. 根据 idle_backend 选择是否启动原 SessionAutoCommitScheduler。 + 3. 原 QueueFS handler 注册、TaskTracker 恢复、queue_manager.start()。 + 4. Redis backend 启用时创建 API runtime，注册业务后统一 start_storage/start_roles。 + 5. 再完成其余初始化并对外提供服务。 + + publisher_separate_process=false 时 API runtime 额外运行 publisher。 + 中途失败须清理已建 expiry storage；不能留下空转 callback。 + """ + ... + + async def _initialize_expiry(self) -> None: + """启动组装层先调用所有已启用业务的注册入口，再统一启动 runtime。 + + 仅在 session 选择启用 Redis idle backend 时，调用 + self._session_service.configure_expiry(runtime.scheduler)。 + API runtime 固定使用 ExpiryRuntimeRoles(report=True, consume=True)， + publisher 是否同进程由 publisher_separate_process 决定。 + timer 为其他业务启用时不能顺带给 legacy session 安装 Redis 上报适配器。 + 新业务由其自己的初始化入口完成注册，core 不 import 各业务 handler 类。 + Redis idle backend 未启用时不 import redis，不创建线程。 + """ + ... + + async def _start_expiry_roles(self) -> None: + """所有业务注册完成后依次 start_storage/start_roles；无 runtime 则直接返回。""" + ... + + async def _stop_expiry(self) -> None: + """保持runtime强引用，await runtime.close；成功后才清runtime并摘除adapter。 + + runtime.close抛shard_not_stopped时原样向外传播，保留runtime和adapter， + 禁止新calls由runtime.accepting_calls=False负责，不能提前detach造成状态不一致。 + 不能继续关闭QueueFS/AGFS或其他未退出工作依赖；允许后续重试关闭/观察。 + """ + runtime = self._expiry_runtime + if runtime is None: + return + await runtime.close() + self._expiry_runtime = None + self._session_service.set_session_expiry_adapter(None) + + async def close(self) -> None: + """在QueueManager.stop之前await _stop_expiry；未退出错误传播并中止依赖关闭。 + + 其余startup/optional backend和既有无timer关闭路径保持不变。 + """ + ... +``` + +旧 scanner 创建条件具体改为： + +```python +use_legacy_idle = ( + session_auto_commit_config.enabled + and session_auto_commit_config.idle_backend == "legacy" +) +# 仅 use_legacy_idle 为 True 时走原 SessionAutoCommitScheduler 构造/start 分支。 +# enabled 的 policy 默认值逻辑继续传给 SessionService，不因为 Redis backend 被关闭。 +``` + +## 16. expiry_bootstrap.py：内部 publisher 子进程 + +```python +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import signal +from pathlib import Path +from typing import Any + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """解析父进程传入的配置路径与 readiness 状态文件。""" + parser = argparse.ArgumentParser(description="Internal OpenViking expiry publisher") + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--startup-status", required=True, type=Path) + return parser.parse_args(argv) + + +def load_expiry_config(path: Path) -> "ExpiryConfig": + """读取完整 ov.conf，确认 Redis backend 和独立 publisher 均已启用。""" + from openviking_cli.utils.config.open_viking_config import OpenVikingConfig + raw = json.loads(os.path.expandvars(path.expanduser().read_text(encoding="utf-8-sig"))) + config = OpenVikingConfig.from_dict(raw) + ... + return config.expiry + + +def install_signal_handlers(stop: asyncio.Event) -> None: + """在主线程 loop 上把 SIGINT/SIGTERM 转为 stop.set，信号处理器不做 I/O。""" + ... + + +async def run_publisher(config: "ExpiryConfig", stop: asyncio.Event, + startup_status: Path) -> None: + """创建 runtime，启动 storage/roles，同时等待 stop 或角色故障。 + + 任一角色异常立即结束命令并返回失败；finally await runtime.close。 + storage/role 启动成功后写 ready；不能仅 await stop.wait 而让 publisher + task 异常后进程继续假存活。 + """ + ... + + +def main(argv: list[str] | None = None) -> int: + """解析/配置失败返回非零；正常信号退出走有界关闭；不打印密码值。""" + ... +``` + +该模块不注册公开 CLI。`openviking-server` 通过 `python -m` 拉起它，readiness 失败则拒绝启动 API；运行期异常退出会触发父进程退出。publisher 自动遍历全部 shard 逐个抢锁，只有成功取得 lease 才处理；不依赖 Pod ID 或静态分片列表。 + +## 17. pyproject.toml 与包资源 + +这里只列需要合并进现有 TOML table 的字段，不重复创建已有 table： + +```toml +[project.optional-dependencies] +# 新增到原 table；redis 版本是待验证的候选范围。 +expiry = ["redis>=5,<6"] + +[tool.setuptools.package-data] +# 保留原全部 package-data；增加独立 package 的 Lua 数据。 +"openviking.scheduling.expiry" = ["lua/*.lua"] +``` + +新增 Python 文件沿用仓库许可证头。`lua/` 仅保存数据文件，无需 Python `__init__.py`；`scheduling/expiry/__init__.py` 要存在。验证构建后的 wheel 包含全部 Lua；publisher 只作为 server 内部模块运行。 + +## 18. 启动与四条运行链路 + +```text +启动 +core汇总已启用Expiry业务 -> 无业务则跳过 + -> 创建runtime -> SessionService.configure_expiry(runtime.scheduler) + -> session及其他业务逐个注册handler/adapter + -> start_storage -> namespace实际配置/C初始0 + -> start_roles -> 冻结注册/开放report + +报告 +Session._append_messages_authoritatively + -> 锁内消息/meta保存，构造无revision快照 -> 释放锁 + -> SessionExpiryAdapter.report_after_append -> ExpiryScheduler.report + -> upsert_resource.lua(R)：TIME + 未来due + ZADD + -> ensure_metadata.lua(M)：身份 + max(2*timeout,24小时) TTL + -> 两步成功返回；R成功M失败报错但保留R + +逐shard调度 +ExpiryPublisher.run：0..N-1递增遍历，本轮后等待下一分钟边界 + -> ShardRunner.run_if_acquired(SCHEDULER,shard_id,process,stop) + -> Coordinator.try_acquire取得120秒lease，runner启动每20秒heartbeat + -> ExpiryPublisher.process_shard：读C/源TIME，U截断至分钟 + -> U=C/无数据跳过；有数据定位下一非空分钟M + -> read_source_page(R) -> stage_items(B)，读完整分钟 + -> publish_minute.lua(G,C,owner)，C=M -> cleanup_source(R) + -> 持锁追平全部到期积压 -> revoke/停heartbeat/有界join -> 可信终止时token释放 -> 更大shard + -> 续租失败：lost/revoke/cancel process与children，有界join；pending阻止同role再抢 + -> 旧tasks真实退出后，下次新token/新execution从C重新开始 + +回调 +ExpiryConsumer.run：全部shard递增遍历，轮后callback_poll_seconds + -> ShardRunner.run_if_acquired(CALLBACK,shard_id,process,stop) -> process_shard + -> reset state/retry缓存（旧tasks必须已退出），ShardExecution.wait执行poll等待 + -> 分页读G -> claim.lua(B,max_attempts=3) + -> claims：execution.spawn工厂创建runner跟踪子task，解析完整身份、读M（可缺失） -> SessionTimeoutHandler + -> SessionService.commit_on_timeout -> get(auto_create=False)加载最新session/meta + -> _check_timeout_commit：外层policy/timeout/no_messages/cooldown/last_message_due纯检查 + -> 检查通过：直接原Session.commit_async(keep_recent_count=0, + persist_keep_recent_count=False, record_auto_commit_success=True) + -> 不做持锁最终idle检查，期间新消息允许一起提交 + -> 原API返回accepted -> _record_idle_commit_time：服务层锁内重读最新meta + -> 仅patch接受时间(max前后)/updated_at；失败日志仍accepted，删除不重建 + -> handler返回后ensure_active先检查current_task.cancelling；吞取消也同步unknown/CancelledError，不ACK + -> 正常返回原accepted且检查通过 -> ack.lua(B) -> 空B摘G；不删除未来R/M + -> 未耗尽失败：留B；最终失败/claim.exhausted：日志 -> 条件ACK + -> 当前shard积压/在途清空后释放；仅旧tasks全部退出后清共享state，重试等待期间继续续租 + +关闭（四条业务链路之外的生命周期） +core保持runtime强引用 -> runtime.stop_roles拒绝新调用/set stop + -> runner grace期间heartbeat继续，仍监视lost/截止；失锁立即revoke/cancel + -> grace超时cancel，有界join role tasks/runner pending + -> 未退出：shard_not_stopped向外传播，保留runtime/tasks/pending，不关闭QueueFS/AGFS/store/I/O + -> role/pending退出 -> io.quiesce拒绝外部请求，等真实_RedisRequest.finished + -> quiesce超时：保留requests和依赖，shard_not_stopped；run多次取消仍桥接至实际退出 + -> 未关闭store：run_cleanup(store.close,timeout)提交Redis loop raw parent，nested io.run关client + -> cleanup成功置_storage_closed=True -> io.close再次quiesce -> 一次loop.stop -> 剩余budget join + -> 线程alive：保留loop/thread及_storage_started/storage_ready，重试跳过成功cleanup、不重复stop + -> 线程真实退出：清I/O引用和storage就绪标志；core成功后才清runtime/adapter并关业务依赖 + +冷却/活动重投 +SessionService外层纯检查：冷却未满足或最新last_message_due未到 + -> deferred：冷却due=检查now+timeout，活动due=last_message_at+timeout + -> SessionTimeoutHandler._report_deferred，不进入Session提交内部 + -> 统一report，R成功再M成功 -> COMPLETED -> ACK旧B，不删除未来R/M + -> report失败：RETRY，保留旧B；skipped/deferred不更新时间 +``` + +## 19. 检验范围与证据边界 + +静态检查覆盖 Markdown 围栏、Python 骨架语法、runtime/shard_runner 相关签名与两文档控制流一致性；省略体和模拟任务不能作为真实 Redis 行为证据。 + +生命周期回归须区分以下边界： + +- stop grace 期间仍监视失锁/截止；迟到 renew 不复活，普通可信业务异常可释放，forced/失锁/unknown 留 TTL。 +- handler 吞取消返回 COMPLETED 时，ensure_active 同步检查 cancelling、置 unknown、抛 CancelledError，阻止 ACK，不等待 done callback。 +- Redis operation 抑制取消及调用方多次取消时，wrapper/requests 保留到真实 finished；quiesce 拒绝外部新请求，超时保留依赖。 +- run_cleanup 的 raw parent 不先占 gate，nested io.run 在 quiesce 后可关 client；timeout/cancel 不冒充 cleanup 已退出。 +- io.close quiesce 后只发一次 stop，以剩余预算 join；线程 alive 保留引用、禁止 start 第二线程。 +- `test_runtime_retries_thread_join_without_resubmitting_closed_storage_cleanup` 覆盖 cleanup 成功后 join 超时：_storage_closed=True，_storage_started/storage_ready 保留，重试不重复提交 store.close，只重新观察 join。 +- runner pending 在 cleanup 再次取消时不丢；同 role 全 shard 禁重领，旧 callback finally 不污染新 state；core 保留 runtime/adapter 并传播未退出异常，阻止业务依赖提前关闭。 + +自动化完整回归结果：**235 passed，8 条既有 deprecation warnings，40.68s**（2026-10-11，日志 `/tmp/OpenViking_shard_refactor_final3.log`）。该结果包含线程 join 重试回归，不等同于真实 Redis/Lua、Cluster 故障注入或吞吐验证。 + +真实 Redis/Lua、Cluster 接管故障和吞吐测试未执行。Lua 共12个文件，R→M、源 TIME、空范围、未决跨槽请求、仅日志终结等协议限制保持不变；本地取消和真实 task done 仍不证明服务端写入已撤回，不构成跨槽 fencing。 + +Session 外层纯检查、原 commit 参数、best-effort 两字段 patch、deferred report 成功才 ACK 及 WM/QueueFS 兼容验收继续保留。检查与 commit 不原子，接受时间可能记录失败，不承诺严格 idle 边界、零重复或每次成功可恢复；原通用 ready/QueueFS 恢复不用于恢复 idle 记录。其他 Redis 可靠性限制以主文档为准。 diff --git a/docs/zh/design/session-idle-auto-commit-scheduler-vibecoding.md b/docs/zh/design/session-idle-auto-commit-scheduler-vibecoding.md new file mode 100644 index 0000000000..b0972316a6 --- /dev/null +++ b/docs/zh/design/session-idle-auto-commit-scheduler-vibecoding.md @@ -0,0 +1,453 @@ +# Session 空闲超时自动提交调度器 VibeCoding 实施文档 + +- 状态：**采用临时 shard 租约、真实 Redis I/O 生命周期跟踪和有界关闭观察；可靠性与检验边界见第 10、13、14 节。** +- 来源：[内核超时自动触发技术方案](https://bytedance.my.larkoffice.com/docx/KUbYdV4SJoxxsyxhgz1mW1wMypb)，最近读取 revision 3059；与文档不同之处以本轮用户明确修正为准。 +- 代码基线：OpenViking `7d77e3d68fea82070a0b700a9acd64c40e7f4a9a`；本轮对比原基线 `6e1030d9951c4c19048e4758ef130be5da4abfe0` 后的 7 个提交。 +- 更新时间：2026-10-11。 +- 附录：[逐文件代码骨架](session-idle-auto-commit-scheduler-code-design.md)。主稿说明行为，附录列出字段、函数签名及 Lua 契约；省略体只省去实现细节，不作为行为验收证据。 + +## 1. 当前结论与本轮变更 + +采用业务统一接入、Redis 资源桶、调度桶、注册表及游标的结构，首期支持 session 的 IDLE_ONESHOT。调度和回调实例均通过 Redis 临时领取 shard，持锁处理全部积压后释放，轮内按 shard_id 递增。 + +本轮明确移除资源伴随状态 V、资源更新 revision、Redis 失败记录及元数据格式版本字段。报告按 R → M 执行；调度按读 R → 写 B → 原子写 G/C 执行。仅 report 写入/更新 R 的 score，调度只按已发布 C 删除源成员，不把调度读取与报告更新混为同一条调用链。 + +**有两个保证不能由这些步骤直接推出：** R 写完、M 尚未写完时退出会留下缺失元数据；续租失败后取消本地任务不能撤销已发出的跨槽 Redis 请求。前者由完整 item_id 回源兜底；后者仍是当前共享调度桶在自动接管时的残余风险，见第 10.3 节。本稿不把这两点写成已证明安全，也不擅自加入新的桶隔离标识。 + +### 1.1 最新内核适配与本轮放宽保证 + +Redis 是可选能力，默认 `idle_backend=legacy` 与既有提交能力保持不变。本轮覆盖旧的锁内 idle 检查及 idle 归档恢复方案：`SessionService.commit_on_timeout` 加载最新 session/meta，在服务层完成纯检查；通过后直接调用原 `Session.commit_async`。Session 的通用提交内部不增加 Redis 或 idle 判断，也不为 timer 拆分提交实现。 + +检查与提交不在同一把业务锁内原子执行。检查通过后、原提交读取消息前追加的新消息允许一起提交；policy 或 timeout 在此窗口变化也不触发一次新的最终 idle 检查。`last_idle_commit_accepted_at` 仅为外层 best-effort 接受记录，写失败不改变 accepted，不提供归档查证或修复缓存。这一方案不承诺严格空闲边界、零重复提交或每次 accepted 都能持久记录。 + +| 最新代码契约 | 本方案保留方式 | +|---|---| +| Session.commit/commit_async、SessionService.commit/commit_async 的 enable_working_memory | 原签名与三态透传保持不变，False 不能当作未传；不新增 idle 专用 commit API | +| MemoryPolicy.enable_working_memory 默认 False | idle 调用不覆盖 WM，继承当前 session/user policy；保留显式启用策略 | +| accepted 和原有 skipped 结果包含 effective_enable_working_memory | 直接返回原 commit 结果，不重建丢字段的精简结果 | +| QueueFS 的 memory_policy_version 及完整 policy 快照 | 沿用原序列化、兼容读取和恢复，不把它当成 Redis M.schema_version | +| .done 的 enable_working_memory 与旧字段兼容；关闭 WM 可无 summary | 原 ArchiveStore 能力保持不变；timer 只以原 commit API 返回 accepted 为成功，不等待后台任务或摘要 | + +原通用提交已有的 ready、QueueFS、恢复及 Working Memory 契约均继续保留，不成为 idle 接受时间的存储或恢复机制。 + +## 2. 已确认约束 + +1. 业务只使用 report、unregister、register_handler 三个方法，通过依赖注入取得协议；调度、Redis、消费、生命周期均封装在内部。 +2. 各业务自行注册处理器，通用模块不硬编码 session 类型、policy 或并发映射。 +3. 首期 IDLE_ONESHOT；FIXED 仅预留，调用时明确不支持。 +4. 消息成功追加后直接 await report，无内存去抖；按先 R 后 M 的顺序写 Redis。 +5. 不使用 V、源关闭水位或更新顺序号；接受同 item_id 近间隔并发写入时旧请求覆盖新请求的误差。 +6. 只处理实际到期数据：U 为当前时间去掉秒和毫秒，扫描 `(C,U]`；U=C 或没有数据就跳过。 +7. 不使用 group_id。分钟调度桶全部写完后，G/C 同槽 Lua 注册并推进游标；已发布桶不再生产。 +8. 元数据 TTL 为 `max(2×timeout,24小时)`；完整 item_id 承载 type/account/user/local_id，M 缺失时仍可回源。 +9. 已投递任务由服务层检查加载时的最新状态；未到最后消息到期时间则 deferred。检查通过后不再做持锁最终 idle 校验，期间追加的新消息允许一起提交；新活动的未来计时独立保留。 +10. idle 冷却统一使用 `last_idle_commit_accepted_at`，它是服务层 best-effort 接受记录，不使用通用自动提交统计时间作判断，也不建立 idle 归档指针或修复缓存。 +11. timeout 修改不批量重写旧 R；回调冷却校验不通过时 report 新计时，due 为本次检查时间加当前 timeout；report 成功后完成旧事件。 +12. 同一调度事件最多三次调用机会，包含首次；耗尽只写日志并条件清除 B，不在 Redis 保存失败记录。 +13. 每个实例每个角色一次领取一个 shard，处理全部积压后释放，再尝试更大的 shard；一轮结束从头开始。 +14. 锁 TTL 120 秒，每 20 秒续租；续租失败立即停止本次处理，不续跑旧任务，后续重新领取必须使用新 token。 +15. 不处理同名资源删除重建；不增加 incarnation、持久待注册集合、outbox 或零漏触发承诺。旧扫描器仅参考业务处理步骤。 + +## 3. 业务契约与内部边界 + +```python +from typing import Protocol +from openviking.scheduling.expiry import ( + HandlerOptions, ItemIdentity, ExpiryHandler, ExpiryRegistration, ExpiryRegistrationResult, +) + + +class ExpiryScheduler(Protocol): + async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: + """先写入资源计时，再创建或刷新元数据；等待本次结果。""" + ... + + async def unregister(self, identity: ItemIdentity) -> None: + """幂等注销当前计时及元数据；已投递事件按业务当前状态结束。""" + ... + + def register_handler(self, item_type: str, handler: ExpiryHandler, *, + options: HandlerOptions | None = None) -> None: + """业务启动时注册自己的处理器及处理能力。""" + ... +``` + +| 类型 | 字段及约定 | +|---|---| +| ItemIdentity | item_type、account_id、user_id、local_id | +| ExpiryRegistration | identity、due_at_ms、timeout_seconds、mode=IDLE_ONESHOT；无 revision | +| ExpiryRegistrationResult | accepted_due_at_ms；不再返回需要版本比较才能判断的 DUPLICATE/STALE | +| HandlerOptions | concurrency 默认 4、max_attempts 默认 3（含首次），同类型各副本保持一致 | +| CallbackRequest | identity；框架不透出 claim、缓存命中、游标或 Redis 状态 | +| CallbackResult | COMPLETED / RETRY 与原因码 | +| ExpiryHandler | async handle(request) -> CallbackResult | +| ExpiryError | 业务错误码、错误信息、retryable | + +`api.py` 定义公开契约，包入口只重导出上述协议/类型。业务不构造 store、keys、runtime，不调用启动、关闭、冻结或 ACK。SessionService.configure_expiry 委托 session_expiry.install_session_expiry，业务模块自行安装上报适配器并注册 handler；core 只组装、注入和启停。 + +`api.py` 中的 `SUPPORTED_ITEM_TYPES` 是类型信息的唯一来源。内部 HandlerRegistry 只允许注册清单内类型，并在启动消费前冻结；report 在写 Redis 前按同一清单校验。独立调度器直接使用代码清单定位未发布桶，不访问 Redis 类型目录；处理器和选项只在本地。只有 G 注册后的桶可消费。 + +同一 namespace 的回调副本加载相同的业务处理器集合；缺 handler 的类型不领取，限频报警，不能假设其他实例会自动接管该类型。session 只有选择 Redis idle backend 时才安装本业务接入，其他业务启用 timer 不应顺带启动 session 的 Redis 计时。 + +完整 item_id 使用 `item_type:encoded_account:encoded_user:encoded_local_id`，身份组件 UTF-8 后用无填充 URL-safe Base64，严格解码并重编码验证，禁止花括号或冒号改变 key 结构。编码规则和 xxhash64 seed 固定，不增加逐资源格式版本字段。 + +## 4. Redis 数据结构与 report + +`<...>` 是占位符，只有明确列出的 `{...}` 才是 Redis hash tag；service_key 为编码后的共享 service_name。 + +| 名称 | Key 示意 | 内容 | +|---|---|---| +| R 发布源 | `expiry:publisher:source::` | ZSET，member=完整 item_id，score=due_at_ms | +| M 元数据 | `expiry:metadata::` | HASH，item_type、local_id、account_id、user_id；带 TTL | +| B 消费桶 | `expiry:consumer:bucket::::` | ZSET，初始 score=0；minute 为窗口结束边界 | +| G 注册表 | `expiry:consumer:registry:{:}` | ZSET，member=type:shard:minute，score=minute | +| C 发布进度 | `expiry:publisher:progress:{:}` | 最后已发布的包含式分钟上界，UTC 毫秒 | +| owner | `expiry:publisher:owner:{:}` 或 `expiry:consumer:owner::{:}` | 当前领取 token，TTL=120秒；publisher/consumer 分开 | +| namespace | `expiry:namespace:` | 初始化状态、固定 N/hash/编码配置及内部类型名称目录 | + +仅 G/C/publisher owner 需要同槽发布；R、M、B 各自的单 key 操作不必同槽。没有 V 或 Redis 失败存储 key。 + +### 4.1 元数据与格式字段 + +M 只保存身份字段。原设计的 `schema_version` 用于区分序列化格式；当前没有多版本解析或迁移流程，字段本身不能完成兼容处理，**本期删除**。配置仍比较实际的 shard_count、hash_name、hash_seed、identity_encoding，防止不同实例使用不同寻址规则；这属于命名空间配置校验，不是给每条资源增加版本。 + +TTL 由框架内部统一计算，业务不另传 TTL： + +```text +metadata_ttl_ms = max(timeout_seconds × 2000, 86_400_000) +``` + +创建、续期使用同一公式，从 Redis 执行时开始计时。timeout≤12小时取24小时，超过12小时取两倍 timeout。ensure_metadata.lua 在单 key 内检查身份、必要时 HSET、最后 PEXPIRE，避免 EXISTS/EXPIRE 的临界过期竞态。 + +### 4.2 report 的真实顺序和错误语义 + +1. 校验身份、模式、正 timeout 和整数范围，确认任务类型已发布。 +2. 调用只访问 R 的 upsert_resource.lua，用源节点 TIME 校正为未来 due，然后 ZADD 覆盖 score。 +3. R 明确成功后，调用 ensure_metadata.lua 写 M 或续 TTL。 +4. 两步均成功才返回 accepted_due_at_ms。R 失败不继续写 M；R 成功、M 失败则接口报错，但不回滚已写 R，避免覆盖或删除并发的新报告。 + +先 R 后 M **不能保证 R 中每个成员都有 M**：两步间退出、M 写失败或后来过期都会造成缺失。完整 item_id 回源是必要正常路径。M 的 Redis 网络错误与明确缺失仍区分，前者按可重试错误处理。 + +本期接受上报乱序覆盖，不做 CAS、不保留更新 revision、不识别重复请求。报告重试可能覆盖较新 due，或在原任务已处理后建立新计时；不声称资源级无限幂等。删除 R/M 可重复执行，但并发删除与新上报的生命周期顺序由业务负责，同名重建不在范围内。 + +### 4.3 report 与调度并行时的时间边界 + +report 不读取 C；调度读取 C 时也不通过该调用更新 R。不同请求仍可以同时执行 report 和调度，不能因为代码属于两个阶段就认为全局互斥。 + +保留一个无需伴随状态的单 R Lua 约束： + +```text +accepted_due = max(requested_due, source_redis_now_ms + 1) +ZADD R accepted_due item_id +``` + +TIME 与 ZADD 在同一脚本中执行；调度的 U 也取该 R 所在 Redis 节点的 TIME。节点时间正常前进时，调度取得 U 后到达的 report 会写到 U 之后；之前写入的未来项随时间到期后可扫描。此设计不比较业务请求先后，仅确保写入时 due 在未来。 + +调度取得 U 的脚本只读时间、不修改 R。若改用另一个节点时间或在客户端算完时间后长时间滞留再 ZADD，这个保证不成立。未保留历史关闭水位时，也不承诺任意时钟回退下无遗漏；UC 时，按 score/member 分页查找下一条到期成员，确定下一个有数据的分钟 M。没有成员即结束本轮范围。 +3. 使用 `SUPPORTED_ITEM_TYPES`，确定 C 未覆盖 M 后清理该分钟未注册的临时 B，按当前 R 重建；不恢复旧内存分组。 +4. 分页读该分钟所有成员，按 type 批量 ZADD 0 到 B。每页有限，但不因页数/分钟数达到某个配额而交出 shard。 +5. M 全部读完且 B 写入结果明确后，单条 Lua 检查 owner token 和预期 C，注册实际有数据的 B 并设 C=M；随后按 C 清理 R。 +6. 继续后续有数据分钟。扫描追平本轮 U 后重新检查当前上界和新增到期积压；确认无可处理数据，结束并释放锁，继续更大的 shard。 + +扫描期间成员可能续期或被删除。若选定分钟在真正分页时已经变空，只跳过该分钟，不发布；本地查询下界向前移动以避免空转，持久 C 保持不变。若已读成员随后活跃，则本次事件仍可投递，新 due 留在 R。 + +G/C Lua 允许跨过已经扫描确认为空的分钟，不再要求 M=C+60000，只要求 M>C 且已完成中间扫描。分钟结束前不推进 C。读取 U、读 R、写 B 都不写入或更新 R 的 score；源删除是发布后单独的清理步骤。 + +### 6.3 分页与恢复 + +源页使用 `(last_score,last_item_id)`，不使用递增 OFFSET。单 key Lua 根据游标当前 rank 定位；末项续期或被删时可按 score/member 二分定位，约 O(log²N+page_size)，不全量扫描同 score 集合。 + +重领后丢弃旧内存状态，重新读 C。C 已覆盖的分钟不清桶、不重投，G 已被消费摘除也不能重新发布。确定未注册的分钟按当前 R 重做。空范围不推进 C，曾写入但未注册、成员后来全部续期的 B 可能成为临时遗留 key；不注册它，不给已注册 B 设置短 TTL，必要时离线核对清理。 + +发布响应未知时，租约仍有效可读 C 查证：C 已推进则确认；C 未推进不证明旧请求永远不会执行，保留已写完 B 并重试同一个发布，不能回到 reset/stage。租约失败立即取消本次处理，不在重新领取后接着执行旧的挂起函数。已发出跨槽命令的残余影响见第 10.3 节。 + +源清理使用 ZREMRANGEBYSCORE R -inf C，保留正常未来 due。复杂度 O(logN+K)，大积压和集中到期需测单命令耗时；若过大再改为单 key 有界批删，不把分页投递等同于删除已无大 key 风险。 + +## 7. Lua 与故障处理 + +| 脚本 | 原子域 | 职责 | +|---|---|---| +| upsert_resource | R | TIME + 未来 due + ZADD，不读 C、不写伴随状态 | +| source_time | R 所在 slot 的只读脚本 | 返回源节点 TIME，不修改 R | +| ensure_metadata | M | 校验身份、创建/刷新和 TTL | +| read_source_page | R | score/member 有界分页 | +| publish_minute | G/C/scheduler owner | 比较 token、预期 C，注册非空分钟并推进 | +| read_registry_page | G | 已发布桶有界发现 | +| claim | B | 原子领取并计数，最后一次超时只返回待终结项，不再领取 | +| ack | B | 按 claim_score 条件删除，成功和耗尽终结共用 | +| acquire/renew/release_owner | 对应 owner | NX 领取、比较 token 续期、比较 token 释放 | + +脚本写前检查参数、key 类型；Lua 运行异常不提供跨命令回滚。删除 R 使用单 key ZREM，删除 M 使用 DEL，不再需要注销伴随状态脚本。 + +| 故障 | 处理 | +|---|---| +| R 成功、M 失败 | report 报错，保留 R；完整身份仍可回源 | +| M 缺失 | 解码完整身份，正常调用业务 | +| 写部分 B 后取消/退出 | 未发布不消费；重新领取后读 C 和当前 R | +| 发布后响应丢失 | 查 C 或在同一有效租约下重试原发布，不重写已发布 B | +| 清理失败 | 以后只补清理，不重投已发布分钟 | +| 续租失败 | 本次租约立即失效，取消处理；旧 token 不复活 | +| callback 未耗尽且失败 | 保留 B，超时后重领 | +| 第三次失败/超时 | 写耗尽日志，然后条件 ACK；无 Redis 失败记录 | +| report 重新计时失败 | 旧 B 不 ACK，按剩余机会重试 | + +## 8. 回调、三次机会与仅日志终结 + +回调实例持有 shard 后，分页发现 G，按 type 的进程级并发处理全部可执行积压。发现循环需继续遍历，不能被最老页或单个慢类型阻塞。重试等待期间仍续租；已注册桶清空、在途任务收尾后释放 shard。未知 handler 的桶保留并报警，不无限持锁等待不存在的处理器。 + +```text +score = attempt × 2**40 + started_at_seconds +initial_score = 0 +session attempt = 0,1,2 → 最多三次机会 +``` + +claim 在 B 内原子占用一次机会，每个 claim 至多调用一次 handler。进程可能领取后、调用前退出，因此实际调用可能少于三次；不退还结果不明的机会。新事件包括冷却重投产生的事件有独立预算。各类型上限由业务 HandlerOptions 注册，session 固定采用3。 + +按 attempt 层有界查询超时项；新项优先但为重试保留预算。第三次超时的成员不生成第四次 claim，而是返回 exhausted 列表，由 consumer 写日志后条件 ACK。脚本不直接删除这些项，避免正常路径中先删后无法输出日志。 + +明确 RETRY/异常且最后机会耗尽时，consumer 同样先写结构化日志，再条件 ACK。日志包含资源身份、桶分钟、claim_score、机会数、失败原因及是否结果未知，不记录消息正文。Redis 不保存失败 HASH、索引、死信或重放记录。ACK 失败时留 B 以后重复终结，日志可能重复；日志输出与 Redis 删除不构成事务，不能承诺故障下日志必达或恰好一次。 + +COMPLETED 后 ACK 出错，只重试同一条件 ACK，不再调用 handler、不算一次新的业务失败。最后一次 accepted 后 ACK 是否生效无法查清时，最多记录结果未知并清理，不额外调用业务来突破三次上限。条件 ACK 不匹配则不删除新 claim。B 空后单独摘 G，不删新活动 R/M。 + +回调处理超时与 shard 租约是独立参数；前者决定任务何时可重领，后者每20秒续租，不以任务耗时长为由主动释放。 + +## 9. Session idle 时间与冷却重投 + +### 9.1 服务层 best-effort 接受记录 + +SessionMeta 仅增加 `last_idle_commit_accepted_at`，缺字段按空串处理；现有普通统计字段保持兼容但不参与 idle 决策。此字段不是原通用提交 ready 的原子记录，不存储 idle archive 指针，不提供 idle readiness 查证或恢复。 + +原 `Session.commit_async` 返回 `status=accepted` 即为本次 timer 业务成功，不等待后台任务。之后服务层 await `_record_idle_commit_time(session_id, ctx, accepted_at)`：复用既有 session root exact 路径锁，锁内直接重读最新 raw meta 字典，仅 patch `last_idle_commit_accepted_at` 和 `updated_at`，写入传 `lease_ref=lease`，不以检查时的旧 Session 对象全量保存。接受时间只以有效新旧值的 `max` 合并：已有有效更大/相等时间时不写，坏旧值记录日志后可覆盖，避免较早并发调用迟到回写旧成功时间。 + +记录更新失败只写日志，仍原样返回 accepted；若 session 已删除，跳过记录，不创建目录或重建 meta。该 helper 的读取/锁/写异常属于 best-effort 记录失败；入口加载、纯检查和原 commit 的异常则向外传播，不能伪装成无消息或 accepted。无消息等 skipped 和 deferred 不更新时间。 + +### 9.2 timeout 变更、外层检查与重新报告 + +timeout 更新不批量改已有 score。`SessionService.commit_on_timeout` 加载最新 session/meta 后，以本次 `now` 调用纯检查 `_check_timeout_commit(session: Session, now: datetime) -> Optional[dict]`，顺序如下： + +1. 明确 session 已删除：skipped/not_found，handler 转 COMPLETED；其他加载/读取异常向外传播。 +2. policy 禁用或 timeout≤0：skipped/idle_disabled，不重新计时。 +3. 没有可提交消息：skipped/no_messages，不更新时间，避免无内容 session 反复重投。 +4. 有消息且 `now-last_idle_commit_accepted_at < current_timeout`：返回 deferred/idle_cooldown，due 为本次检查 `now + 当前 timeout`。 +5. 冷却通过但 `now < last_message_at + current_timeout`：返回 deferred/idle_activity，due 为 `last_message_at + 当前 timeout`。 +6. deferred 仅由 SessionService 和 SessionTimeoutHandler 掌握，不修改 meta 或调用 commit。handler 统一 report；R→M 两步成功后才返回 COMPLETED、ACK 旧 B，失败传播以保留旧 B。旧 B 的 ACK 不删除未来 R/M。 +7. 检查返回 None：直接 `await session.commit_async(keep_recent_count=0, persist_keep_recent_count=False, record_auto_commit_success=True)`，不新增 idle 专用提交入口，不在提交锁内进行最终 idle 判断。原结果为 accepted 才执行 best-effort 接受时间记录，然后返回原结果；commit 异常传播。 + +例如最近 idle accepted 为12:00，旧事件12:10到期，而 timeout 已调大为30分钟。12:10回调因冷却未满足，report 新 due=12:40；不是回填12:30，也不复用旧 timeout。这是“当前时间+当前 timeout”语义。最后消息尚未到期的活动延后则使用最后消息时间加当前 timeout。 + +服务层复用 `parse_iso_datetime` 和 `get_current_timestamp` 处理时间，纯检查只读加载状态，不获取提交锁、不写 Redis 或 meta。重投不是成功提交，不更新接受时间；重复重投可能顺延 R，仍接受无版本顺序保护的覆盖行为。首次无接受时间不构造虚假冷却窗口。 + +检查完成后的追加、policy/timeout 变化不触发最终复检，允许新消息一起提交；检查与 commit 之间的并发以及接受时间记录失败都可能弱化冷却、导致重复调用。这是本轮明确放宽的保证，不要求从原 ready 反查或补写该字段。 + +### 9.3 与 Working Memory 策略的关系 + +idle timeout 的外层触发检查与 Working Memory 策略独立。直接调用原 `commit_async` 时不传一次性 WM 覆盖，沿用默认 `enable_working_memory=None`，继承 session/user 设置，未配置时采用默认 False。原始归档、QueueFS 及符合 policy 的长期记忆提取继续存在。 + +原手工提交的 `enable_working_memory=True/False` 只覆盖本次有效 policy，不写回用户或 SessionMeta 默认 policy；原队列快照、`memory_policy_version` 及恢复兼容契约不变。本方案移除的仅是 Redis 元数据中的 schema_version，不删除原通用提交能力。 + +WM 关闭时没有 `.overview.md` 是正常结果，不据此重试或推迟 timer ACK。后台任务继续使用既有失败/恢复流程，不回流为 timer 的三次回调重试；accepted 后 timer 只处理自身 ACK。 + +## 10. 实例抢占、续租与退出 + +### 10.1 一次一个 shard，轮内递增 + +每次抢占生成独立随机 owner token，不依赖进程身份或 Kubernetes。每个角色独立遍历 `0..N-1`，一次最多持有一个 shard。抢不到继续尝试更大的编号；抢到则处理全部积压，完成才释放；遍历和轮间节奏由worker自己控制：scheduler本轮后等下一分钟边界，callback本轮后等callback_poll_seconds，再从0开始。示例编号1..256与代码0..255仅是编号约定差异。 + +不使用成员名单、Rendezvous Hash、固定 shard 分配或每批强制移交。分页和 pipeline 有界只是控制内存与单命令工作量，不是释放锁的依据。热点 shard 若持续产生积压，可能长期由同一实例处理；其余实例继续争抢其他 shard，不承诺严格轮转公平或按资源量均衡。 + +### 10.2 token 操作、单 shard 执行与 worker 节奏 + +职责只拆到必要边界，不引入通用 worker 框架： + +| 对象/文件 | 职责 | +|---|---| +| ShardLeaseCoordinator / leases.py | 仅 try_acquire、renew、release，调用既有 Redis token 操作；不遍历 shard、不创建业务/heartbeat任务、不决定轮询节奏 | +| ShardLease / models.py | owner、保守 monotonic 截止、lost 与 revoked 两种单向状态；ensure_active 检查两者 | +| ShardExecution / shard_runner.py | owner、shard_id、ensure_active、spawn 工厂、wait；强引用跟踪 processor 子任务和辅助 waiters，汇总异常/unknown | +| ShardRunner / shard_runner.py | run_if_acquired 一次执行一个 shard；后台 heartbeat、stop grace、失锁撤销取消、有界 join、同 role pending 保护 | +| publisher.py / consumer.py | 自己遍历所有 shard，分别控制 publisher 下一分钟边界、callback poll 节奏；只把单 shard process 交给 runner | +| runtime.py | 构造同一个 runner 并注入两个 worker，保留角色任务，协调有界停止和依赖关闭 | + +1. `run_if_acquired(role, shard_id, process, stop)` 在 stop 时不领取、busy 返回 False；成功取得新随机 token 后构造全新 execution/process。每 role 只允许一个 active，**同 role 任一 shard 尚有 pending 都禁止再抢锁**，抛 `shard_not_stopped`；另一角色独立。 +2. Coordinator 的 try_acquire 使用 SET NX PX 120000；renew 比较 token，runner 后台每20秒调用。请求异常、超时、取消、token 不同/缺失或响应晚于之前本地截止，均保守置 lost；迟到成功也不能复活 lease。截止为请求发起 monotonic+TTL，不按收到响应重新延长。 +3. lost 表示 ownership 已不可信；revoked 表示本地执行已撤销，正常清理也会设置，**不把正常撤销冒充失锁**。`ShardExecution.ensure_active` 首先检查 `asyncio.current_task().cancelling()`，若存在取消请求，**同步置 outcome_unknown 并抛 CancelledError**，再检查 lease 的 lost、revoked 和截止。即使 handler 吞掉取消并返回 COMPLETED，后续检查点也不能放行 ACK，不依赖较晚执行的 child done callback 才标记未知。spawn 先检查，再创建执行工厂的 task，业务 processor 不取得续租/释放接口。 +4. runner 监视 processor、heartbeat、lost 和有效截止。正常 stop 开始 shutdown grace，heartbeat 继续；**grace 期间仍监视失锁及有效截止**，失锁立即 revoke/cancel，不只等待慢 processor。grace 超时、外层取消或 processor 返回却仍有 children，均按 forced 收尾。 +5. 收尾先 revoke，再 cancel process、heartbeat、children/waiters；若 renew 正在请求中，取消前同步置 lost，不能被抑制取消后的成功响应复活。join 用 `asyncio.wait(timeout=request_budget_seconds)` 有界观察，不能用取消后无界 await/gather，或把 wait_for 超时当作任务已经终止。 +6. 未终止 task 在等待前放入 runner pending 强引用（含 acquire/release/heartbeat），done callback 读取异常且只摘除自己的集合；二次取消 cleanup 也不能孤儿化 task。预算结束仍未退出则报 `shard_not_stopped`，worker 必须向 runtime 传播而非继续下一 shard。旧 tasks 真实退出后才允许新 token/新 execution；不恢复旧函数挂起点。 +7. process/children 普通业务异常但任务全部终止、无 forced/unknown 且 ownership 仍可信，允许停止 heartbeat 后比较 token 释放，再传播异常；**不是所有业务异常都等于失锁**。forced、lost、未知结果或本地工作未退出均保守停止续租、留 TTL，不主动释放。release 本身也有界观察，超时/未知不能假装已交接。 + +consumer 每个 callback 用 execution.spawn 工厂创建并被 runner 跟踪；poll 用 execution.wait，不自建旧式 lost task 等待。正常 stop 可等已启动 callback，期限/失锁由 runner 在外层持续监视。仅旧 tasks 全部退出后才清共享队列、reserved 和 task state，避免旧 finally 释放槽污染新轮；retry 层缓存在每次单 shard 执行边界清理，不跨租约累积。 + +所有有界等待都以事件循环能继续调度为前提。async task 不能消除进程暂停、阻塞事件循环，也**不能强杀抑制取消的协程**；pending 强引用和禁止同 role 重领是本地生命周期防护，不证明 Redis 请求已撤销。 + +### 10.3 必须保留的可靠性限制 + +本轮按用户指定的“续租失败停止、下次重新抢锁”实现，不加入按 token 隔离的 B。该措施可以停止本地后续动作，**不能撤回 Redis 已接收但尚未完成的 ZADD/UNLINK，也不能让跨槽的检查与写入原子化**。 + +因此仍有窗口：旧写入已发出或进程暂停在检查之后，新实例因 TTL 到期接管并发布同一个 B，旧请求随后执行。G/C/owner Lua 会拒绝旧发布，但无法拒绝另一个 slot 上的旧 B 写入。加长 TTL、取消 coroutine、换新 token 均不构成完整防护。 + +上线前需要故障注入验证这个边界，并决定是否接受残余风险。如果要求在暂停、分区和请求延迟下仍严格防止旧写入影响新 owner，需要桶隔离或 B 所在槽的写入版本校验等进一步变更；本轮不替用户自动选择。回调侧的 claim/条件 ACK 只能保护消费记录，不能抵御生产者后来无条件 ZADD 0。 + +## 11. 配置、启动与文件清单 + +顶层 `expiry` 配置保留 `service_name` 并增加 `publisher_separate_process=true`，删除 `reporting_enabled`、`scheduler_enabled`、`callback_enabled` 三个可组合角色开关。默认 `session_auto_commit.idle_backend=legacy`；只有 `enabled=true && idle_backend=redis` 时才整体启用 report、publisher、consumer。`service_name` 是所有实例共享且稳定的文档服务名。`shard_count` 默认 128、范围为 1 到 512，身份编码/hash seed 统一；共享同一 namespace 的实例必须使用相同 N。Session consumer 并发默认 8，可通过 `session_consumer_concurrency` 配置为任意正整数。Redis 扩物理节点由 slot 迁移处理。 + +新增公共角色参数 `owner_ttl_seconds=120`、`owner_renew_interval_seconds=20`，删除静态 scheduler_shards、callback_shards、领取数量上限及处理若干分钟就移交的参数。保留 scan_batch_size、registry_page_size、类型并发、processing_timeout_seconds、callback_poll_seconds、shutdown_timeout_seconds。 + +API 顺序：配置检查 → Redis backend 未启用则跳过 Runtime → 创建 API Runtime → 注册 session handler/reporter → 统一启动 storage/namespace → 冻结处理器并启动 consumer；`publisher_separate_process=false` 时同一 Runtime 也启动 publisher。默认值为 `true`，此时 `openviking-server` 在启动 API 前拉起并监督唯一 publisher 子进程。关闭先停止 expiry 角色及在途业务调用，再关闭QueueFS/AGFS/Redis。runtime用asyncio.wait有界等待role tasks和runner pending，仅真实done才移除引用；未退出抛shard_not_stopped并保持相关依赖打开，不能在finally强关。core._stop_expiry保持runtime强引用，runtime.close成功后才清runtime并detach adapter；pending失败时runtime和adapter均保留，禁止新calls由runtime.accepting_calls=False负责；shard_not_stopped向外传播，中止后续QueueFS/AGFS关闭，允许后续再观察/重试关闭。其余core启动与optional backend不变。只启动storage不代表report已开放。API Runtime 内部固定允许report，以便 consumer 完成冷却重投。 + +runtime组装为 `store → ShardLeaseCoordinator(store, config, metrics) → ShardRunner(leases, config, metrics)`，同一个runner注入 `ExpiryPublisher(store, runner, config, metrics)` 与 `ExpiryConsumer(store, runner, bindings, config, metrics)`。 + +RedisLoopExecutor 以 `_RedisRequest` 强引用登记实际 Redis-loop task；`finished` 仅在该 task 及其原始 operation 真正终止时完成。`run` 使用 shield 等待 finished，取消时只向远端发一次取消，再通过 `_join` 保留调用方 wrapper 至实际退出；多次取消也不能切断桥接。因此 runner 能观察未终止 I/O，而不是把跨线程 Future 的 cancelled 状态当作远端退出。`quiesce(timeout_seconds)` 拒绝新外部请求，以有界等待观察所有登记请求（含尚未开始的请求）的真实 finished；未退出抛 `shard_not_stopped`，保留请求和相关依赖。 + +`ExpiryRuntime.close` 依次执行 `stop_roles → io.quiesce → io.run_cleanup(store.close) → io.close`。`run_cleanup(operation, timeout_seconds)` 在 Redis loop 提交受跟踪的 raw parent，不先占用 gate；store.close 内部可嵌套 io.run 关闭 client，既允许 quiesce 后清理，也避免 max_inflight=1 时递归等待 gate。cleanup 超时/取消不等于退出，仍保留 request。io.close 再次 quiesce 后仅发一次 loop.stop，以本次 close 的剩余预算 join 线程；线程仍 alive 则保留 loop/thread/gate 引用并报未退出，后续 close 可重试、不重复 stop，start 不得创建第二线程。runtime 增加 `_storage_closed=False`，start_storage 成功重置；仅在其为 False 时 run_cleanup，成功后置 True。若随后线程 join 超时，仍保留 `_storage_started/storage_ready`，重试 close 跳过已成功的 store cleanup，只重新观察/等待 join，不向已 stop_requested 的 loop 再提交 cleanup。只有线程真实退出才清 I/O 引用及 `_storage_started/storage_ready`。 + +各关闭阶段分别使用配置预算，不能宣称整个 runtime.close 只有一个总预算；run 取消后的 join、原始 operation 收尾及线程内清理也不提供强杀保证。外层有界观察失败时保留实际工作引用与依赖，不把取消、超时或 join 返回误写成已关闭。 + +内部 `openviking_cli.expiry_bootstrap` 模块读取完整 `ov.conf` 后只运行 publisher，不初始化 AGFS、模型或 SessionService。它通过 readiness 文件与 server 父进程握手，自动遍历全部 N 竞争，不要求显式分片列表；启动失败阻止 API 启动，运行期异常退出会带动父进程退出。 + +| 文件 | 计划职责 | +|---|---| +| scheduling/expiry/api.py、__init__.py | 三个业务方法、DTO、处理器选项及受限导出 | +| scheduling/expiry/models.py | 内部桶/页/claim/租约、TTL与时间计算，无资源版本 | +| scheduling/expiry/config.py | 轻量 expiry 配置 | +| scheduling/expiry/keys.py | 完整身份、hash、R/M/B/G/C/owner寻址 | +| scheduling/expiry/metrics.py | 低基数指标和耗尽日志入口 | +| scheduling/expiry/registry.py、service.py | 本地处理器注册、私有facade、R→M调用链 | +| scheduling/expiry/redis_store.py、lua/ | 单key操作、G/C发布、抢锁续租、结果未知分类 | +| scheduling/expiry/leases.py | 仅try_acquire/renew/release，Redis token操作，不持有worker循环 | +| scheduling/expiry/shard_runner.py | ShardExecution检查点/spawn/wait；单shard heartbeat、grace/失锁撤销、有界join、pending强引用 | +| scheduling/expiry/publisher.py | 自遍历全部shard/分钟节奏，到期积压、空范围、发布和清理 | +| scheduling/expiry/consumer.py | 自遍历全部shard/poll节奏，runner跟踪子tasks、旧tasks退出后清state、三次机会及日志终结 | +| scheduling/expiry/runtime.py | 共享runner组装、I/O loop、有界启停，未退出工作保留引用/依赖 | +| service/session_expiry.py | 消息上报、业务handler及冷却重投 | +| session/session.py | 追加reporter、SessionMeta接受时间字段；原通用提交不加idle/Redis判断 | +| service/session_service.py | 最新状态加载、纯超时检查、直接原commit调用、best-effort时间patch | +| service/core.py | 注入、正确启停与backend选择 | +| openviking_cli/utils/config/*.py | timer配置集成及idle_backend兼容 | +| openviking_cli/expiry_bootstrap.py、server/bootstrap.py、pyproject.toml | 内部 publisher 子进程、父进程监督、Redis可选依赖及Lua包资源 | + +上述内核路径均以 openviking/ 为前缀。QueueFS/SessionCommitProcessor、原通用 ready/恢复、阈值提交和手工提交语义继续保留。 + +## 12. 实施顺序 + +| 阶段 | 交付与验证 | +|---|---| +| P1 | 公开契约、namespace实际配置、身份编码、store和单key Lua | +| P2 | R→M上报、TTL、注销、元数据缺失回源 | +| P3 | Coordinator三操作/ShardRunner单shard执行、heartbeat/grace失锁监视、lost/revoked不可复活、有界join/pending及可信异常释放 | +| P4 | 当前分钟扫描、非空分钟发布、分页、未知结果恢复 | +| P5 | runner跟踪回调子tasks/旧tasks退出后清state与每轮retry缓存；并发、三次机会、仅日志终结、旧ACK保护 | +| P6 | 服务层纯检查/直接原commit、锁内最新meta局部patch、记录失败仍accepted、冷却/活动重投 | +| P7 | CLI/多worker/legacy回归、Cluster故障注入和容量压测 | + +跨槽旧请求风险应在早期验证，不能到最后才用普通功能测试代替接管安全性检查。 + +## 13. 验收矩阵 + +| 场景 | 预期 | +|---|---| +| 同local_id跨account/user | 身份与回源隔离，绝不提升为root | +| R失败 / R成功M失败 / M过期 | 顺序明确；缺M可回源；无错误回滚R | +| timeout为1小时/12小时/13小时 | TTL分别24/24/26小时 | +| 旧A覆盖新B | 接受覆盖，不出现隐藏CAS或revision状态 | +| report与调度并行、延迟report到达 | 单R TIME+ZADD写未来due；正常时钟下不落入本轮旧范围 | +| 12:34:04.234、边界due±1ms | U=12:34:00；包含等值，晚1ms下轮处理 | +| U=C / 无到期成员 / 空分钟跨度很大 | 跳过，无空桶和空推进，无逐空分钟循环 | +| 分页末项续期/删除、大量同score | 不因OFFSET位移漏扫 | +| 处理大量积压 | 保持租约直到追平，不按页/分钟配额交出shard | +| 两个角色、多实例、轮内递增 | 每角色一次一个shard，锁分开，下一轮补扫跳过项 | +| 20秒续租失败 / 迟到成功 / 长暂停 | lost永久生效，不复活旧lease；revoke/cancel旧process/children，保留未决命令限制 | +| 正常stop grace期间失锁 | heartbeat继续且失锁/有效截止仍被监视，立即撤销，不等grace耗尽 | +| 抑制取消的process/child/renew/acquire/release | asyncio.wait有界join，pending强引用不丢；同role所有shard停止再抢，另一role独立；不宣称强杀 | +| 普通业务异常但任务已终止 | 无forced/unknown、ownership可信可按token释放，错误仍传播；revoked不等于lost | +| forced / 失锁 / 未知结果 | 不主动release，停止续租留TTL；本地取消不证明Redis写已撤回 | +| callback旧task的finally迟到 | 未退出前不清state/reserved、不重领同role；退出后再清，retry缓存每次单shard运行清理 | +| handler吞取消后返回COMPLETED | ensure_active先检查current_task.cancelling，同步unknown并抛CancelledError，不继续ACK，也不主动release | +| run取消及多次取消、远端operation抑制取消 | wrapper保留至真实finished；requests强引用不丢，quiesce超时报shard_not_stopped，依赖仍保留 | +| quiesce后新外部请求 / nested cleanup | 外部run拒绝；run_cleanup raw parent在Redis loop运行，nested io.run可关client，max_inflight=1无gate递归 | +| runtime/core关闭仍有工作 | stop_roles/quiesce未退出保留runtime/roles/pending/requests；shard_not_stopped传播，不提前关依赖 | +| store已关但I/O线程join超时 | _storage_closed=True而_storage_started/storage_ready保留；重试跳过cleanup、不重复stop，只观察join；真实退出才清引用，start不能建第二线程 | +| store cleanup成功后thread重试回归 | test_runtime_retries_thread_join_without_resubmitting_closed_storage_cleanup验证不向stop_requested loop重复提交cleanup | +| worker角色节奏 | 各自遍历0..N-1；scheduler等下一分钟边界，callback按poll，Coordinator无轮次逻辑 | +| publish响应未知 | 不盲目重做可能已发布的B | +| claim attempt=0/1/2 | 不生成第四次调用；最后失败日志后条件清理 | +| 日志输出/ACK故障 | 可重复日志，无Redis失败归档；不伪称日志与删除原子 | +| last_idle_commit_accepted_at记录 | 原commit accepted后外层best-effort patch；无idle归档指针/查证/修复要求 | +| accepted后并发append/config更新 | root exact锁内重读最新raw meta，仅patch接受时间/updated_at，写入传lease_ref=lease | +| 并发成功记录时间乱序或坏旧值 | 有效更大/相等时间不写，较旧调用不回写；坏旧值日志后可覆盖 | +| accepted后记录读取/锁/写失败或session删除 | 失败日志后仍accepted；missing不重建，不从ready恢复记录 | +| 检查通过后新消息到达 | 原commit允许一起提交，无锁内最终idle检查；不承诺严格空闲或零重复 | +| 入口读取/commit异常 | 向外传播供重试，不伪装skipped或accepted | +| 最新last_message_at尚未到期 | 外层deferred/idle_activity，report成功才ACK旧B且保留未来R/M | +| timeout调大、冷却未满足 | report due=检查now+当前timeout，成功后才ACK旧B | +| 重投report失败 | 旧B保留，未耗尽可重试；R成功M失败的部分结果可解释 | +| 无消息/禁用policy | skipped后完成，不更新时间，不产生无限空重投 | +| 新版 commit 参数 None/True/False | HTTP/Service/Session 透传，False 不丢失；非 bool 拒绝；一次性覆盖不持久化 | +| 默认 WM 关闭 / 业务 policy 显式开启 | 沿用原commit，accepted后ACK、不等后台任务；按有效policy决定摘要，保留effective_enable_working_memory | +| 新旧 QueueFS policy 快照、重启前后配置不同 | 新任务保留显式 WM 值；旧任务按 from_dict 兼容规则恢复，不重新套用当前用户 policy | +| 新旧 .done 字段、无 summary 的成功归档 | 原ArchiveStore兼容契约不变；无摘要不影响timer accepted判定，不修改原通用恢复 | +| 多loop/独立CLI/legacy | Redis连接不跨loop误用，CLI不加载业务依赖，原提交路径回归 | + +实施时复用最新版已有回归：`tests/session/test_working_memory_default_off.py`、`tests/unit/session/test_memory_policy.py`、`tests/unit/session/test_session_commit_resume.py`、`tests/session/test_session_commit.py`、`tests/server/test_api_sessions.py`，并在新增 idle 路径中覆盖上述交互。测试执行结论以最终 suite 输出为准，文档矩阵只定义预期。 + +## 14. 性能、容量与证据 + +报告至少一次R操作和一次M操作，通常跨槽两次往返；无V，无每资源更新版本成本。schema字段删除不代表取消实际namespace配置校验。延长M到至少24小时后，按最近一天内上报的去重资源量估算容量，不能按几分钟活跃量估算。 + +调度扫描量与到期数据相关，但每轮逐shard抢锁、读C/时间、空扫和释放存在固定成本。20秒续租约为每个持锁角色每分钟3次；需要测大量逻辑内核/实例同时轮询时的总负载。轮内递增减少回头竞争，不保证严格公平。空范围不推进C后，C年龄不能单独视为积压，应同时看最老到期R成员、G年龄和实际处理延迟。 + +B/G随积压增长；无Redis失败归档容量，但日志吞吐、保留时间和查询能力需要现有日志设施承担。每类型并发限制回调/原commit调用，不限制已接受后的后台任务总积压。批量范围删除、单type大分钟桶、热点shard和持续积压都需要真实Redis Cluster压测。 + +默认关闭 WM 可省去未显式启用时的摘要和 checkpoint 相关工作；长期记忆提取仍可能执行。不能把“没有摘要”等同于“没有后台成本”，也不能以此提高 timer 并发而跳过 QueueFS 积压与模型调用容量评估。本轮未提供新调度器的性能数据。 + +证据分为源码/骨架静态核对、自动化回归、真实 Redis/Lua 集成、Cluster 故障注入和吞吐压测，不能相互替代。静态核对重点为取消检查点、真实任务 finished 桥接、quiesce/cleanup/join 顺序及重试状态。2026-10-11 完整自动化回归为 **235 passed、8条既有 deprecation warnings、40.68s**，包含 store cleanup 成功后的线程 join 重试边界；日志及检验范围见附录第19节。 + +真实 Redis/Lua、Cluster 接管故障和吞吐测试未执行；自动化回归须以最终 suite 输出为准。不宣称已经解决第10.3节的残余风险，也不以文档骨架或模拟任务测试代表真实服务端行为已通过验收。 + +## 15. 工程约束与上线检查 + +以下约束与可靠性限制需在上线检查中保持一致： + +- 上报通过单R Lua校正过去due到源TIME之后，不读取C，不引入关闭水位；任意时钟回退仍不在无状态保证内。 +- C初始为0、非空发布允许跳过空分钟；完整身份及namespace寻址配置需全实例一致。 +- 三次预算按领取占用，耗尽仅日志；log与ACK非事务，允许重复或故障下缺失日志。 +- 冷却重投只有R/M均成功后才完成旧事件；失败不等于已经写入的R被撤销。 +- shard租约按120/20执行；取消本地工作不能撤回已发跨槽请求，自动接管安全性仍需明确接受范围或进一步修改协议。 +- 切换前已静默session是否做一次性导入、具体性能预算和灰度范围仍在上线计划中确认，不恢复周期全量扫描。 + +## 16. 逐文件代码附录 + +[代码骨架](session-idle-auto-commit-scheduler-code-design.md)与本文同步：删除V/更新revision/Redis失败记录/格式版本字段，同步leases.py三操作、shard_runner.py单shard执行、worker自有节奏、consumer子tasks与state、runtime/core未退出不关依赖；保留R→M、空范围跳过和Session外层冷却重投。新增/修改方法列出签名、注释及关键控制流；已有文件未涉及部分保留，省略体只省去实现细节。 diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md index a39025aed7..878befeb92 100644 --- a/docs/zh/guides/01-configuration.md +++ b/docs/zh/guides/01-configuration.md @@ -1722,6 +1722,7 @@ openviking-server --config /path/to/ov.conf "memory": { "session_auto_commit": { "enabled": false, + "idle_backend": "legacy", "check_interval_seconds": 600.0, "scan_rate_limit_files_per_second": 2.0 } @@ -1732,6 +1733,7 @@ openviking-server --config /path/to/ov.conf | 参数 | 类型 | 说明 | 默认值 | |------|------|------|--------| | `enabled` | bool | 自动 commit 总开关。开启后：未显式传入 `auto_commit_policy` 的新 session 会套用默认 policy，同时启动后台 idle 扫描器；关闭后两者都不发生 | `false` | +| `idle_backend` | string | idle 超时发现后端：`legacy` 扫描 session 元数据；`redis` 使用分布式 expiry 调度 | `legacy` | | `check_interval_seconds` | float | 两轮 idle 扫描之间的最小间隔，单位秒，必须大于 `0` | `600.0` | | `scan_rate_limit_files_per_second` | float | idle 扫描时每秒最多读取的 session `.meta.json` 文件数，必须大于 `0`，用于限制大量 session 扫描时的存储 IO 压力 | `2.0` | @@ -1742,22 +1744,56 @@ openviking-server --config /path/to/ov.conf - `enabled=false` 时：既无显式 policy、也无 `server.user_config_defaults.auto_commit_policy` 的新 Session 保持 auto commit 关闭，并返回 `auto_commit_policy: null`；同时不启动 `SessionAutoCommitScheduler`。 - `enabled=true` 时： - 既无显式 policy、也无部署级默认 policy 的新 Session 会带上下方内置 policy。 - - `SessionAutoCommitScheduler` 启动后立即开始一轮扫描，之后每轮结束后至少等待 `check_interval_seconds` 再开始下一轮；若一轮耗时已超过该值，则立即开始下一轮。 - - 扫描过程中以 `scan_rate_limit_files_per_second` 为上限串行读取 session `.meta.json`，避免瞬时 IO 洪峰。 - - 不会做单独的启动恢复扫描，idle 检查只发生在周期扫描时。 + - `idle_backend="legacy"` 时，`SessionAutoCommitScheduler` 启动后立即开始扫描；扫描过程以 `scan_rate_limit_files_per_second` 为上限串行读取 session `.meta.json`，每轮结束后至少等待 `check_interval_seconds`。 + - `idle_backend="redis"` 时，服务通过顶层 `expiry` 配置整体启用 expiry 上报、发布和 callback 消费。 - token 和 message-count 自动触发在消息写入后内联执行，不依赖 scheduler，但同样以 session 是否带有 `auto_commit_policy` 为前提。 +#### 分布式 Expiry 配置 + +共享同一份 expiry 状态的 API、publisher 和 callback 实例必须使用相同且稳定的 `service_name`。它表示部署的文档服务名，不是单个 Pod 名。 + +```json +{ + "expiry": { + "service_name": "openviking-documents", + "publisher_separate_process": true, + "shard_count": 128, + "session_consumer_concurrency": 8, + "redis": { + "mode": "cluster", + "endpoints": ["redis://redis-proxy.example.com:6379"], + "username": "openviking", + "password_env": "OPENVIKING_EXPIRY_REDIS_PASSWORD" + } + } +} +``` + +| 参数 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `service_name` | string | 使用同一份 expiry 数据的所有实例共享的稳定文档服务名 | `""` | +| `publisher_separate_process` | bool | 由 `openviking-server` 在独立子进程中托管 publisher；为 `false` 时 publisher 在各 API 进程内运行 | `true` | +| `shard_count` | int | expiry 逻辑分片数；所有共享同一 namespace 的实例必须一致，取值范围为 `1..512` | `128` | +| `session_consumer_concurrency` | int | 每个 API 进程中 Session expiry callback 的并发数，必须大于 `0` | `8` | +| `redis.mode` | string | Redis 部署模式：`standalone` 或 `cluster` | `standalone` | +| `redis.endpoints` | string array | Redis 地址；集群模式可以只配置一个统一 proxy 地址 | `[]` | +| `redis.username` | string | Redis ACL 用户名 | `""` | +| `redis.password_env` | string | 保存 Redis 密码的环境变量名；变量不存在时启动失败 | `""` | +| `redis.password` | string | 兼容用明文密码；建议使用 `password_env`，两者互斥 | `""` | + +启用 Redis idle auto commit 时必须配置 `service_name` 和 Redis 地址。用户名与密码只能使用上述独立字段，不能写入 endpoint URI。默认 `publisher_separate_process=true` 时，`openviking-server` 会在开始服务前拉起一个内部 publisher 子进程，在服务关闭时停止它，并在子进程异常退出时一并退出。publisher 不再暴露为公共 CLI 命令。 + ##### 单 session 自动 commit 策略 -当 session 带有 `auto_commit_policy` 时，未传的字段会回退到下方内置默认值。没有存储 policy 的 session 保持 auto commit 关闭。取值会被 clamp 到 `[0, 上限]`，未知字段会以 `InvalidArgumentError` 拒绝。设置和查看方式见 [Sessions API](../api/05-sessions.md#create-session)。 +当 session 带有 `auto_commit_policy` 时，未传的字段会回退到下方内置默认值。没有存储 policy 的 session 保持 auto commit 关闭。除 idle 超时的写入下限外，取值会被 clamp 到 `[0, 上限]`；未知字段会以 `InvalidArgumentError` 拒绝。设置和查看方式见 [Sessions API](../api/05-sessions.md#create-session)。 -| 字段 | 类型 | 默认值 | 上限 | 说明 | -|------|------|--------|------|------| -| `pending_token_threshold` | int | 150000 | 1000000 | 当未提交的 pending token 超过该值（严格大于）时，会在消息写入后触发一次自动 commit。 | -| `message_count_threshold` | int | 100 | 1000 | 当未提交的 live message 数量超过该值（严格大于）时，会在消息写入后触发一次自动 commit。 | -| `idle_timeout_seconds` | int | 86400 | 604800 | 有未提交内容的 session 在空闲这么多秒后，进入服务端 idle scheduler 的处理范围。idle 触发的 commit 会归档全部积压消息，并忽略 `keep_recent_count`。 | -| `keep_recent_count` | int | 0 | 500 | 阈值触发的自动 commit 后保留（不归档）的最近 live message 数量。idle 超时触发的 commit 会忽略该值并归档所有消息。 | -| `min_commit_interval_seconds` | int | 0 | 604800 | 两次自动 commit 之间的最小间隔秒数（节流）。 | +| 字段 | 类型 | 默认值 | 最小值 | 上限 | 说明 | +|------|------|--------|--------|------|------| +| `pending_token_threshold` | int | 150000 | 0 | 1000000 | 当未提交的 pending token 超过该值（严格大于）时，会在消息写入后触发一次自动 commit。 | +| `message_count_threshold` | int | 100 | 0 | 1000 | 当未提交的 live message 数量超过该值（严格大于）时，会在消息写入后触发一次自动 commit。 | +| `idle_timeout_seconds` | int | 86400 | 0（关闭）或 300 | 604800 | 新传入的正数最短为 5 分钟；存量的小于 300 的正值继续按原值读取。idle 触发的 commit 会归档全部积压消息，并忽略 `keep_recent_count`。 | +| `keep_recent_count` | int | 0 | 0 | 500 | 阈值触发的自动 commit 后保留（不归档）的最近 live message 数量。idle 超时触发的 commit 会忽略该值并归档所有消息。 | +| `min_commit_interval_seconds` | int | 0 | 0 | 604800 | 两次自动 commit 之间的最小间隔秒数（节流）。 | 代码入口：`openviking/session/auto_commit_policy.py:AutoCommitPolicy`。 diff --git a/openviking/scheduling/expiry/api.py b/openviking/scheduling/expiry/api.py index 13ffed0fbe..4b54cdf4ff 100644 --- a/openviking/scheduling/expiry/api.py +++ b/openviking/scheduling/expiry/api.py @@ -65,6 +65,7 @@ class CallbackResult: class HandlerOptions: concurrency: int = 4 max_attempts: int = 3 + metadata_enabled: bool = True class ExpiryHandler(Protocol): diff --git a/openviking/scheduling/expiry/config.py b/openviking/scheduling/expiry/config.py index 7ee2e084ea..cb27eabc5c 100644 --- a/openviking/scheduling/expiry/config.py +++ b/openviking/scheduling/expiry/config.py @@ -4,6 +4,7 @@ from __future__ import annotations +import os from typing import Literal from urllib.parse import urlsplit @@ -11,13 +12,13 @@ class ExpiryRedisConfig(BaseModel): - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="forbid", hide_input_in_errors=True) mode: Literal["standalone", "cluster"] = "standalone" endpoints: list[str] = Field(default_factory=list) - username: str | None = None - password: str | None = None - password_env: str | None = None + username: str = "" + password_env: str = "" + password: str = Field(default="", repr=False) tls: bool = False connect_timeout_seconds: float = Field(default=1.0, gt=0) socket_timeout_seconds: float = Field(default=2.0, gt=0) @@ -27,7 +28,7 @@ class ExpiryRedisConfig(BaseModel): @model_validator(mode="after") def validate_connection(self) -> "ExpiryRedisConfig": - if self.password is not None and self.password_env is not None: + if self.password and self.password_env: raise ValueError("Redis password and password_env are mutually exclusive") if self.mode == "standalone" and len(self.endpoints) > 1: raise ValueError("standalone Redis accepts at most one endpoint") @@ -38,25 +39,48 @@ def validate_connection(self) -> "ExpiryRedisConfig": if "://" in endpoint else (("rediss" if self.tls else "redis") + f"://{endpoint}") ) - parsed = urlsplit(normalized) - if parsed.scheme not in {"redis", "rediss"} or not parsed.hostname or not parsed.port: - raise ValueError(f"invalid Redis endpoint: {endpoint}") - if parsed.username or parsed.password or parsed.path not in {"", "/"}: - raise ValueError("Redis endpoints must not contain credentials or paths") + try: + parsed = urlsplit(normalized) + valid_address = bool(parsed.hostname and parsed.port) + except ValueError: + # urlsplit/.port errors can echo untrusted endpoint fragments. + raise ValueError("invalid Redis endpoint") from None + if parsed.scheme not in {"redis", "rediss"} or not valid_address: + raise ValueError("invalid Redis endpoint") + if ( + parsed.username is not None + or parsed.password is not None + or parsed.path not in {"", "/"} + or parsed.query + or parsed.fragment + ): + raise ValueError( + "Redis endpoints must not contain credentials, paths, " + "query parameters, or fragments; use dedicated Redis fields" + ) schemes.add(parsed.scheme) if len(schemes) > 1: raise ValueError("Redis cluster endpoints must use one transport scheme") return self + def resolve_password(self) -> str | None: + if self.password_env.strip(): + try: + return os.environ[self.password_env] + except KeyError as exc: + raise ValueError( + f"Redis secret environment variable {self.password_env} is not set" + ) from exc + return self.password if self.password.strip() else None + class ExpiryConfig(BaseModel): - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="forbid", hide_input_in_errors=True) service_name: str = "" - reporting_enabled: bool = False - scheduler_enabled: bool = False - callback_enabled: bool = False - shard_count: Literal[512] = 512 + publisher_separate_process: bool = True + shard_count: int = Field(default=128, ge=1, le=512) + session_consumer_concurrency: int = Field(default=8, ge=1) precision_seconds: Literal[60] = 60 owner_ttl_seconds: Literal[120] = 120 owner_renew_interval_seconds: Literal[20] = 20 @@ -68,19 +92,16 @@ class ExpiryConfig(BaseModel): shutdown_timeout_seconds: float = Field(default=30.0, gt=0) redis: ExpiryRedisConfig = Field(default_factory=ExpiryRedisConfig) - @property - def enabled(self) -> bool: - return self.reporting_enabled or self.scheduler_enabled or self.callback_enabled - - @model_validator(mode="after") - def validate_roles(self) -> "ExpiryConfig": - if self.enabled: - if not self.service_name.strip(): - raise ValueError("expiry.service_name is required when enabled") - if not self.redis.endpoints: - raise ValueError("expiry.redis.endpoints is required when enabled") - if self.redis.request_budget_seconds >= self.owner_renew_interval_seconds: - raise ValueError("Redis request budget must be shorter than owner renew interval") - if self.callback_enabled and not self.reporting_enabled: - raise ValueError("callback_enabled requires reporting_enabled for deferred reports") - return self + def validate_for_runtime(self) -> None: + if not self.service_name.strip(): + raise ValueError( + "expiry.service_name is required when Redis session auto commit is enabled" + ) + if not self.redis.endpoints: + raise ValueError( + "expiry.redis.endpoints is required when Redis session auto commit is enabled" + ) + if self.redis.mode == "standalone" and len(self.redis.endpoints) != 1: + raise ValueError("standalone Redis requires exactly one endpoint") + if self.redis.request_budget_seconds >= self.owner_renew_interval_seconds: + raise ValueError("Redis request budget must be shorter than owner renew interval") diff --git a/openviking/scheduling/expiry/consumer.py b/openviking/scheduling/expiry/consumer.py index 3a630acd33..934ed1e3c6 100644 --- a/openviking/scheduling/expiry/consumer.py +++ b/openviking/scheduling/expiry/consumer.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio +import logging import secrets from dataclasses import dataclass, field from typing import Awaitable, Callable @@ -25,6 +26,8 @@ from .registry import HandlerBinding from .shard_runner import ShardExecution, ShardRunner +logger = logging.getLogger(__name__) + @dataclass class _ConsumerState: @@ -56,7 +59,7 @@ def __init__( self._binding = binding self._config = config self._metrics = metrics - self._state = _ConsumerState() + self._callback_tasks: set[asyncio.Task[None]] = set() self._next_shard = ( secrets.randbelow(config.shard_count) if start_shard is None else start_shard ) @@ -79,7 +82,18 @@ async def run(self, stop: asyncio.Event) -> None: ) except StoreOperationError as exc: if exc.code == "shard_not_stopped": - raise + logger.warning( + "expiry consumer skipped unfinished shard execution " + "item_type=%s shard_id=%d", + self._item_type, + shard_id, + ) + self._metrics.count( + "timer_shard_round", + item_type=self._item_type, + outcome="orphaned", + ) + continue self._metrics.count( "timer_shard_round", item_type=self._item_type, @@ -99,16 +113,11 @@ async def run(self, stop: asyncio.Event) -> None: pass async def process_shard(self, lease: ShardExecution, stop: asyncio.Event) -> None: - self._reset_state() - try: - bucket = await self._discover_bucket(lease) - if bucket is None: - return - await self._process_bucket(lease, stop, bucket) - finally: - # The runner retains children which have not actually stopped. - if not any(not task.done() for task in self._state.tasks): - self._reset_state() + state = _ConsumerState() + bucket = await self._discover_bucket(lease) + if bucket is None: + return + await self._process_bucket(lease, stop, bucket, state) async def _discover_bucket(self, lease: ShardExecution) -> BucketRef | None: cursor = None @@ -131,10 +140,11 @@ async def _process_bucket( lease: ShardExecution, stop: asyncio.Event, bucket: BucketRef, + state: _ConsumerState, ) -> None: while not stop.is_set(): lease.ensure_active() - available = self._binding.options.concurrency - self._state.reserved + available = self._binding.options.concurrency - state.reserved scan = _BucketScan() redis_now_ms = 0 if available > 0: @@ -145,9 +155,10 @@ async def _process_bucket( bucket, redis_now_ms=redis_now_ms, available=available, + state=state, ) - active = {task for task in self._state.tasks if not task.done()} + active = {task for task in state.tasks if not task.done()} if active: await lease.wait(self._config.callback_poll_seconds, stop) continue @@ -168,7 +179,7 @@ async def _process_bucket( ) return - tasks = {task for task in self._state.tasks if not task.done()} + tasks = {task for task in state.tasks if not task.done()} if tasks: await asyncio.wait(tasks) lease.ensure_active() @@ -180,6 +191,7 @@ async def _claim_ready_items( *, redis_now_ms: int, available: int, + state: _ConsumerState, ) -> _BucketScan: next_poll_at_ms: int | None = None started_at_seconds = redis_now_ms // 1000 @@ -216,7 +228,7 @@ async def _claim_ready_items( lease.ensure_active() if claim is None: return _BucketScan(next_poll_at_ms, conflicted=True) - self._start_callback(lease, claim) + self._start_callback(lease, claim, state) available -= 1 if available == 0: break @@ -239,27 +251,36 @@ def _ready_at_ms(self, item: BucketItem, expected_attempt: int) -> int | None: ) return (started_at_seconds + self._config.processing_timeout_seconds) * 1000 - def _start_callback(self, lease: ShardExecution, claim: Claim) -> None: - task = lease.spawn(self._callback_operation(lease, claim)) - self._state.reserved += 1 - self._state.tasks.add(task) - task.add_done_callback(self._state.tasks.discard) - - def _reset_state(self) -> None: - if any(not task.done() for task in self._state.tasks): - raise StoreOperationError("shard_not_stopped", "Previous callbacks are still running") - self._state.tasks.clear() - self._state.reserved = 0 + def _start_callback( + self, + lease: ShardExecution, + claim: Claim, + state: _ConsumerState, + ) -> None: + task = lease.spawn(self._callback_operation(lease, claim, state)) + state.reserved += 1 + state.tasks.add(task) + self._callback_tasks.add(task) + task.add_done_callback(state.tasks.discard) + task.add_done_callback(self._callback_tasks.discard) def _callback_operation( - self, lease: ShardExecution, claim: Claim + self, + lease: ShardExecution, + claim: Claim, + state: _ConsumerState, ) -> Callable[[], Awaitable[None]]: async def operation() -> None: - await self._execute(lease, claim) + await self._execute(lease, claim, state) return operation - async def _execute(self, lease: ShardExecution, claim: Claim) -> None: + async def _execute( + self, + lease: ShardExecution, + claim: Claim, + state: _ConsumerState, + ) -> None: try: try: lease.ensure_active() @@ -296,7 +317,7 @@ async def _execute(self, lease: ShardExecution, claim: Claim) -> None: else: await self._finish_failed_attempt(lease, claim, "callback_retry") finally: - self._state.reserved = max(0, self._state.reserved - 1) + state.reserved = max(0, state.reserved - 1) async def _finish_failed_attempt( self, lease: ShardExecution, claim: Claim, reason: str @@ -341,6 +362,8 @@ async def _resolve_request(self, claim: Claim) -> CallbackRequest: identity = decode_item_id(claim.item_id) if identity.item_type != claim.bucket.item_type: raise ExpiryError("invalid_identity", "Bucket type differs from item ID") + if not self._binding.options.metadata_enabled: + return CallbackRequest(identity) metadata = await self._store.read_metadata(claim.item_id) if metadata is not None and metadata.identity != identity: raise ExpiryError("invalid_identity", "Metadata differs from item ID") @@ -352,7 +375,7 @@ async def _resolve_request(self, claim: Claim) -> CallbackRequest: return CallbackRequest(identity) async def drain(self, timeout_seconds: float) -> None: - tasks = {task for task in self._state.tasks if not task.done()} + tasks = {task for task in self._callback_tasks if not task.done()} if not tasks: return _, pending = await asyncio.wait(tasks, timeout=max(0.0, timeout_seconds)) diff --git a/openviking/scheduling/expiry/keys.py b/openviking/scheduling/expiry/keys.py index 12157ec59c..dbd685ee41 100644 --- a/openviking/scheduling/expiry/keys.py +++ b/openviking/scheduling/expiry/keys.py @@ -1,6 +1,6 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -"""Redis key construction and reversible timer identities.""" +"""Redis key construction and reversible expiry identities.""" from __future__ import annotations @@ -72,36 +72,36 @@ class ExpiryKeys: service_key: str def namespace(self) -> str: - return f"resource_timer:namespace:{self.service_key}" + return f"expiry:namespace:{self.service_key}" def metadata(self, item_id: str) -> str: - return f"resource_meta:item:{self.service_key}:{item_id}" + return f"expiry:metadata:{self.service_key}:{item_id}" def source(self, shard_id: int) -> str: - return f"resource_sche:item_list:{self.service_key}:{shard_id}" + return f"expiry:publisher:source:{self.service_key}:{shard_id}" def control_tag(self, shard_id: int) -> str: return f"{{{self.service_key}:{shard_id}}}" def progress(self, shard_id: int) -> str: - return f"resource_sche:process:consume_progress:{self.control_tag(shard_id)}" + return f"expiry:publisher:progress:{self.control_tag(shard_id)}" def registry(self, shard_id: int) -> str: - return f"resource_consume:bucket_register:{self.control_tag(shard_id)}" + return f"expiry:consumer:registry:{self.control_tag(shard_id)}" def owner(self, role: OwnerRole, shard_id: int, *, item_type: str | None = None) -> str: if role is OwnerRole.SCHEDULER: if item_type is not None: raise ValueError("Scheduler owner must not include an item type") - return f"resource_timer:owner:{role.value}:{self.control_tag(shard_id)}" + return f"expiry:publisher:owner:{self.control_tag(shard_id)}" if item_type is None: raise ValueError("Callback owner requires an item type") validate_item_type(item_type) - return f"resource_timer:owner:{role.value}:{item_type}:{self.control_tag(shard_id)}" + return f"expiry:consumer:owner:{item_type}:{self.control_tag(shard_id)}" def bucket(self, ref: BucketRef) -> str: return ( - f"resource_consume:item_list:{self.service_key}:" + f"expiry:consumer:bucket:{self.service_key}:" f"{ref.item_type}:{ref.shard_id}:{ref.minute_ms}" ) diff --git a/openviking/scheduling/expiry/redis_store.py b/openviking/scheduling/expiry/redis_store.py index 80e34b0e3f..b9fc415efd 100644 --- a/openviking/scheduling/expiry/redis_store.py +++ b/openviking/scheduling/expiry/redis_store.py @@ -7,7 +7,6 @@ import asyncio import hashlib import json -import os import time from importlib import resources from typing import Any, Literal, Sequence, cast @@ -144,9 +143,20 @@ async def upsert_resource( ) return ExpiryRegistrationResult(int(raw)) - async def unregister_resource(self, shard_id: int, item_id: str) -> None: + async def unregister_resource( + self, + shard_id: int, + item_id: str, + *, + delete_metadata: bool = True, + ) -> None: await self._command("zrem", self._keys.source(shard_id), item_id, retry="idempotent") - await self._command("delete", self._keys.metadata(item_id), retry="idempotent") + if delete_metadata: + await self._single_key_command( + "DEL", + self._keys.metadata(item_id), + retry="idempotent", + ) async def read_progress(self, shard_id: int) -> int: raw = await self._command("get", self._keys.progress(shard_id)) @@ -204,11 +214,10 @@ async def reset_unpublished_minute( self._keys.bucket(BucketRef(item_type, owner.shard_id, minute_ms)) for item_type in item_types ] - # Bucket keys deliberately have no shared cluster hash tag. Delete - # them one at a time so Redis Cluster never rejects this reset with a - # CROSSSLOT error. + # Bucket keys deliberately have no shared cluster hash tag. Delete them + # one at a time, bypassing redis-py's cross-slot pipeline helper. for key in keys: - await self._command("unlink", key, retry="idempotent") + await self._single_key_command("UNLINK", key, retry="idempotent") async def stage_items( self, owner: OwnerToken, bucket: BucketRef, item_ids: Sequence[str] @@ -598,11 +607,9 @@ async def _open_on_loop(self) -> None: endpoint = self._config.endpoints[0] if "://" not in endpoint: endpoint = ("rediss" if self._config.tls else "redis") + "://" + endpoint - password = self._config.password - if password is None and self._config.password_env: - password = os.environ.get(self._config.password_env) - kwargs = { - "username": self._config.username, + password = self._config.resolve_password() + kwargs: dict[str, Any] = { + "username": self._config.username or None, "password": password, "decode_responses": True, "socket_connect_timeout": self._config.connect_timeout_seconds, @@ -625,7 +632,16 @@ async def _open_on_loop(self) -> None: parsed = urlsplit(normalized) use_tls = parsed.scheme == "rediss" nodes.append(ClusterNode(parsed.hostname or "", parsed.port or 6379)) - self._client = RedisCluster(startup_nodes=nodes, ssl=use_tls, **kwargs) + # Store retry policies must be the only authority for replaying + # commands: a lost CAS/SET NX response cannot safely be replayed. + # These instance-local flags leave MOVED/ASK routing intact. + self._client = RedisCluster( + startup_nodes=nodes, + ssl=use_tls, + cluster_error_retry_attempts=0, + connection_error_retry_attempts=0, + **kwargs, + ) try: await self._client.ping() except Exception as exc: @@ -633,13 +649,16 @@ async def _open_on_loop(self) -> None: raise self._translate_redis_error(exc, retry="read") from exc async def _close_on_loop(self) -> None: - client, self._client = self._client, None + client = self._client if client is not None: close = getattr(client, "aclose", None) or getattr(client, "close", None) if close is not None: result = close() if result is not None: await result + # Cancellation/failure is not proof of a closed client. Keep the + # reference until cleanup succeeds so partial startup can retry. + self._client = None async def _command( self, name: str, *args: object, retry: RetryPolicy = "read", **kwargs: Any @@ -654,6 +673,17 @@ async def operation() -> Any: return await self._io.run(operation) + async def _single_key_command( + self, + command: str, + key: str, + *, + retry: RetryPolicy, + ) -> Any: + # redis-py 5.3.1 routes DEL/UNLINK helpers through ClusterPipeline, + # which skips execution when cluster_error_retry_attempts is zero. + return await self._command("execute_command", command, key, retry=retry) + async def _eval( self, script: str, diff --git a/openviking/scheduling/expiry/registry.py b/openviking/scheduling/expiry/registry.py index 23af7e965f..2b70d71511 100644 --- a/openviking/scheduling/expiry/registry.py +++ b/openviking/scheduling/expiry/registry.py @@ -39,10 +39,15 @@ def register(self, item_type: str, handler: ExpiryHandler, options: HandlerOptio raise ExpiryError("invalid_handler", "Handler concurrency must be positive") if type(options.max_attempts) is not int or not 1 <= options.max_attempts <= 256: raise ExpiryError("invalid_handler", "Handler max_attempts must be in [1, 256]") + if type(options.metadata_enabled) is not bool: + raise ExpiryError("invalid_handler", "Handler metadata_enabled must be a boolean") if item_type in self._bindings: raise ExpiryError("duplicate_handler", f"Handler already registered: {item_type}") self._bindings[item_type] = HandlerBinding(handler, options) + def get(self, item_type: str) -> HandlerBinding | None: + return self._bindings.get(item_type) + def freeze(self) -> Mapping[str, HandlerBinding]: self._frozen = True return MappingProxyType(dict(self._bindings)) diff --git a/openviking/scheduling/expiry/runtime.py b/openviking/scheduling/expiry/runtime.py index 2ecd5cbd11..6e4e9a81c6 100644 --- a/openviking/scheduling/expiry/runtime.py +++ b/openviking/scheduling/expiry/runtime.py @@ -6,6 +6,7 @@ import asyncio import concurrent.futures +import logging import threading import time from dataclasses import dataclass, field @@ -25,6 +26,18 @@ from .shard_runner import ShardRunner T = TypeVar("T") +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ExpiryRuntimeRoles: + report: bool = False + publish: bool = False + consume: bool = False + + def __post_init__(self) -> None: + if self.consume and not self.report: + raise ValueError("Expiry consumer requires reporting for deferred callbacks") @dataclass(eq=False) @@ -63,7 +76,8 @@ async def start(self) -> None: ready = await asyncio.to_thread(self._ready.wait, 5.0) if not ready or self._loop is None: raise RuntimeError("Expiry Redis loop failed to start") - self._accepting = True + with self._requests_lock: + self._accepting = True async def run(self, operation: Callable[[], Awaitable[T]]) -> T: loop = self._loop @@ -71,8 +85,6 @@ async def run(self, operation: Callable[[], Awaitable[T]]) -> T: # their tracked parent request, and must be allowed after quiescing. if loop is not None and asyncio.get_running_loop() is loop: return await self._execute(operation) - if not self._accepting or loop is None: - raise StoreOperationError("not_started", "Redis expiry I/O loop is not running") request = self._submit(operation) finished = asyncio.wrap_future(request.finished) try: @@ -86,8 +98,8 @@ async def run(self, operation: Callable[[], Awaitable[T]]) -> T: async def quiesce(self, timeout_seconds: float) -> None: """Reject new work and observe actual remote termination before closing Redis.""" - self._accepting = False with self._requests_lock: + self._accepting = False requests = list(self._requests) if requests: futures = {asyncio.wrap_future(request.finished) for request in requests} @@ -121,6 +133,11 @@ async def close(self, timeout_seconds: float) -> None: deadline = time.monotonic() + max(0.0, timeout_seconds) await self.quiesce(timeout_seconds) loop, thread = self._loop, self._thread + if loop is None and thread is not None and thread.is_alive(): + # start() may have been cancelled before the new thread published + # its loop. Wait only within the close budget, then stop that loop. + await asyncio.to_thread(self._ready.wait, max(0.0, deadline - time.monotonic())) + loop = self._loop if loop is not None and not self._stop_requested: self._stop_requested = True loop.call_soon_threadsafe(loop.stop) @@ -137,31 +154,30 @@ async def close(self, timeout_seconds: float) -> None: def _submit( self, operation: Callable[[], Awaitable[T]], *, cleanup: bool = False ) -> _RedisRequest: - loop = self._loop - if loop is None or self._stop_requested: - raise StoreOperationError("not_started", "Redis expiry I/O loop is not running") - request = _RedisRequest() - # Register before scheduling: shutdown must also see not-yet-started I/O. with self._requests_lock: + loop = self._loop + if (not cleanup and not self._accepting) or loop is None or self._stop_requested: + raise StoreOperationError("not_started", "Redis expiry I/O loop is not running") + request = _RedisRequest() + # Register before scheduling: shutdown must also see not-yet-started I/O. self._requests.add(request) - async def cleanup_operation() -> T: - return await operation() + async def cleanup_operation() -> T: + return await operation() - def start_request() -> None: - request.task = asyncio.create_task( - cleanup_operation() if cleanup else self._execute(operation) - ) - request.task.add_done_callback(lambda task: self._request_done(request, task)) - if request.cancelled.is_set(): - request.task.cancel() + def start_request() -> None: + request.task = asyncio.create_task( + cleanup_operation() if cleanup else self._execute(operation) + ) + request.task.add_done_callback(lambda task: self._request_done(request, task)) + if request.cancelled.is_set(): + request.task.cancel() - try: - loop.call_soon_threadsafe(start_request) - except BaseException: - with self._requests_lock: + try: + loop.call_soon_threadsafe(start_request) + except BaseException: self._requests.discard(request) - raise + raise return request def _cancel(self, request: _RedisRequest) -> None: @@ -262,9 +278,11 @@ def __init__( self, config: ExpiryConfig, *, + roles: ExpiryRuntimeRoles, metrics: ExpiryMetrics | None = None, ) -> None: self._config = config + self._roles = roles self._metrics = metrics or ExpiryMetrics() self._keys = ExpiryKeys(encode_component(config.service_name)) self._io = RedisLoopExecutor( @@ -279,46 +297,85 @@ def __init__( self._registry, self._state, shard_count=config.shard_count, - reporting_enabled=config.reporting_enabled, + reporting_enabled=roles.report, metrics=self._metrics, ) self._leases = ShardLeaseCoordinator(self._store, config, self._metrics) self._shard_runner = ShardRunner(self._leases, config, self._metrics) - self._scheduler: ExpiryPublisher | None = None + self._publisher: ExpiryPublisher | None = None self._consumers: dict[str, ExpiryConsumer] = {} self._tasks: dict[str, asyncio.Task[None]] = {} self._stop = asyncio.Event() self._storage_started = False self._storage_closed = False + # Readiness is not ownership: cancellation can leave resources behind + # before start_storage has completed. + self._storage_owned = False + self._store_open_attempted = False + self._storage_start_task: asyncio.Task[None] | None = None @property - def timer(self) -> ExpiryScheduler: + def scheduler(self) -> ExpiryScheduler: return self._facade async def start_storage(self) -> None: if self._storage_started: return - await self._io.start() - try: - await self._store.open() - await self._store.ensure_namespace( - NamespaceSpec( - self._config.shard_count, - HASH_NAME, - HASH_SEED, - IDENTITY_ENCODING, - ) + if self._storage_owned or self._storage_start_task is not None: + raise StoreOperationError( + "shard_not_stopped", "Previous expiry storage startup has not been cleaned up" ) - except Exception: + self._storage_owned = True + self._storage_closed = False + # Keep the actual startup operation visible if it resists cancellation. + # The caller's cancellation must not make close assume it has stopped. + startup = asyncio.create_task(self._initialize_storage()) + startup.add_done_callback(RedisLoopExecutor._consume_exception) + self._storage_start_task = startup + try: + await asyncio.shield(startup) + except BaseException: + if not startup.done(): + startup.cancel() try: - await self._store.close() - finally: - await self._io.close(self._config.shutdown_timeout_seconds) + await self.close() + except BaseException: + # Preserve the original failure/cancellation. Ownership and the + # startup task remain available for a later close retry. + logger.warning("Expiry partial startup cleanup did not complete", exc_info=True) raise + self._storage_start_task = None self._storage_started = True - self._storage_closed = False self._state.storage_ready = True + async def _initialize_storage(self) -> None: + await self._io.start() + self._store_open_attempted = True + await self._store.open() + await self._store.ensure_namespace( + NamespaceSpec( + self._config.shard_count, + HASH_NAME, + HASH_SEED, + IDENTITY_ENCODING, + ) + ) + + async def _settle_storage_start(self) -> None: + startup = self._storage_start_task + if startup is None: + return + if not startup.done(): + startup.cancel() + done, _ = await asyncio.wait({startup}, timeout=self._config.shutdown_timeout_seconds) + if startup not in done: + raise StoreOperationError( + "shard_not_stopped", "Expiry storage startup is still running" + ) + if not startup.cancelled(): + startup.exception() + self._storage_start_task = None + async def start_roles(self) -> None: if not self._storage_started: raise RuntimeError("Expiry storage is not started") @@ -326,12 +383,12 @@ async def start_roles(self) -> None: return bindings = self._registry.freeze() self._stop.clear() - if self._config.scheduler_enabled: - self._scheduler = ExpiryPublisher( + if self._roles.publish: + self._publisher = ExpiryPublisher( self._store, self._shard_runner, self._config, self._metrics ) - self._spawn_role("scheduler", self._scheduler.run(self._stop)) - if self._config.callback_enabled: + self._spawn_role("publisher", self._publisher.run(self._stop)) + if self._roles.consume: for item_type, binding in bindings.items(): consumer = ExpiryConsumer( self._store, @@ -379,19 +436,23 @@ async def stop_roles(self) -> None: async def close(self) -> None: await self.stop_roles() - if not self._storage_started: + if not self._storage_owned: return + await self._settle_storage_start() # Stop admission and check real Redis-loop tasks, not just their # application-loop wrappers, before closing any dependency. await self._io.quiesce(self._config.shutdown_timeout_seconds) - try: - if not self._storage_closed: + if not self._storage_closed: + if self._store_open_attempted: await self._io.run_cleanup(self._store.close, self._config.shutdown_timeout_seconds) - self._storage_closed = True - finally: - await self._io.close(self._config.shutdown_timeout_seconds) - self._storage_started = False - self._state.storage_ready = False + self._storage_closed = True + # Keep ownership if client cleanup or thread termination failed. Once + # client cleanup succeeds, retries only observe the I/O thread join. + await self._io.close(self._config.shutdown_timeout_seconds) + self._storage_owned = False + self._store_open_attempted = False + self._storage_started = False + self._state.storage_ready = False def _spawn_role(self, name: str, operation: Coroutine[Any, Any, None]) -> None: task: asyncio.Task[None] = asyncio.create_task(operation, name=f"expiry-{name}") diff --git a/openviking/scheduling/expiry/service.py b/openviking/scheduling/expiry/service.py index 6bff8cf626..4bb264c5bf 100644 --- a/openviking/scheduling/expiry/service.py +++ b/openviking/scheduling/expiry/service.py @@ -56,7 +56,9 @@ async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: item_id = encode_item_id(task.identity) shard_id = shard_for(item_id, self._shard_count) result = await self._store.upsert_resource(shard_id, item_id, task) - await self._store.ensure_metadata(item_id, task) + binding = self._registry.get(task.identity.item_type) + if binding is None or binding.options.metadata_enabled: + await self._store.ensure_metadata(item_id, task) except StoreOperationError as exc: raise self._business_error(exc) from exc self._metrics.count("timer_report", item_type=task.identity.item_type, outcome="written") @@ -68,7 +70,12 @@ async def unregister(self, identity: ItemIdentity) -> None: self._ensure_supported_type(identity.item_type) item_id = encode_item_id(identity) try: - await self._store.unregister_resource(shard_for(item_id, self._shard_count), item_id) + binding = self._registry.get(identity.item_type) + await self._store.unregister_resource( + shard_for(item_id, self._shard_count), + item_id, + delete_metadata=binding is None or binding.options.metadata_enabled, + ) except StoreOperationError as exc: raise self._business_error(exc) from exc diff --git a/openviking/scheduling/expiry/shard_runner.py b/openviking/scheduling/expiry/shard_runner.py index 05ca3d6777..ae26e756b9 100644 --- a/openviking/scheduling/expiry/shard_runner.py +++ b/openviking/scheduling/expiry/shard_runner.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio +import logging import time from typing import Awaitable, Callable @@ -20,7 +21,9 @@ ) ShardProcessor = Callable[["ShardExecution", asyncio.Event], Awaitable[None]] -RunnerScope = tuple[OwnerRole, str | None] +RunnerScope = tuple[OwnerRole, str | None, int] + +logger = logging.getLogger(__name__) class ShardExecution: @@ -120,7 +123,7 @@ def _consume_exception(task: asyncio.Task[object]) -> None: class ShardRunner: - """Run one shard at a time per role, retaining all work that cannot stop.""" + """Run isolated shard executions while retaining work that cannot stop.""" def __init__( self, @@ -154,8 +157,8 @@ async def run_if_acquired( *, item_type: str | None = None, ) -> bool: - scope = (role, item_type) - if scope in self._active or any(not task.done() for task in self._pending.get(scope, ())): + scope = (role, item_type, shard_id) + if scope in self._active: raise self._not_stopped(scope) if stop.is_set(): return False @@ -256,11 +259,11 @@ async def guarded_process() -> None: for task in local_tasks: if not task.done(): task.cancel() - await self._settle(scope, local_tasks) + await self._settle(scope, local_tasks, allow_orphans=True) # task.done() is terminal even if the task raised an ordinary exception. # Read the exception before deciding release, then propagate it afterwards. - if not processor.cancelled(): + if processor.done() and not processor.cancelled(): error = error or processor.exception() error = error or execution._error unknown = execution._outcome_unknown or ( @@ -298,7 +301,13 @@ async def _heartbeat(self, lease: ShardLease, renewing: asyncio.Event) -> None: finally: renewing.clear() - async def _settle(self, scope: RunnerScope, tasks: set[asyncio.Task]) -> None: + async def _settle( + self, + scope: RunnerScope, + tasks: set[asyncio.Task], + *, + allow_orphans: bool = False, + ) -> None: """Never couple the deadline to completion of a cancelled task.""" if not tasks: return @@ -329,8 +338,24 @@ def pending_done( retained.discard(task) if not retained and self._pending.get(scope) is retained: self._pending.pop(scope) - if any(not task.done() for task in tasks): - self._metrics.count("timer_shard_stop", outcome="pending") + orphaned = sum(not task.done() for task in tasks) + if orphaned: + role, item_type, shard_id = scope + self._metrics.count( + "timer_shard_orphaned", + orphaned, + item_type=item_type or "", + outcome=role.value, + ) + logger.warning( + "expiry shard execution left orphaned tasks role=%s item_type=%s " + "shard_id=%d count=%d", + role.value, + item_type or "", + shard_id, + orphaned, + ) + if orphaned and not allow_orphans: raise self._not_stopped(scope) def _pending_done( @@ -346,8 +371,9 @@ def _pending_done( @staticmethod def _not_stopped(scope: RunnerScope) -> StoreOperationError: - role, item_type = scope + role, item_type, shard_id = scope label = role.value if item_type is None else f"{role.value}:{item_type}" return StoreOperationError( - "shard_not_stopped", f"The {label} shard still has active local work" + "shard_not_stopped", + f"The {label} shard {shard_id} still has active local work", ) diff --git a/openviking/server/bootstrap.py b/openviking/server/bootstrap.py index 29e45e1bfc..2bf83042da 100644 --- a/openviking/server/bootstrap.py +++ b/openviking/server/bootstrap.py @@ -7,14 +7,18 @@ import json import os import shutil +import signal import socket import subprocess import sys import tempfile +import threading import time -from dataclasses import dataclass +from contextlib import contextmanager +from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from types import FrameType +from typing import Iterator, NoReturn, Optional import uvicorn @@ -41,6 +45,30 @@ class BotProcess: log_file: Optional[object] = None +@dataclass +class ExpiryPublisherProcess: + process: subprocess.Popen + stopping: threading.Event = field(default_factory=threading.Event) + monitor_thread: Optional[threading.Thread] = None + + +def _raise_sigterm_as_system_exit(signum: int, _frame: FrameType | None) -> NoReturn: + raise SystemExit(128 + signum) + + +@contextmanager +def _sigterm_runs_finally() -> Iterator[None]: + """Make Uvicorn's replayed SIGTERM unwind through managed-process cleanup.""" + if os.name == "nt" or threading.current_thread() is not threading.main_thread(): + yield + return + previous = signal.signal(signal.SIGTERM, _raise_sigterm_as_system_exit) + try: + yield + finally: + signal.signal(signal.SIGTERM, previous) + + def _get_version() -> str: try: from openviking import __version__ @@ -129,6 +157,100 @@ def _resolve_cli_config_for_bot(config_path: Optional[str]) -> Optional[str]: return None +def _wait_for_expiry_publisher_ready( + process: subprocess.Popen, + status_path: Path, + *, + timeout_seconds: float = 60.0, +) -> None: + started = time.monotonic() + while time.monotonic() - started < timeout_seconds: + returncode = process.poll() + if returncode is not None: + raise RuntimeError(f"Expiry publisher exited before becoming ready (code {returncode})") + try: + status = json.loads(status_path.read_text(encoding="utf-8")) + except (FileNotFoundError, ValueError): + status = {} + if status.get("status") == "failed": + raise RuntimeError(status.get("error") or "Expiry publisher initialization failed") + if status.get("status") == "ready": + return + time.sleep(0.1) + raise TimeoutError("Expiry publisher readiness timed out") + + +def _monitor_expiry_publisher(managed: ExpiryPublisherProcess) -> None: + returncode = managed.process.wait() + if managed.stopping.is_set(): + return + print( + f"Error: managed expiry publisher exited unexpectedly (code {returncode}); " + "stopping OpenViking server.", + file=sys.stderr, + ) + os.kill(os.getpid(), signal.SIGTERM) + + +def _start_expiry_publisher(config_path: Path) -> ExpiryPublisherProcess: + startup_directory = tempfile.TemporaryDirectory(prefix="openviking-expiry-startup-") + status_path = Path(startup_directory.name) / "status.json" + process: subprocess.Popen | None = None + try: + process = subprocess.Popen( + [ + sys.executable, + "-m", + "openviking_cli.expiry_bootstrap", + "--config", + str(config_path), + "--startup-status", + str(status_path), + ], + env=os.environ.copy(), + start_new_session=os.name != "nt", + creationflags=( + getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0 + ), + ) + _wait_for_expiry_publisher_ready(process, status_path) + managed = ExpiryPublisherProcess(process) + managed.monitor_thread = threading.Thread( + target=_monitor_expiry_publisher, + args=(managed,), + name="openviking-expiry-publisher-monitor", + daemon=True, + ) + managed.monitor_thread.start() + print(f"Expiry publisher started (PID: {process.pid})") + return managed + except BaseException: + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise + finally: + startup_directory.cleanup() + + +def _stop_expiry_publisher(managed: ExpiryPublisherProcess) -> None: + managed.stopping.set() + process = managed.process + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + if managed.monitor_thread is not None: + managed.monitor_thread.join(timeout=1) + + def main(): """Main entry point for openviking-server command.""" parser = argparse.ArgumentParser( @@ -208,6 +330,8 @@ def main(): from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton # Load server config from ov.conf + resolved_config_path: Path | None = None + ov_config = None try: resolved_config_path = resolve_config_path( args.config, @@ -215,7 +339,7 @@ def main(): DEFAULT_OV_CONF, ) config = load_server_config(args.config) - OpenVikingConfigSingleton.initialize(config_path=args.config) + ov_config = OpenVikingConfigSingleton.initialize(config_path=args.config) except (FileNotFoundError, ValueError) as e: if isinstance(e, ValueError): print( @@ -249,7 +373,8 @@ def main(): try: from openviking_cli.utils.ollama import detect_ollama_in_config, ensure_ollama_for_server - ov_config = OpenVikingConfigSingleton.get_instance() + if ov_config is None: + ov_config = OpenVikingConfigSingleton.get_instance() uses_ollama, ollama_host, ollama_port = detect_ollama_in_config(ov_config) if uses_ollama: result = ensure_ollama_for_server(ollama_host, ollama_port) @@ -311,44 +436,62 @@ def main(): ) sys.exit(1) - # Create and run server app - app = create_app( - config, - config_path=( - str(resolved_config_path) if resolved_config_path is not None else args.config - ), - ) - workers_info = f" (workers: {config.workers})" if config.workers > 1 else "" - print(f"OpenViking HTTP Server is running on {config.host}:{config.port}{workers_info}") - + expiry_publisher_process: Optional[ExpiryPublisherProcess] = None try: - workers = config.workers - if workers > 1: - # Multi-worker mode requires an import string so each worker - # can independently import the application. We stash the - # resolved config path in an env-var so that the factory can - # pick it up (ServerConfig already reads OPENVIKING_CONFIG_FILE). - os.environ[WORKER_WITH_BOT_ENV] = "1" if config.with_bot else "0" - os.environ[WORKER_BOT_API_URL_ENV] = config.bot_api_url - uvicorn.run( - "openviking.server.app:create_worker_app", - factory=True, - host=config.host, - port=config.port, - workers=workers, - timeout_keep_alive=config.timeout_keep_alive, - log_config=None, - ) - else: - uvicorn.run( - app, - host=config.host, - port=config.port, - timeout_keep_alive=config.timeout_keep_alive, - log_config=None, + with _sigterm_runs_finally(): + idle = getattr(getattr(ov_config, "memory", None), "session_auto_commit", None) + expiry = getattr(ov_config, "expiry", None) + if ( + idle is not None + and idle.enabled + and idle.idle_backend == "redis" + and expiry is not None + and expiry.publisher_separate_process + ): + if resolved_config_path is None: + raise RuntimeError( + "Cannot start expiry publisher without a resolved ov.conf path" + ) + expiry_publisher_process = _start_expiry_publisher(resolved_config_path) + + # Create and run server app only after managed dependencies are ready. + app = create_app( + config, + config_path=( + str(resolved_config_path) if resolved_config_path is not None else args.config + ), ) + workers_info = f" (workers: {config.workers})" if config.workers > 1 else "" + print(f"OpenViking HTTP Server is running on {config.host}:{config.port}{workers_info}") + + workers = config.workers + if workers > 1: + # Multi-worker mode requires an import string so each worker + # can independently import the application. We stash the + # resolved config path in an env-var so that the factory can + # pick it up (ServerConfig already reads OPENVIKING_CONFIG_FILE). + os.environ[WORKER_WITH_BOT_ENV] = "1" if config.with_bot else "0" + os.environ[WORKER_BOT_API_URL_ENV] = config.bot_api_url + uvicorn.run( + "openviking.server.app:create_worker_app", + factory=True, + host=config.host, + port=config.port, + workers=workers, + timeout_keep_alive=config.timeout_keep_alive, + log_config=None, + ) + else: + uvicorn.run( + app, + host=config.host, + port=config.port, + timeout_keep_alive=config.timeout_keep_alive, + log_config=None, + ) finally: - # Cleanup vikingbot process on shutdown + if expiry_publisher_process is not None: + _stop_expiry_publisher(expiry_publisher_process) if bot_process is not None: _stop_vikingbot_gateway(bot_process) diff --git a/openviking/server/config.py b/openviking/server/config.py index fa75c8eb9f..979de8289c 100644 --- a/openviking/server/config.py +++ b/openviking/server/config.py @@ -162,9 +162,16 @@ class UserConfigDefaults(UserConfig): @field_validator("auto_commit_policy", mode="before") @classmethod def validate_auto_commit_policy(cls, value: Any) -> Optional[Dict[str, Any]]: - from openviking.session.auto_commit_policy import AutoCommitPolicy + from openviking.session.auto_commit_policy import ( + AutoCommitPolicy, + validate_idle_timeout_input, + ) - return None if value is None else AutoCommitPolicy.from_dict(value).to_dict() + if value is None: + return None + if isinstance(value, dict) and "idle_timeout_seconds" in value: + validate_idle_timeout_input(value["idle_timeout_seconds"]) + return AutoCommitPolicy.from_dict(value).to_dict() class MetricsAccountDimensionConfig(BaseModel): diff --git a/openviking/server/routers/sessions.py b/openviking/server/routers/sessions.py index 3423bdfc7d..a17471c6f8 100644 --- a/openviking/server/routers/sessions.py +++ b/openviking/server/routers/sessions.py @@ -17,6 +17,7 @@ from openviking.server.models import Response from openviking.server.responses import error_response from openviking.server.telemetry import run_operation +from openviking.session.auto_commit_policy import validate_idle_timeout_input from openviking.telemetry import TelemetryRequest from openviking.utils.image_search import is_viking_uri from openviking_cli.utils import get_logger @@ -78,9 +79,9 @@ class ToolPartRequest(BaseModel): class AutoCommitPolicyRequest(BaseModel): """Session-level auto-commit policy overrides. - All fields are optional. Bounds are not enforced here: ``AutoCommitPolicy`` - clamps every value into ``[0, max]`` so clamping lives in one place across - the HTTP, SDK, and CLI entrypoints. + All fields are optional. ``idle_timeout_seconds`` validates its write-time + minimum here so legacy stored policies remain readable. Other bounds are + clamped by ``AutoCommitPolicy``. """ pending_token_threshold: Optional[int] = None @@ -91,6 +92,11 @@ class AutoCommitPolicyRequest(BaseModel): model_config = {"extra": "forbid"} + @field_validator("idle_timeout_seconds") + @classmethod + def validate_idle_timeout_seconds(cls, value: Optional[int]) -> Optional[int]: + return None if value is None else validate_idle_timeout_input(value) + @model_validator(mode="after") def reject_explicit_null_fields(self) -> "AutoCommitPolicyRequest": null_fields = [field for field in self.model_fields_set if getattr(self, field) is None] diff --git a/openviking/service/core.py b/openviking/service/core.py index 3b938b713a..c11049d95f 100644 --- a/openviking/service/core.py +++ b/openviking/service/core.py @@ -580,12 +580,10 @@ async def initialize(self) -> None: session_auto_commit_config = SessionAutoCommitConfig() self._session_service.set_session_auto_commit_config(session_auto_commit_config) expiry_config = get_openviking_config().expiry - expiry_enabled = expiry_config.enabled session_expiry_enabled = ( session_auto_commit_config.enabled and getattr(session_auto_commit_config, "idle_backend", "legacy") == "redis" ) - has_expiry_work = session_expiry_enabled if ( session_auto_commit_config.enabled and getattr(session_auto_commit_config, "idle_backend", "legacy") == "legacy" @@ -661,16 +659,22 @@ async def initialize(self) -> None: self._queue_manager.start() logger.info("QueueManager workers started") - if expiry_enabled and has_expiry_work: - from openviking.scheduling.expiry.runtime import ExpiryRuntime + if session_expiry_enabled: + from openviking.scheduling.expiry.runtime import ExpiryRuntime, ExpiryRuntimeRoles - self._expiry_runtime = ExpiryRuntime(expiry_config) + self._expiry_runtime = ExpiryRuntime( + expiry_config, + roles=ExpiryRuntimeRoles( + report=True, + publish=not expiry_config.publisher_separate_process, + consume=True, + ), + ) try: - if session_expiry_enabled: - self._session_service.configure_expiry( - self._expiry_runtime.timer, - reporting_enabled=expiry_config.reporting_enabled, - ) + self._session_service.configure_expiry( + self._expiry_runtime.scheduler, + consumer_concurrency=expiry_config.session_consumer_concurrency, + ) await self._expiry_runtime.start_storage() await self._expiry_runtime.start_roles() except Exception: @@ -707,7 +711,7 @@ async def _stop_expiry(self) -> None: """Stop expiry work before releasing dependencies used by callbacks.""" runtime = self._expiry_runtime if runtime is None: - self._session_service.set_session_timer_adapter(None) + self._session_service.set_session_expiry_adapter(None) return try: await runtime.close() @@ -718,7 +722,7 @@ async def _stop_expiry(self) -> None: # callbacks. The caller can retry close after they terminate. raise else: - self._session_service.set_session_timer_adapter(None) + self._session_service.set_session_expiry_adapter(None) self._expiry_runtime = None logger.info("Expiry stopped") diff --git a/openviking/service/session_timer.py b/openviking/service/session_expiry.py similarity index 79% rename from openviking/service/session_timer.py rename to openviking/service/session_expiry.py index 7900e1fba4..76dde89f09 100644 --- a/openviking/service/session_timer.py +++ b/openviking/service/session_expiry.py @@ -1,6 +1,6 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -"""Session business adapter for the generic distributed resource timer.""" +"""Session business adapter for distributed expiry scheduling.""" from __future__ import annotations @@ -21,7 +21,7 @@ ) from openviking.server.identity import RequestContext, Role from openviking.session.auto_commit_policy import AutoCommitPolicy -from openviking.session.session import SessionTimerSnapshot +from openviking.session.session import SessionExpirySnapshot from openviking_cli.session.user_id import UserIdentifier from openviking_cli.utils import get_logger @@ -31,18 +31,22 @@ logger = get_logger(__name__) -def install_session_timer( - timer: ExpiryScheduler, +def install_session_expiry( + scheduler: ExpiryScheduler, sessions: "SessionService", *, - reporting_enabled: bool, + concurrency: int = 8, ) -> None: - timer.register_handler( + scheduler.register_handler( SESSION_ITEM_TYPE, - SessionTimeoutHandler(sessions, timer), - options=HandlerOptions(concurrency=4, max_attempts=3), + SessionTimeoutHandler(sessions, scheduler), + options=HandlerOptions( + concurrency=concurrency, + max_attempts=3, + metadata_enabled=False, + ), ) - sessions.set_session_timer_adapter(SessionTimerAdapter(timer) if reporting_enabled else None) + sessions.set_session_expiry_adapter(SessionExpiryAdapter(scheduler)) def effective_idle_timeout(policy: dict | None) -> int | None: @@ -60,11 +64,11 @@ def timestamp_ms(value: str) -> int: return int(parsed.astimezone(timezone.utc).timestamp() * 1000) -class SessionTimerAdapter: - def __init__(self, timer: ExpiryScheduler) -> None: - self._timer = timer +class SessionExpiryAdapter: + def __init__(self, scheduler: ExpiryScheduler) -> None: + self._scheduler = scheduler - def build_report(self, snapshot: SessionTimerSnapshot) -> ExpiryRegistration | None: + def build_report(self, snapshot: SessionExpirySnapshot) -> ExpiryRegistration | None: timeout = effective_idle_timeout(snapshot.auto_commit_policy) if timeout is None: return None @@ -80,11 +84,11 @@ def build_report(self, snapshot: SessionTimerSnapshot) -> ExpiryRegistration | N mode=ExpiryMode.IDLE_ONESHOT, ) - async def report_after_append(self, snapshot: SessionTimerSnapshot) -> None: + async def report_after_append(self, snapshot: SessionExpirySnapshot) -> None: try: report = self.build_report(snapshot) if report is not None: - await self._timer.report(report) + await self._scheduler.report(report) except asyncio.CancelledError: raise except Exception as exc: @@ -92,7 +96,7 @@ async def report_after_append(self, snapshot: SessionTimerSnapshot) -> None: async def unregister_after_delete(self, identity: ItemIdentity) -> None: try: - await self._timer.unregister(identity) + await self._scheduler.unregister(identity) except asyncio.CancelledError: raise except Exception as exc: @@ -100,7 +104,7 @@ async def unregister_after_delete(self, identity: ItemIdentity) -> None: def record_report_failure(self, reason: str, exc: Exception | None = None) -> None: logger.warning( - "Session timer operation failed reason=%s error=%s", + "Session expiry operation failed reason=%s error=%s", reason, type(exc).__name__ if exc is not None else "unknown", exc_info=exc is not None, @@ -108,9 +112,9 @@ def record_report_failure(self, reason: str, exc: Exception | None = None) -> No class SessionTimeoutHandler: - def __init__(self, sessions: "SessionService", timer: ExpiryScheduler) -> None: + def __init__(self, sessions: "SessionService", scheduler: ExpiryScheduler) -> None: self._sessions = sessions - self._timer = timer + self._scheduler = scheduler async def handle(self, request: CallbackRequest) -> CallbackResult: result = await self._sessions.commit_on_timeout( @@ -124,7 +128,7 @@ async def handle(self, request: CallbackRequest) -> CallbackResult: async def _report_deferred(self, identity: ItemIdentity, result: dict[str, Any]) -> None: if result.get("reason") not in {"idle_cooldown", "idle_activity"}: raise ValueError("Unexpected deferred session result") - await self._timer.report( + await self._scheduler.report( ExpiryRegistration( identity=identity, due_at_ms=result["due_at_ms"], diff --git a/openviking/service/session_service.py b/openviking/service/session_service.py index 48a95a3a39..f4966d8361 100644 --- a/openviking/service/session_service.py +++ b/openviking/service/session_service.py @@ -60,7 +60,7 @@ def _parse_utc_timestamp(value: str) -> datetime: if TYPE_CHECKING: from openviking.config.vlm import VLMResolver from openviking.scheduling.expiry import ExpiryScheduler - from openviking.service.session_timer import SessionTimerAdapter + from openviking.service.session_expiry import SessionExpiryAdapter from openviking.session.compressor_v3 import SessionCompressorV3 from openviking.usage_reporter import UsageReporter @@ -92,15 +92,24 @@ def __init__( self._auto_commit_inflight_lock = asyncio.Lock() # Strong refs so spawned tasks aren't GC'd mid-await. self._auto_commit_tasks: set[asyncio.Task] = set() - self._session_timer_adapter: Optional["SessionTimerAdapter"] = None + self._session_expiry_adapter: Optional["SessionExpiryAdapter"] = None - def configure_expiry(self, scheduler: "ExpiryScheduler", *, reporting_enabled: bool) -> None: - from openviking.service.session_timer import install_session_timer + def configure_expiry( + self, + scheduler: "ExpiryScheduler", + *, + consumer_concurrency: int = 8, + ) -> None: + from openviking.service.session_expiry import install_session_expiry - install_session_timer(scheduler, self, reporting_enabled=reporting_enabled) + install_session_expiry( + scheduler, + self, + concurrency=consumer_concurrency, + ) - def set_session_timer_adapter(self, adapter: Optional["SessionTimerAdapter"]) -> None: - self._session_timer_adapter = adapter + def set_session_expiry_adapter(self, adapter: Optional["SessionExpiryAdapter"]) -> None: + self._session_expiry_adapter = adapter def set_dependencies( self, @@ -240,7 +249,7 @@ def session( memory_policy_provider=lambda: self._get_user_memory_policy(ctx), usage_reporter=self._usage_reporter, vlm_resolver=self._vlm_resolver, - timer_hook=self._session_timer_adapter, + expiry_reporter=self._session_expiry_adapter, ) async def _get_user_memory_policy(self, ctx: RequestContext) -> Optional[Dict[str, Any]]: @@ -385,10 +394,10 @@ async def delete(self, session_id: str, ctx: RequestContext) -> bool: raise NotFoundError(session_id, "session") await self._viking_fs.rm(session_uri, recursive=True, ctx=ctx) - if self._session_timer_adapter is not None: + if self._session_expiry_adapter is not None: from openviking.scheduling.expiry import SESSION_ITEM_TYPE, ItemIdentity - await self._session_timer_adapter.unregister_after_delete( + await self._session_expiry_adapter.unregister_after_delete( ItemIdentity(SESSION_ITEM_TYPE, ctx.account_id, ctx.user.user_id, session_id) ) logger.info(f"Deleted session: {session_id}") @@ -582,11 +591,15 @@ async def _record_idle_commit_time( return metadata["last_idle_commit_accepted_at"] = accepted_at metadata["updated_at"] = get_current_timestamp() + # The held lease is an exact lock on the session directory, used + # to serialize against concurrent append/config meta rewrites. + # Writing the child .meta.json must take its own auto-pathlock + # (the directory lease does not cover a child-path write), just + # like Session._save_meta() does under the same directory lock. await fs.write_file( f"{session_uri}/.meta.json", json.dumps(metadata, ensure_ascii=False), ctx=ctx, - lease_ref=lease, ) finally: await fs._async_agfs.pathlock_release(lease) diff --git a/openviking/session/auto_commit_policy.py b/openviking/session/auto_commit_policy.py index e01eb34ece..050fdcad39 100644 --- a/openviking/session/auto_commit_policy.py +++ b/openviking/session/auto_commit_policy.py @@ -22,7 +22,8 @@ DEFAULT_KEEP_RECENT_COUNT = 0 DEFAULT_MIN_COMMIT_INTERVAL_SECONDS = 0 -# PRD upper bounds. +# PRD bounds. +MIN_IDLE_TIMEOUT_SECONDS = 300 # 5 minutes MAX_PENDING_TOKEN_THRESHOLD = 1000000 MAX_MESSAGE_COUNT_THRESHOLD = 1000 MAX_IDLE_TIMEOUT_SECONDS = 604800 # 7 days @@ -48,6 +49,20 @@ def _coerce_int(value: Any, *, field: str, minimum: int, maximum: int) -> int: return parsed +def validate_idle_timeout_input(value: Any) -> int: + """Validate a newly supplied idle timeout without changing stored policies.""" + try: + parsed = int(value) + except (TypeError, ValueError): + raise ValueError("auto_commit_policy.idle_timeout_seconds must be an integer") + if parsed != 0 and parsed < MIN_IDLE_TIMEOUT_SECONDS: + raise ValueError( + "auto_commit_policy.idle_timeout_seconds must be 0 or at least " + f"{MIN_IDLE_TIMEOUT_SECONDS}" + ) + return parsed + + @dataclass class AutoCommitPolicy: """Effective automatic-commit policy for one session.""" @@ -103,9 +118,7 @@ def from_dict(cls, data: Any) -> "AutoCommitPolicy": maximum=MAX_MESSAGE_COUNT_THRESHOLD, ), min_commit_interval_seconds=_coerce_int( - data.get( - "min_commit_interval_seconds", default.min_commit_interval_seconds - ), + data.get("min_commit_interval_seconds", default.min_commit_interval_seconds), field="min_commit_interval_seconds", minimum=0, maximum=MAX_IDLE_TIMEOUT_SECONDS, diff --git a/openviking/session/session.py b/openviking/session/session.py index a5006a6e00..4ace1d092a 100644 --- a/openviking/session/session.py +++ b/openviking/session/session.py @@ -415,7 +415,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "SessionMeta": @dataclass(frozen=True) -class SessionTimerSnapshot: +class SessionExpirySnapshot: account_id: str user_id: str session_id: str @@ -423,8 +423,8 @@ class SessionTimerSnapshot: auto_commit_policy: Optional[Dict[str, Any]] -class SessionTimerHook(Protocol): - async def report_after_append(self, snapshot: SessionTimerSnapshot) -> None: ... +class SessionExpiryReporter(Protocol): + async def report_after_append(self, snapshot: SessionExpirySnapshot) -> None: ... def record_report_failure(self, reason: str, exc: Optional[Exception] = None) -> None: ... @@ -448,7 +448,7 @@ def __init__( agent_evolution_enabled_provider: Optional[Callable[[], bool | Awaitable[bool]]] = None, memory_policy_provider: Optional[MemoryPolicyProvider] = None, vlm_resolver: Optional["VLMResolver"] = None, - timer_hook: Optional[SessionTimerHook] = None, + expiry_reporter: Optional[SessionExpiryReporter] = None, ): self._viking_fs = viking_fs self._vikingdb_manager = vikingdb_manager @@ -488,7 +488,7 @@ def __init__( self._memory_policy_provider = memory_policy_provider self._usage_reporter = usage_reporter self._vlm_resolver = vlm_resolver - self._timer_hook = timer_hook + self._expiry_reporter = expiry_reporter async def _get_vlm_config(self): if self._vlm_resolver is None: @@ -734,7 +734,7 @@ async def _append_messages_authoritatively(self, messages: List[Message]) -> Non lease = await self._viking_fs._async_agfs.pathlock_acquire_exact( session_path, timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS ) - snapshot: Optional[SessionTimerSnapshot] = None + snapshot: Optional[SessionExpirySnapshot] = None snapshot_error: Optional[Exception] = None metadata_reload_error: Optional[Exception] = None try: @@ -774,36 +774,36 @@ async def _append_messages_authoritatively(self, messages: List[Message]) -> Non ctx=self.ctx, ) await self._save_meta() - if self._timer_hook is not None: + if self._expiry_reporter is not None: try: # _save_meta() made this exact in-memory state authoritative # under the Session path lock, even when the preceding reload # failed. Reporting it prevents the previous due time from - # firing early and repairs a missing timer registration. - snapshot = self._build_timer_snapshot_locked() + # firing early and repairs a missing expiry registration. + snapshot = self._build_expiry_snapshot_locked() except Exception as exc: snapshot_error = exc finally: await self._viking_fs._async_agfs.pathlock_release(lease) - if self._timer_hook is not None: + if self._expiry_reporter is not None: if metadata_reload_error is not None: logger.warning( "Session metadata reload failed before append; reporting repaired state", exc_info=metadata_reload_error, ) if snapshot is not None: - await self._timer_hook.report_after_append(snapshot) + await self._expiry_reporter.report_after_append(snapshot) else: - self._timer_hook.record_report_failure("snapshot_build_failed", snapshot_error) + self._expiry_reporter.record_report_failure("snapshot_build_failed", snapshot_error) - def _build_timer_snapshot_locked(self) -> SessionTimerSnapshot: + def _build_expiry_snapshot_locked(self) -> SessionExpirySnapshot: if not self._meta.last_message_at: raise ValueError("Session last_message_at is missing") account_id = self._meta.created_by_account_id or self.ctx.account_id user_id = self._meta.created_by_user_id or self.ctx.user.user_id if not account_id or not user_id: raise ValueError("Session owner identity is incomplete") - return SessionTimerSnapshot( + return SessionExpirySnapshot( account_id=account_id, user_id=user_id, session_id=self.session_id, diff --git a/openviking_cli/expiry_bootstrap.py b/openviking_cli/expiry_bootstrap.py index 1fdedd28e1..34038c9cc5 100644 --- a/openviking_cli/expiry_bootstrap.py +++ b/openviking_cli/expiry_bootstrap.py @@ -1,6 +1,6 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -"""Standalone distributed expiry publisher.""" +"""Internal distributed expiry publisher process.""" from __future__ import annotations @@ -14,22 +14,35 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser(description="OpenViking expiry publisher") + parser = argparse.ArgumentParser(description="Internal OpenViking expiry publisher") parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--startup-status", required=True, type=Path) return parser.parse_args(argv) def load_expiry_config(path: Path): - from openviking.scheduling.expiry.config import ExpiryConfig + from openviking_cli.utils.config.open_viking_config import OpenVikingConfig raw = json.loads(os.path.expandvars(path.expanduser().read_text(encoding="utf-8-sig"))) - section = dict(raw.get("expiry", {})) - section["reporting_enabled"] = False - section["callback_enabled"] = False - config = ExpiryConfig.model_validate(section) - if not config.scheduler_enabled: - raise ValueError("expiry.scheduler_enabled must be true for expiry CLI") - return config + config = OpenVikingConfig.from_dict(raw) + idle = config.memory.session_auto_commit + if not idle.enabled or idle.idle_backend != "redis": + raise ValueError( + "expiry publisher requires memory.session_auto_commit.enabled=true " + "and idle_backend='redis'" + ) + if not config.expiry.publisher_separate_process: + raise ValueError("expiry publisher process is disabled by configuration") + return config.expiry + + +def write_startup_status(path: Path, status: str, error: str | None = None) -> None: + payload: dict[str, object] = {"pid": os.getpid(), "status": status} + if error: + payload["error"] = error + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(payload), encoding="utf-8") + os.replace(temporary, path) def install_signal_handlers(stop: asyncio.Event) -> None: @@ -41,36 +54,74 @@ def install_signal_handlers(stop: asyncio.Event) -> None: signal.signal(signum, lambda *_args: loop.call_soon_threadsafe(stop.set)) -async def run_publisher(config, stop: asyncio.Event) -> None: - from openviking.scheduling.expiry.runtime import ExpiryRuntime +async def run_publisher(config, stop: asyncio.Event, startup_status: Path) -> None: + from openviking.scheduling.expiry.runtime import ExpiryRuntime, ExpiryRuntimeRoles - runtime = ExpiryRuntime(config) + runtime = ExpiryRuntime(config, roles=ExpiryRuntimeRoles(publish=True)) + wait_task: asyncio.Task[None] | None = None + stop_task: asyncio.Task[bool] | None = None try: await runtime.start_storage() await runtime.start_roles() + write_startup_status(startup_status, "ready") wait_task = asyncio.create_task(runtime.wait()) stop_task = asyncio.create_task(stop.wait()) - done, _ = await asyncio.wait({wait_task, stop_task}, return_when=asyncio.FIRST_COMPLETED) - if wait_task in done: - await wait_task + try: + done, _ = await asyncio.wait( + {wait_task, stop_task}, return_when=asyncio.FIRST_COMPLETED + ) + if wait_task in done: + await wait_task + finally: + stop.set() + await runtime.close() finally: - await runtime.close() + if wait_task is not None: + if not wait_task.done(): + wait_task.cancel() + await asyncio.gather(wait_task, return_exceptions=True) + if stop_task is not None: + if not stop_task.done(): + stop_task.cancel() + await asyncio.gather(stop_task, return_exceptions=True) + if wait_task is None: + await runtime.close() def main(argv: list[str] | None = None) -> int: + startup_status: Path | None = None try: args = parse_args(argv) - config = load_expiry_config(args.config) + current_startup_status: Path = args.startup_status + startup_status = current_startup_status + try: + config = load_expiry_config(args.config) + except ValueError as exc: + # Validation errors are already safely formatted. Avoid emitting + # chained validation tracebacks containing raw configuration input. + write_startup_status(current_startup_status, "failed", str(exc)) + logging.error("Expiry publisher configuration failed: %s", exc) + return 1 + write_startup_status(current_startup_status, "starting") async def runner() -> None: stop = asyncio.Event() install_signal_handlers(stop) - await run_publisher(config, stop) + await run_publisher(config, stop, current_startup_status) asyncio.run(runner()) return 0 except KeyboardInterrupt: return 0 - except Exception: + except Exception as exc: + if startup_status is not None: + try: + write_startup_status(startup_status, "failed", str(exc)) + except Exception: + pass logging.exception("Expiry publisher failed") return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py index 31205c65ca..082ebf6623 100644 --- a/openviking_cli/utils/config/open_viking_config.py +++ b/openviking_cli/utils/config/open_viking_config.py @@ -457,15 +457,8 @@ def _validate_expiry_backend(self) -> "OpenVikingConfig": def _check_expiry_backend(self) -> None: idle = self.memory.session_auto_commit - if ( - idle.enabled - and idle.idle_backend == "redis" - and not self.expiry.reporting_enabled - ): - raise ValueError( - "memory.session_auto_commit.idle_backend='redis' requires " - "expiry.reporting_enabled=true" - ) + if idle.enabled and idle.idle_backend == "redis": + self.expiry.validate_for_runtime() agent_evolution: AgentEvolutionConfig = RuntimeField( default_factory=AgentEvolutionConfig, @@ -498,7 +491,7 @@ def _check_expiry_backend(self) -> None: ), ) - model_config = {"arbitrary_types_allowed": True} + model_config = {"arbitrary_types_allowed": True, "hide_input_in_errors": True} @classmethod def from_dict(cls, config: Dict[str, Any]) -> "OpenVikingConfig": @@ -585,7 +578,7 @@ def from_dict(cls, config: Dict[str, Any]) -> "OpenVikingConfig": error=e, path_prefix="memory", ) - ) from e + ) from None instance._check_expiry_backend() # Apply parser configurations @@ -619,7 +612,9 @@ def from_dict(cls, config: Dict[str, Any]) -> "OpenVikingConfig": ) return instance except ValidationError as e: - raise ValueError(format_validation_error(root_model=cls, error=e)) from e + # The formatted error omits input values; do not reattach a raw + # validation traceback which could expose nested credentials. + raise ValueError(format_validation_error(root_model=cls, error=e)) from None def to_dict(self) -> Dict[str, Any]: """Convert configuration to dictionary.""" diff --git a/pyproject.toml b/pyproject.toml index a7423fcf2f..bd0bcd0162 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -217,7 +217,6 @@ ov = "openviking_cli.rust_cli:main" # Rust CLI 入口（极简包装器） openviking = "openviking_cli.rust_cli:main" # Rust CLI 入口（放弃 python CLI） openviking-server = "openviking_cli.server_bootstrap:main" # also: `openviking-server ingest ...` openviking-context-gateway = "context_gateway.cli:main" -openviking-timer = "openviking_cli.expiry_bootstrap:main" vikingbot = "vikingbot.cli.commands:app" [tool.setuptools_scm] diff --git a/tests/scheduling/expiry/test_config.py b/tests/scheduling/expiry/test_config.py index c935f6f335..41a39bddcb 100644 --- a/tests/scheduling/expiry/test_config.py +++ b/tests/scheduling/expiry/test_config.py @@ -1,22 +1,48 @@ +import traceback + import pytest from pydantic import ValidationError -from openviking.scheduling.expiry.config import ExpiryConfig +from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig from openviking_cli.utils.config.open_viking_config import OpenVikingConfig -def test_timer_disabled_by_default_without_redis_dependency(): - assert ExpiryConfig().enabled is False +def test_publisher_uses_separate_process_by_default(): + assert ExpiryConfig().publisher_separate_process is True + + +def test_incomplete_expiry_config_is_allowed_when_redis_backend_is_disabled(): + config = OpenVikingConfig.from_dict({"expiry": {"service_name": ""}}) + + assert config.expiry.redis.endpoints == [] + + +def test_expiry_capacity_defaults(): + config = ExpiryConfig() + + assert config.shard_count == 128 + assert config.session_consumer_concurrency == 8 + +@pytest.mark.parametrize("value", [1, 64, 128, 512]) +def test_shard_count_accepts_configured_value(value): + assert ExpiryConfig(shard_count=value).shard_count == value -def test_enabled_expiry_requires_service_name_and_endpoint(): + +@pytest.mark.parametrize("value", [0, 513]) +def test_shard_count_is_bounded(value): with pytest.raises(ValidationError): - ExpiryConfig(reporting_enabled=True) + ExpiryConfig(shard_count=value) + +@pytest.mark.parametrize("value", [1, 8, 1024]) +def test_session_consumer_concurrency_accepts_positive_value(value): + assert ExpiryConfig(session_consumer_concurrency=value).session_consumer_concurrency == value -def test_shard_count_is_fixed_for_the_initial_protocol(): + +def test_session_consumer_concurrency_must_be_positive(): with pytest.raises(ValidationError): - ExpiryConfig(shard_count=256) + ExpiryConfig(session_consumer_concurrency=0) @pytest.mark.parametrize("value", [0, 4097]) @@ -25,20 +51,19 @@ def test_cleanup_batch_size_is_bounded(value): ExpiryConfig(cleanup_batch_size=value) -def test_callback_requires_reporting_for_cooldown_reschedule(): - with pytest.raises(ValidationError): - ExpiryConfig( - service_name="documents", - callback_enabled=True, - redis={"endpoints": ["localhost:6379"]}, - ) +@pytest.mark.parametrize( + "field", + ["reporting_enabled", "scheduler_enabled", "callback_enabled"], +) +def test_legacy_role_switches_are_rejected(field): + with pytest.raises(ValidationError, match=field): + ExpiryConfig.model_validate({field: True}) def test_cluster_rejects_mixed_transport_schemes(): with pytest.raises(ValidationError, match="one transport scheme"): ExpiryConfig( service_name="documents", - reporting_enabled=True, redis={ "mode": "cluster", "endpoints": ["redis://one:6379", "rediss://two:6379"], @@ -49,7 +74,6 @@ def test_cluster_rejects_mixed_transport_schemes(): def test_redis_password_can_be_configured_directly(): config = ExpiryConfig( service_name="documents", - reporting_enabled=True, redis={ "endpoints": ["localhost:6379"], "password": "secret", @@ -63,7 +87,6 @@ def test_redis_password_sources_are_mutually_exclusive(): with pytest.raises(ValidationError, match="mutually exclusive"): ExpiryConfig( service_name="documents", - reporting_enabled=True, redis={ "endpoints": ["localhost:6379"], "password": "secret", @@ -72,8 +95,85 @@ def test_redis_password_sources_are_mutually_exclusive(): ) -def test_redis_idle_backend_requires_reporting(): - with pytest.raises(ValueError, match="reporting_enabled"): +@pytest.mark.parametrize("password", ["x", "private-redis-password-" * 20]) +@pytest.mark.parametrize("level", ["redis", "expiry", "aggregate"]) +@pytest.mark.parametrize("failure", ["endpoint", "password_sources", "field_type"]) +def test_redis_config_validation_traceback_does_not_expose_password(password, level, failure): + redis = {"endpoints": ["localhost:6379"], "password": password} + if failure == "endpoint": + redis["endpoints"] = ["invalid-endpoint"] + elif failure == "password_sources": + redis["password_env"] = "REDIS_PASSWORD" + else: + redis["max_connections"] = "invalid" + with pytest.raises((ValueError, ValidationError)) as error: + if level == "redis": + ExpiryRedisConfig.model_validate(redis) + elif level == "expiry": + ExpiryConfig.model_validate({"redis": redis}) + else: + OpenVikingConfig.from_dict({"expiry": {"redis": redis}}) + formatted = "".join(traceback.format_exception(error.value)) + # Short passwords are tested as raw input values rather than arbitrary + # characters which can also occur in ordinary error text. + assert f"'password': '{password}'" not in formatted + if len(password) > 1: + assert password not in formatted + assert "input_value=" not in formatted + + +@pytest.mark.parametrize( + "endpoint", + [ + "redis://user:embedded-secret@localhost", + "redis://user:embedded-secret@localhost:bad-port", + "redis://user:embedded-secret@[bad-address]:6379", + "redis://user:embedded-secret@localhost:6379", + ], +) +def test_redis_endpoint_validation_never_echoes_credentials(endpoint): + with pytest.raises(ValidationError) as error: + ExpiryRedisConfig(endpoints=[endpoint]) + assert "embedded-secret" not in "".join(traceback.format_exception(error.value)) + + +def test_aggregate_validation_hides_redis_password_for_unrelated_error(): + with pytest.raises(ValueError) as error: + OpenVikingConfig.from_dict( + {"default_account": [], "expiry": {"redis": {"password": "private-password"}}} + ) + assert "private-password" not in "".join(traceback.format_exception(error.value)) + + +def test_password_env_must_exist_when_resolved(monkeypatch): + monkeypatch.delenv("MISSING_REDIS_PASSWORD", raising=False) + config = ExpiryRedisConfig(password_env="MISSING_REDIS_PASSWORD") + + with pytest.raises(ValueError, match="MISSING_REDIS_PASSWORD"): + config.resolve_password() + + +def test_password_env_is_resolved_without_exposing_it_in_endpoint(monkeypatch): + monkeypatch.setenv("EXPIRY_REDIS_PASSWORD", "secret") + config = ExpiryRedisConfig(password_env="EXPIRY_REDIS_PASSWORD") + + assert config.resolve_password() == "secret" + + +def test_redis_endpoint_rejects_embedded_credentials(): + with pytest.raises(ValidationError, match="dedicated Redis fields"): + ExpiryRedisConfig(endpoints=["redis://user:secret@localhost:6379"]) + + +@pytest.mark.parametrize( + ("expiry", "message"), + [ + ({}, "service_name"), + ({"service_name": "documents"}, "redis.endpoints"), + ], +) +def test_redis_idle_backend_requires_complete_expiry_config(expiry, message): + with pytest.raises(ValueError, match=message): OpenVikingConfig.from_dict( { "memory": { @@ -81,7 +181,8 @@ def test_redis_idle_backend_requires_reporting(): "enabled": True, "idle_backend": "redis", } - } + }, + "expiry": expiry, } ) @@ -91,11 +192,21 @@ def test_expiry_top_level_config_is_parsed(): { "expiry": { "service_name": "documents", - "reporting_enabled": True, + "publisher_separate_process": False, + "shard_count": 64, + "session_consumer_concurrency": 16, "redis": {"endpoints": ["localhost:6379"]}, - } + }, + "memory": { + "session_auto_commit": { + "enabled": True, + "idle_backend": "redis", + } + }, } ) assert config.expiry.service_name == "documents" - assert config.expiry.reporting_enabled is True + assert config.expiry.publisher_separate_process is False + assert config.expiry.shard_count == 64 + assert config.expiry.session_consumer_concurrency == 16 diff --git a/tests/scheduling/expiry/test_consumer.py b/tests/scheduling/expiry/test_consumer.py index cec9b2c3f5..d0707e999b 100644 --- a/tests/scheduling/expiry/test_consumer.py +++ b/tests/scheduling/expiry/test_consumer.py @@ -2,6 +2,7 @@ import pytest +from openviking.scheduling.expiry import consumer as consumer_module from openviking.scheduling.expiry.api import ( CallbackResult, CallbackStatus, @@ -181,12 +182,19 @@ def config(**overrides): return ExpiryConfig.model_construct(**values) -def consumer(store, handler, *, max_attempts=3, metrics=None): +def consumer(store, handler, *, max_attempts=3, metadata_enabled=True, metrics=None): timer_config = config() metrics = metrics or ExpiryMetrics() coordinator = ShardLeaseCoordinator(store, timer_config, metrics) shard_runner = ShardRunner(coordinator, timer_config, metrics) - binding = HandlerBinding(handler, HandlerOptions(concurrency=1, max_attempts=max_attempts)) + binding = HandlerBinding( + handler, + HandlerOptions( + concurrency=1, + max_attempts=max_attempts, + metadata_enabled=metadata_enabled, + ), + ) return ( ExpiryConsumer( store, @@ -264,6 +272,31 @@ async def test_completed_callback_with_missing_metadata_acks_claim_and_releases_ assert shard_runner.has_pending is False +@pytest.mark.asyncio +async def test_completed_callback_skips_metadata_lookup_when_disabled(): + store = Store() + handler = Handler() + handler.release.set() + resource_consumer, shard_runner = consumer(store, handler, metadata_enabled=False) + + assert ( + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + is True + ) + assert [request.identity for request in handler.requests] == [store.identity] + assert store.metadata_reads == [] + assert store.acked == [store.claims[0]] + + @pytest.mark.asyncio @pytest.mark.parametrize("failure", ["retry", "exception"]) @pytest.mark.parametrize("last_attempt", [False, True]) @@ -288,7 +321,7 @@ async def test_failed_callback_advances_immediately_and_acks_after_last_attempt( ) try: await asyncio.wait_for(handler.started.wait(), timeout=2.0) - callback_tasks = set(resource_consumer._state.tasks) + callback_tasks = set(resource_consumer._callback_tasks) assert callback_tasks handler.release.set() await asyncio.wait_for(handler.finished.wait(), timeout=2.0) @@ -578,6 +611,7 @@ async def test_owner_loss_keeps_resistant_callback_pending_and_prevents_late_ack item_type="session", ) ) + next_run = None try: await asyncio.wait_for(handler.started.wait(), timeout=2.0) await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) @@ -587,29 +621,46 @@ async def test_owner_loss_keeps_resistant_callback_pending_and_prevents_late_ack assert run_task in finished, ( "Consumer cleanup must not join a resistant callback indefinitely" ) - with pytest.raises(StoreOperationError) as error: - run_task.result() - assert error.value.code == "shard_not_stopped" + assert run_task.result() is True assert shard_runner.has_pending is True assert await asyncio.wait_for(shard_runner.wait_pending(0.01), timeout=2.0) is False assert store.acked == [] assert store.released == [] - # The pending child still owns its reserved slot; do not reset local - # accounting while its cancellation has not completed. - assert resource_consumer._state.reserved == 1 + assert len(resource_consumer._callback_tasks) == 1 + + store.lose_on_renew = False + store.now_ms += 3_000 + next_run = asyncio.create_task( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ) + ) + + async def wait_for_second_callback(): + while len(handler.requests) < 2: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_second_callback(), timeout=2.0) + assert len(resource_consumer._callback_tasks) == 2 + handler.release.set() + assert await asyncio.wait_for(next_run, timeout=2.0) is True assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True assert handler.finished.is_set() - assert store.acked == [] - assert store.released == [] - assert resource_consumer._state.reserved == 0 - assert not resource_consumer._state.tasks + assert store.acked == [store.claims[1]] + assert len(store.released) == 1 + assert not resource_consumer._callback_tasks assert shard_runner.has_pending is False finally: stop.set() store.allow_renew.set() handler.release.set() - await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) + tasks = [run_task] if next_run is None else [run_task, next_run] + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=2.0) assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True @@ -639,16 +690,31 @@ async def test_consumer_worker_does_not_enter_runner_when_already_stopped(): @pytest.mark.asyncio -async def test_consumer_worker_propagates_pending_work_instead_of_acquiring_next_shard(): +async def test_consumer_worker_warns_and_continues_after_unfinished_shard(monkeypatch): shard_runner = Runner( stop_on_shard=1, error=StoreOperationError("shard_not_stopped", "callback still running"), ) resource_consumer = worker_consumer(Store(empty=True), shard_runner, config(shard_count=3)) - with pytest.raises(StoreOperationError) as error: - await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) - assert error.value.code == "shard_not_stopped" - assert len(shard_runner.calls) == 1 + events = [] + resource_consumer._metrics = ExpiryMetrics( + lambda name, value, labels: events.append((name, value, labels)) + ) + warnings = [] + monkeypatch.setattr( + consumer_module.logger, + "warning", + lambda *args, **kwargs: warnings.append((args, kwargs)), + ) + + await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) + + assert [shard_id for _, _, shard_id, _ in shard_runner.calls] == [0, 1] + assert any( + name == "timer_shard_round" and labels["outcome"] == "orphaned" + for name, _, labels in events + ) + assert warnings @pytest.mark.asyncio diff --git a/tests/scheduling/expiry/test_expiry_bootstrap.py b/tests/scheduling/expiry/test_expiry_bootstrap.py new file mode 100644 index 0000000000..b49cbeca7d --- /dev/null +++ b/tests/scheduling/expiry/test_expiry_bootstrap.py @@ -0,0 +1,156 @@ +import asyncio +import json +import logging +import subprocess +import sys + +import pytest + +from openviking.scheduling.expiry import runtime as runtime_module +from openviking_cli import expiry_bootstrap + + +def write_config(tmp_path, *, enabled=True, idle_backend="redis", separate=True): + path = tmp_path / "ov.conf" + path.write_text( + json.dumps( + { + "memory": { + "session_auto_commit": { + "enabled": enabled, + "idle_backend": idle_backend, + } + }, + "expiry": { + "service_name": "documents", + "publisher_separate_process": separate, + "redis": { + "mode": "cluster", + "endpoints": ["redis://proxy.example.com:6379"], + }, + }, + } + ), + encoding="utf-8", + ) + return path + + +def test_internal_publisher_loads_validated_expiry_config(tmp_path): + config = expiry_bootstrap.load_expiry_config(write_config(tmp_path)) + + assert config.service_name == "documents" + assert config.redis.mode == "cluster" + + +@pytest.mark.parametrize( + "options", + [ + {"enabled": False}, + {"idle_backend": "legacy"}, + {"separate": False}, + ], +) +def test_internal_publisher_rejects_configuration_that_does_not_select_it(tmp_path, options): + with pytest.raises(ValueError): + expiry_bootstrap.load_expiry_config(write_config(tmp_path, **options)) + + +def test_startup_status_is_replaced_atomically(tmp_path): + status_path = tmp_path / "status.json" + + expiry_bootstrap.write_startup_status(status_path, "failed", "redis unavailable") + + assert json.loads(status_path.read_text(encoding="utf-8"))["error"] == "redis unavailable" + assert list(tmp_path.glob("*.tmp")) == [] + + +@pytest.mark.asyncio +async def test_publisher_closes_runtime_before_joining_wait_task(monkeypatch, tmp_path): + instances = [] + + class Runtime: + def __init__(self, config, *, roles): + self.stopped = asyncio.Event() + self.close_calls = 0 + instances.append(self) + + async def start_storage(self): + pass + + async def start_roles(self): + pass + + async def wait(self): + await self.stopped.wait() + + async def close(self): + self.close_calls += 1 + self.stopped.set() + + monkeypatch.setattr(runtime_module, "ExpiryRuntime", Runtime) + stop = asyncio.Event() + stop.set() + + await asyncio.wait_for( + expiry_bootstrap.run_publisher(object(), stop, tmp_path / "status.json"), + timeout=1.0, + ) + + assert instances[0].close_calls == 1 + + +@pytest.mark.parametrize("password", ["short-secret", "private-password-" * 40]) +@pytest.mark.parametrize("failure", ["endpoint", "password_sources", "unrelated"]) +def test_publisher_config_failure_logs_and_status_do_not_expose_password( + tmp_path, caplog, password, failure +): + config_path = write_config(tmp_path) + data = json.loads(config_path.read_text()) + data["expiry"]["redis"]["password"] = password + if failure == "endpoint": + data["expiry"]["redis"]["endpoints"] = ["invalid-endpoint"] + elif failure == "password_sources": + data["expiry"]["redis"]["password_env"] = "REDIS_PASSWORD" + else: + data["default_account"] = [] + config_path.write_text(json.dumps(data)) + status = tmp_path / "status.json" + with caplog.at_level(logging.ERROR): + result = expiry_bootstrap.main( + ["--config", str(config_path), "--startup-status", str(status)] + ) + assert result == 1 + assert password not in caplog.text + assert password not in status.read_text() + assert json.loads(status.read_text())["status"] == "failed" + assert "configuration failed" in caplog.text + assert all(record.exc_info is None for record in caplog.records) + + +def test_publisher_runtime_failure_retains_exception_logging(monkeypatch, tmp_path, caplog): + config_path = write_config(tmp_path) + + async def fail(*args): + raise RuntimeError("runtime failure") + + monkeypatch.setattr(expiry_bootstrap, "run_publisher", fail) + with caplog.at_level(logging.ERROR): + result = expiry_bootstrap.main( + ["--config", str(config_path), "--startup-status", str(tmp_path / "status")] + ) + assert result == 1 + assert "runtime failure" in caplog.text + assert any(record.exc_info is not None for record in caplog.records) + + +def test_module_entrypoint_invokes_main(): + result = subprocess.run( + [sys.executable, "-m", "openviking_cli.expiry_bootstrap", "--help"], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0 + assert "--startup-status" in result.stdout diff --git a/tests/scheduling/expiry/test_facade_and_registry.py b/tests/scheduling/expiry/test_facade_and_registry.py index 68921ae168..3783f28342 100644 --- a/tests/scheduling/expiry/test_facade_and_registry.py +++ b/tests/scheduling/expiry/test_facade_and_registry.py @@ -34,8 +34,8 @@ async def ensure_metadata(self, item_id, task): if self.fail_metadata: raise StoreOperationError("redis_error", "metadata failed", retryable=True) - async def unregister_resource(self, shard_id, item_id): - self.calls.append("unregister") + async def unregister_resource(self, shard_id, item_id, *, delete_metadata=True): + self.calls.append(("unregister", delete_metadata)) def test_registry_validation_and_freeze(): @@ -74,6 +74,28 @@ async def test_report_writes_resource_before_metadata(): assert store.calls == ["resource", "metadata"] +@pytest.mark.asyncio +async def test_report_skips_metadata_when_handler_disables_it(): + store = Store() + registry = HandlerRegistry() + registry.register("session", Handler(), HandlerOptions(metadata_enabled=False)) + facade = _ExpirySchedulerFacade( + store, + registry, + ExpiryRuntimeState(storage_ready=True, accepting_calls=True), + shard_count=8, + reporting_enabled=True, + metrics=ExpiryMetrics(), + ) + + result = await facade.report( + ExpiryRegistration(ItemIdentity("session", "a", "u", "s"), 1234, 60) + ) + + assert result.accepted_due_at_ms == 1234 + assert store.calls == ["resource"] + + @pytest.mark.asyncio async def test_metadata_failure_does_not_roll_back_resource(): store = Store() @@ -87,9 +109,7 @@ async def test_metadata_failure_does_not_roll_back_resource(): metrics=ExpiryMetrics(), ) with pytest.raises(ExpiryError) as exc: - await facade.report( - ExpiryRegistration(ItemIdentity("session", "a", "u", "s"), 1234, 60) - ) + await facade.report(ExpiryRegistration(ItemIdentity("session", "a", "u", "s"), 1234, 60)) assert exc.value.code == "timer_unavailable" assert store.calls == ["resource", "metadata"] @@ -128,3 +148,30 @@ async def test_unregister_rejects_unsupported_type_before_store_write(): await facade.unregister(ItemIdentity("watch", "a", "u", "s")) assert exc.value.code == "unregistered_type" assert store.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata_enabled", "delete_metadata"), + [(True, True), (False, False)], +) +async def test_unregister_follows_handler_metadata_option(metadata_enabled, delete_metadata): + store = Store() + registry = HandlerRegistry() + registry.register( + "session", + Handler(), + HandlerOptions(metadata_enabled=metadata_enabled), + ) + facade = _ExpirySchedulerFacade( + store, + registry, + ExpiryRuntimeState(storage_ready=True, accepting_calls=True), + shard_count=8, + reporting_enabled=True, + metrics=ExpiryMetrics(), + ) + + await facade.unregister(ItemIdentity("session", "a", "u", "s")) + + assert store.calls == [("unregister", delete_metadata)] diff --git a/tests/scheduling/expiry/test_leases.py b/tests/scheduling/expiry/test_leases.py index a881e11e2e..5e00142e9d 100644 --- a/tests/scheduling/expiry/test_leases.py +++ b/tests/scheduling/expiry/test_leases.py @@ -78,9 +78,9 @@ def config(**overrides): return ExpiryConfig.model_construct(**values) -def runner(store, timer_config=None): +def runner(store, timer_config=None, metrics=None): timer_config = timer_config or config() - metrics = ExpiryMetrics() + metrics = metrics or ExpiryMetrics() coordinator = ShardLeaseCoordinator(store, timer_config, metrics) return ShardRunner(coordinator, timer_config, metrics) @@ -368,9 +368,11 @@ async def process(execution, stop): @pytest.mark.asyncio -async def test_cancellation_resistant_processor_is_bounded_retained_and_blocks_same_role(): +async def test_cancellation_resistant_processor_is_retained_without_blocking_new_executions(): store = Store() - shard_runner = runner(store, config(shutdown_timeout_seconds=0.02)) + events = [] + metrics = ExpiryMetrics(lambda name, value, labels: events.append((name, value, labels))) + shard_runner = runner(store, config(shutdown_timeout_seconds=0.02), metrics) stop = asyncio.Event() started = asyncio.Event() cancelled = asyncio.Event() @@ -400,21 +402,21 @@ async def completed(execution, current_stop): assert run_task in finished, ( "Cleanup must not wait indefinitely for cancellation resistance" ) - with pytest.raises(StoreOperationError) as error: - run_task.result() - assert error.value.code == "shard_not_stopped" + assert run_task.result() is True assert shard_runner.has_pending is True gc.collect() assert process_refs[0]() is not None assert not process_refs[0]().done() assert await asyncio.wait_for(shard_runner.wait_pending(0.01), timeout=2.0) is False - acquisition_count = len(store.acquired) - with pytest.raises(StoreOperationError) as error: - await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 1, completed, asyncio.Event()) - assert error.value.code == "shard_not_stopped" - assert len(store.acquired) == acquisition_count assert store.released == [] - # Pending work blocks only its own role. + assert ( + await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 1, completed, asyncio.Event()) + is True + ) + assert ( + await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, completed, asyncio.Event()) + is True + ) assert ( await shard_runner.run_if_acquired( OwnerRole.CALLBACK, @@ -425,6 +427,10 @@ async def completed(execution, current_stop): ) is True ) + assert any( + name == "timer_shard_orphaned" and value == 1 and labels["outcome"] == "scheduler" + for name, value, labels in events + ) release_process.set() assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True assert shard_runner.has_pending is False @@ -432,7 +438,7 @@ async def completed(execution, current_stop): await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, completed, asyncio.Event()) is True ) - assert len(store.released) == 2 + assert len(store.released) == 4 finally: stop.set() release_process.set() @@ -486,9 +492,7 @@ async def child(): await asyncio.wait_for(child_cancelled.wait(), timeout=2.0) finished, _ = await asyncio.wait({run_task}, timeout=0.3) assert run_task in finished - with pytest.raises(StoreOperationError) as error: - run_task.result() - assert error.value.code == "shard_not_stopped" + assert run_task.result() is True assert shard_runner.has_pending is True assert store.released == [] called = [] diff --git a/tests/scheduling/expiry/test_models_and_keys.py b/tests/scheduling/expiry/test_models_and_keys.py index aa5def2e8f..5f0564bc67 100644 --- a/tests/scheduling/expiry/test_models_and_keys.py +++ b/tests/scheduling/expiry/test_models_and_keys.py @@ -48,19 +48,23 @@ def test_component_round_trip_preserves_unicode(): assert decode_component(encode_component(value)) == value -def test_timer_keys_only_use_a_control_hash_tag_where_required(): +def test_expiry_keys_only_use_a_control_hash_tag_where_required(): keys = ExpiryKeys(encode_component("cluster/resource")) assert keys.service_key == encode_component("cluster/resource") ref = BucketRef("session", 7, 120_000) + tag = keys.control_tag(7) + assert keys.namespace() == f"expiry:namespace:{keys.service_key}" + assert keys.metadata("item") == f"expiry:metadata:{keys.service_key}:item" + assert keys.source(7) == f"expiry:publisher:source:{keys.service_key}:7" + assert keys.progress(7) == f"expiry:publisher:progress:{tag}" + assert keys.registry(7) == f"expiry:consumer:registry:{tag}" + assert keys.bucket(ref) == f"expiry:consumer:bucket:{keys.service_key}:session:7:120000" assert "{" not in keys.source(7) assert "{" not in keys.bucket(ref) - tag = keys.control_tag(7) - assert keys.progress(7) == f"resource_sche:process:consume_progress:{tag}" - assert keys.registry(7) == f"resource_consume:bucket_register:{tag}" scheduler_owner = keys.owner(OwnerRole.SCHEDULER, 7) callback_owner = keys.owner(OwnerRole.CALLBACK, 7, item_type="session") - assert scheduler_owner == f"resource_timer:owner:scheduler:{tag}" - assert callback_owner == f"resource_timer:owner:callback:session:{tag}" + assert scheduler_owner == f"expiry:publisher:owner:{tag}" + assert callback_owner == f"expiry:consumer:owner:session:{tag}" def test_callback_owner_requires_item_type_scope(): diff --git a/tests/scheduling/expiry/test_redis_loop.py b/tests/scheduling/expiry/test_redis_loop.py index 0c21f0e223..ded5bedf84 100644 --- a/tests/scheduling/expiry/test_redis_loop.py +++ b/tests/scheduling/expiry/test_redis_loop.py @@ -38,6 +38,9 @@ def runtime(io, store): result._tasks = {} result._storage_started = True result._storage_closed = False + result._storage_owned = True + result._store_open_attempted = True + result._storage_start_task = None result._store = store result._io = io result._shard_runner = Runner() @@ -181,6 +184,47 @@ async def operation(): assert not thread.is_alive() +@pytest.mark.asyncio +async def test_quiesce_atomically_rejects_request_before_registration(): + io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) + await io.start() + submit_entered = threading.Event() + allow_submit = threading.Event() + operation_started = threading.Event() + original_submit = io._submit + + def delayed_submit(operation, *, cleanup=False): + submit_entered.set() + allow_submit.wait() + return original_submit(operation, cleanup=cleanup) + + io._submit = delayed_submit + + async def operation(): + operation_started.set() + + def run_from_thread(): + try: + asyncio.run(io.run(operation)) + except BaseException as exc: + return exc + return None + + request = asyncio.create_task(asyncio.to_thread(run_from_thread)) + try: + assert await asyncio.to_thread(submit_entered.wait, 2.0) + await io.quiesce(0.1) + allow_submit.set() + result = await asyncio.wait_for(request, timeout=2.0) + assert isinstance(result, StoreOperationError) + assert result.code == "not_started" + assert not operation_started.is_set() + finally: + allow_submit.set() + await asyncio.gather(request, return_exceptions=True) + await io.close(2.0) + + @pytest.mark.asyncio async def test_cleanup_runs_nested_io_without_reopening_admission_or_deadlocking_gate(): io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) diff --git a/tests/scheduling/expiry/test_redis_store.py b/tests/scheduling/expiry/test_redis_store.py index 85fd760e8d..947428b23c 100644 --- a/tests/scheduling/expiry/test_redis_store.py +++ b/tests/scheduling/expiry/test_redis_store.py @@ -1,4 +1,5 @@ -from unittest.mock import AsyncMock +import asyncio +from unittest.mock import AsyncMock, Mock import pytest @@ -75,6 +76,250 @@ def store(client): return result +@pytest.fixture +async def cluster_store(monkeypatch): + cluster = pytest.importorskip("redis.asyncio.cluster") + timer_store = store(None) + timer_store._config = ExpiryRedisConfig(mode="cluster", endpoints=["127.0.0.1:6379"]) + ping = AsyncMock(return_value=True) + monkeypatch.setattr(cluster.RedisCluster, "ping", ping) + await timer_store.open() + ping.assert_awaited_once() + try: + yield timer_store + finally: + await timer_store.close() + + +def configure_cluster_node(monkeypatch, client, key, execute): + cluster = pytest.importorskip("redis.asyncio.cluster") + node = cluster.ClusterNode("127.0.0.1", 6379, server_type="primary") + manager = client.nodes_manager + manager.default_node = node + manager.nodes_cache = {node.name: node} + manager.slots_cache = {client.keyslot(key): [node]} + + async def initialize(_client): + # Replace topology discovery only; execute_command and its retry/routing + # loops remain the real redis-py implementation, including after close. + _client._initialize = False + return _client + + monkeypatch.setattr(cluster.RedisCluster, "initialize", initialize) + parser = Mock(spec=type(client.commands_parser)) + parser.get_keys = AsyncMock(return_value=[key]) + client.commands_parser = parser + monkeypatch.setattr(cluster.ClusterNode, "execute_command", execute) + return node + + +@pytest.mark.asyncio +async def test_cluster_retry_settings_are_local_to_expiry_client(cluster_store): + from redis.asyncio import Redis + from redis.asyncio.cluster import RedisCluster + + client = cluster_store._client + other_cluster = RedisCluster(host="127.0.0.1", port=6379) + other_standalone = Redis(host="127.0.0.1", port=6379) + try: + assert client.cluster_error_retry_attempts == 0 + assert client.connection_error_retry_attempts == 0 + assert client.get_retry() is None + assert all( + node.connection_kwargs["retry"] is None + for node in client.nodes_manager.startup_nodes.values() + ) + assert other_cluster.cluster_error_retry_attempts == 3 + assert other_cluster.connection_error_retry_attempts == 3 + assert other_cluster.get_retry() is not None + assert other_standalone.connection_pool.connection_kwargs["retry"] is None + finally: + await other_cluster.aclose() + await other_standalone.aclose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("command", ["DEL", "UNLINK"]) +async def test_cluster_single_key_delete_bypasses_pipeline( + monkeypatch, cluster_store, command +): + key = f"single-key-{command.lower()}" + calls = [] + + async def execute(node, *args, **kwargs): + calls.append((node.name, args, kwargs)) + return 1 + + node = configure_cluster_node(monkeypatch, cluster_store._client, key, execute) + + result = await cluster_store._single_key_command(command, key, retry="idempotent") + + assert result == 1 + assert calls == [(node.name, (command, key), {})] + + +@pytest.mark.asyncio +async def test_standalone_client_constructor_is_unchanged(monkeypatch): + redis_async = pytest.importorskip("redis.asyncio") + client = Mock(spec=redis_async.Redis) + client.ping = AsyncMock(return_value=True) + client.aclose = AsyncMock() + factory = Mock(return_value=client) + monkeypatch.setattr(redis_async.Redis, "from_url", factory) + timer_store = store(None) + timer_store._config = ExpiryRedisConfig(endpoints=["127.0.0.1:6379"]) + try: + await timer_store.open() + factory.assert_called_once_with( + "redis://127.0.0.1:6379", + username=None, + password=None, + decode_responses=True, + socket_connect_timeout=1.0, + socket_timeout=2.0, + max_connections=64, + ) + client.ping.assert_awaited_once() + finally: + await timer_store.close() + client.aclose.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ["claim", "owner"]) +@pytest.mark.parametrize("error_name", ["ConnectionError", "TimeoutError"]) +async def test_cluster_uncertain_write_is_not_replayed( + monkeypatch, cluster_store, operation, error_name +): + from redis import exceptions + + bucket = BucketRef("session", 3, 60_000) + key = ( + cluster_store._keys.bucket(bucket) + if operation == "claim" + else cluster_store._keys.owner(OwnerRole.CALLBACK, 3, item_type="session") + ) + expected_score = encode_claim_score(1, 123) + state = {"score": expected_score, "owner": None, "calls": 0} + + async def execute(_node, *args, **kwargs): + state["calls"] += 1 + if operation == "claim": + assert args[0] == "EVALSHA" + if state["score"] != int(args[-2]): + return 0 + state["score"] = int(args[-1]) + else: + assert args[0] == "SET" + if state["owner"] is not None: + return False + state["owner"] = args[2] + # The write takes effect before the response is lost. A second call + # would return 0/False and hide the successful first execution. + raise getattr(exceptions, error_name)("response lost after write") + + configure_cluster_node(monkeypatch, cluster_store._client, key, execute) + with pytest.raises(StoreOperationError, match="response lost") as raised: + if operation == "claim": + await cluster_store.claim( + bucket, + BucketItem("item", expected_score), + attempt=1, + started_at_seconds=200, + ) + else: + await cluster_store.try_acquire_owner( + OwnerRole.CALLBACK, 3, "token", 120, item_type="session" + ) + assert raised.value.outcome_unknown is True + assert state["calls"] == 1 + if operation == "claim": + assert state["score"] == encode_claim_score(1, 200) + else: + assert state["owner"] == "token" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("operation", ["read", "idempotent"]) +async def test_cluster_safe_operations_still_retry_in_store(monkeypatch, cluster_store, operation): + from redis.exceptions import ConnectionError + + calls = [] + bucket = BucketRef("session", 3, 60_000) + key = cluster_store._keys.bucket(bucket) + + async def execute(_node, *args, **kwargs): + calls.append(args) + if len(calls) == 1: + raise ConnectionError("temporary") + return "value" if operation == "read" else 1 + + configure_cluster_node(monkeypatch, cluster_store._client, key, execute) + if operation == "read": + assert await cluster_store._command("get", key, retry="read") == "value" + else: + claim = Claim(bucket, "item", encode_claim_score(1, 123)) + assert await cluster_store.ack(claim) is True + assert len(calls) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redirect", ["MOVED", "ASK"]) +async def test_cluster_redirections_still_work_without_network_replay( + monkeypatch, cluster_store, redirect +): + from redis.asyncio.cluster import ClusterNode + from redis.exceptions import AskError, MovedError + + client = cluster_store._client + key = cluster_store._keys.bucket(BucketRef("session", 3, 60_000)) + destination = ClusterNode("127.0.0.1", 6380, server_type="primary") + calls = [] + + async def execute(node, *args, **kwargs): + calls.append((node.name, args)) + if node.port == 6379: + error = MovedError if redirect == "MOVED" else AskError + raise error(f"{client.keyslot(key)} 127.0.0.1:6380") + return "OK" if args[0] == "ASKING" else 1 + + source = configure_cluster_node(monkeypatch, client, key, execute) + client.nodes_manager.nodes_cache[destination.name] = destination + claim = await cluster_store.claim( + BucketRef("session", 3, 60_000), + BucketItem("item", encode_claim_score(1, 123)), + attempt=1, + started_at_seconds=200, + ) + assert claim is not None + assert claim.claim_score == encode_claim_score(1, 200) + assert [name for name, _ in calls] == ( + [source.name, destination.name] + if redirect == "MOVED" + else [source.name, destination.name, destination.name] + ) + assert [args[0] for _, args in calls] == ( + ["EVALSHA", "EVALSHA"] if redirect == "MOVED" else ["EVALSHA", "ASKING", "EVALSHA"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [RuntimeError("close failed"), asyncio.CancelledError()]) +async def test_client_reference_is_retained_until_close_succeeds(error): + class Client: + def __init__(self): + self.aclose = AsyncMock(side_effect=[error, None]) + + client = Client() + timer_store = store(client) + with pytest.raises(type(error)): + await timer_store.close() + assert timer_store._client is client + await timer_store.close() + assert timer_store._client is None + assert client.aclose.await_count == 2 + + @pytest.mark.asyncio async def test_safe_command_retries_transient_redis_failure(): client = FlakyClient([ConnectionError("temporary")]) @@ -116,13 +361,37 @@ async def test_evalsha_falls_back_to_eval_on_cache_miss(): assert [call[0] for call in client.calls] == ["evalsha", "eval"] +@pytest.mark.asyncio +@pytest.mark.parametrize("delete_metadata", [True, False]) +async def test_unregister_optionally_deletes_metadata(monkeypatch, delete_metadata): + timer_store = store(FlakyClient([])) + command = AsyncMock() + single_key_command = AsyncMock() + monkeypatch.setattr(timer_store, "_command", command) + monkeypatch.setattr(timer_store, "_single_key_command", single_key_command) + + await timer_store.unregister_resource(3, "item", delete_metadata=delete_metadata) + + assert [call.args for call in command.await_args_list] == [ + ("zrem", timer_store._keys.source(3), "item") + ] + if delete_metadata: + single_key_command.assert_awaited_once_with( + "DEL", + timer_store._keys.metadata("item"), + retry="idempotent", + ) + else: + single_key_command.assert_not_awaited() + + @pytest.mark.asyncio async def test_bucket_reset_uses_single_key_cluster_commands(monkeypatch): timer_store = store(FlakyClient([])) monkeypatch.setattr(timer_store, "assert_owner", AsyncMock()) monkeypatch.setattr(timer_store, "read_progress", AsyncMock(return_value=0)) - command = AsyncMock() - monkeypatch.setattr(timer_store, "_command", command) + single_key_command = AsyncMock() + monkeypatch.setattr(timer_store, "_single_key_command", single_key_command) await timer_store.reset_unpublished_minute( OwnerToken(OwnerRole.SCHEDULER, 3, "owner", 999), @@ -131,9 +400,9 @@ async def test_bucket_reset_uses_single_key_cluster_commands(monkeypatch): item_types=("session", "watch"), ) - assert [call.args[:2] for call in command.await_args_list] == [ - ("unlink", timer_store._keys.bucket(BucketRef("session", 3, 60_000))), - ("unlink", timer_store._keys.bucket(BucketRef("watch", 3, 60_000))), + assert [call.args[:2] for call in single_key_command.await_args_list] == [ + ("UNLINK", timer_store._keys.bucket(BucketRef("session", 3, 60_000))), + ("UNLINK", timer_store._keys.bucket(BucketRef("watch", 3, 60_000))), ] diff --git a/tests/scheduling/expiry/test_runtime.py b/tests/scheduling/expiry/test_runtime.py index ea3bfddd57..27b746a4a0 100644 --- a/tests/scheduling/expiry/test_runtime.py +++ b/tests/scheduling/expiry/test_runtime.py @@ -1,11 +1,12 @@ import asyncio +import threading import pytest from openviking.scheduling.expiry import runtime as runtime_module from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig from openviking.scheduling.expiry.models import StoreOperationError -from openviking.scheduling.expiry.runtime import ExpiryRuntime +from openviking.scheduling.expiry.runtime import ExpiryRuntime, ExpiryRuntimeRoles from openviking.scheduling.expiry.service import ExpiryRuntimeState from openviking.service.core import OpenVikingService @@ -71,12 +72,243 @@ def runtime(*, pending=False): result._tasks = {} result._storage_started = True result._storage_closed = False + result._storage_owned = True + result._store_open_attempted = True + result._storage_start_task = None result._store = Store() result._io = IO() result._shard_runner = Runner(pending=pending) return result +@pytest.mark.asyncio +@pytest.mark.parametrize("stage", ["io", "open", "namespace"]) +async def test_cancelled_storage_start_closes_partial_resources(stage): + timer_runtime = ExpiryRuntime( + ExpiryConfig(service_name="startup-test"), roles=ExpiryRuntimeRoles() + ) + entered = asyncio.Event() + original_start = timer_runtime._io.start + + class StartupStore(Store): + async def open(self): + if stage == "open": + entered.set() + await asyncio.Event().wait() + + async def ensure_namespace(self, spec): + if stage == "namespace": + entered.set() + await asyncio.Event().wait() + + async def io_start(): + await original_start() + if stage == "io": + entered.set() + await asyncio.Event().wait() + + timer_runtime._store = StartupStore() + timer_runtime._io.start = io_start + start_task = asyncio.create_task(timer_runtime.start_storage()) + thread = None + try: + await asyncio.wait_for(entered.wait(), timeout=2.0) + thread = timer_runtime._io._thread + start_task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(start_task, timeout=2.0) + assert not timer_runtime._storage_started + assert not timer_runtime._storage_owned + assert timer_runtime._storage_start_task is None + assert not timer_runtime._state.storage_ready + assert thread is not None and not thread.is_alive() + assert timer_runtime._store.close_calls == (0 if stage == "io" else 1) + await timer_runtime.close() + assert timer_runtime._store.close_calls == (0 if stage == "io" else 1) + finally: + await asyncio.gather(start_task, return_exceptions=True) + await timer_runtime.close() + + +@pytest.mark.asyncio +async def test_cancelled_start_before_io_loop_is_ready_can_retry_close(): + timer_runtime = ExpiryRuntime( + ExpiryConfig(service_name="startup-test", shutdown_timeout_seconds=0.03), + roles=ExpiryRuntimeRoles(), + ) + started = threading.Event() + release = threading.Event() + original_thread_main = timer_runtime._io._thread_main + + def delayed_thread_main(): + started.set() + release.wait() + original_thread_main() + + timer_runtime._io._thread_main = delayed_thread_main + start_task = asyncio.create_task(timer_runtime.start_storage()) + try: + assert await asyncio.to_thread(started.wait, 2.0) + thread = timer_runtime._io._thread + start_task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(start_task, timeout=2.0) + assert timer_runtime._storage_owned + assert not timer_runtime._storage_started + assert thread is not None and thread.is_alive() + release.set() + await asyncio.wait_for(timer_runtime.close(), timeout=2.0) + assert not timer_runtime._storage_owned + assert not thread.is_alive() + finally: + release.set() + await asyncio.gather(start_task, return_exceptions=True) + await timer_runtime.close() + + +@pytest.mark.asyncio +async def test_resistant_storage_start_is_retained_for_close_retry(): + timer_runtime = ExpiryRuntime( + ExpiryConfig(service_name="startup-test", shutdown_timeout_seconds=0.02), + roles=ExpiryRuntimeRoles(), + ) + entered = asyncio.Event() + release = asyncio.Event() + + class StartupStore(Store): + async def open(self): + entered.set() + while not release.is_set(): + try: + await release.wait() + except asyncio.CancelledError: + pass + + async def ensure_namespace(self, spec): + pass + + timer_runtime._store = StartupStore() + start_task = asyncio.create_task(timer_runtime.start_storage()) + try: + await asyncio.wait_for(entered.wait(), timeout=2.0) + thread = timer_runtime._io._thread + start_task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(start_task, timeout=2.0) + assert timer_runtime._storage_owned + assert timer_runtime._storage_start_task is not None + assert not timer_runtime._storage_start_task.done() + assert timer_runtime._store.close_calls == 0 + with pytest.raises(StoreOperationError, match="Previous expiry storage startup"): + await timer_runtime.start_storage() + release.set() + await asyncio.wait_for(timer_runtime.close(), timeout=2.0) + assert timer_runtime._store.close_calls == 1 + assert not timer_runtime._storage_owned + assert thread is not None and not thread.is_alive() + finally: + release.set() + await asyncio.gather(start_task, return_exceptions=True) + await timer_runtime.close() + + +@pytest.mark.asyncio +async def test_failed_start_preserves_failure_and_cleans_storage(): + timer_runtime = ExpiryRuntime( + ExpiryConfig(service_name="startup-test"), roles=ExpiryRuntimeRoles() + ) + failure = RuntimeError("namespace failed") + + class StartupStore(Store): + async def open(self): + pass + + async def ensure_namespace(self, spec): + raise failure + + timer_runtime._store = StartupStore() + with pytest.raises(RuntimeError) as error: + await timer_runtime.start_storage() + assert error.value is failure + assert timer_runtime._store.close_calls == 1 + assert not timer_runtime._storage_owned + assert timer_runtime._io._thread is None + + +@pytest.mark.asyncio +async def test_storage_can_start_again_after_partial_start_cleanup(): + timer_runtime = ExpiryRuntime( + ExpiryConfig(service_name="startup-test"), roles=ExpiryRuntimeRoles() + ) + + class StartupStore(Store): + fail_open = True + + async def open(self): + if self.fail_open: + raise RuntimeError("open failed") + + async def ensure_namespace(self, spec): + pass + + timer_runtime._store = StartupStore() + with pytest.raises(RuntimeError, match="open failed"): + await timer_runtime.start_storage() + timer_runtime._store.fail_open = False + try: + await timer_runtime.start_storage() + assert timer_runtime._storage_started + assert timer_runtime._storage_owned + assert timer_runtime._state.storage_ready + await timer_runtime.start_storage() + finally: + await timer_runtime.close() + assert timer_runtime._store.close_calls == 2 + assert not timer_runtime._storage_owned + + +@pytest.mark.asyncio +async def test_cancelled_start_with_failed_cleanup_retains_resources_for_retry(): + timer_runtime = ExpiryRuntime( + ExpiryConfig(service_name="startup-test"), roles=ExpiryRuntimeRoles() + ) + entered = asyncio.Event() + + class StartupStore(Store): + async def open(self): + entered.set() + await asyncio.Event().wait() + + async def close(self): + self.close_calls += 1 + if self.close_calls == 1: + raise RuntimeError("cleanup failed") + + timer_runtime._store = StartupStore() + start_task = asyncio.create_task(timer_runtime.start_storage()) + try: + await asyncio.wait_for(entered.wait(), timeout=2.0) + thread = timer_runtime._io._thread + start_task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(start_task, timeout=2.0) + assert timer_runtime._storage_owned + assert not timer_runtime._storage_closed + assert thread is not None and thread.is_alive() + await timer_runtime.close() + assert timer_runtime._store.close_calls == 2 + assert not timer_runtime._storage_owned + assert not thread.is_alive() + finally: + await asyncio.gather(start_task, return_exceptions=True) + await timer_runtime.close() + + +def test_consumer_role_requires_reporting(): + with pytest.raises(ValueError, match="requires reporting"): + ExpiryRuntimeRoles(consume=True) + + @pytest.mark.asyncio async def test_start_roles_creates_one_callback_task_per_item_type(monkeypatch): created = [] @@ -96,16 +328,14 @@ def freeze(self): timer_runtime = object.__new__(ExpiryRuntime) timer_runtime._storage_started = True timer_runtime._state = ExpiryRuntimeState() - timer_runtime._config = ExpiryConfig.model_construct( - scheduler_enabled=False, - callback_enabled=True, - ) + timer_runtime._config = ExpiryConfig.model_construct() + timer_runtime._roles = ExpiryRuntimeRoles(report=True, consume=True) timer_runtime._registry = Registry() timer_runtime._stop = asyncio.Event() timer_runtime._store = object() timer_runtime._shard_runner = object() timer_runtime._metrics = object() - timer_runtime._scheduler = None + timer_runtime._publisher = None timer_runtime._consumers = {} spawned = {} @@ -124,6 +354,46 @@ def spawn_role(name, operation): operation.close() +@pytest.mark.asyncio +async def test_start_roles_creates_publisher_only_when_selected(monkeypatch): + class Publisher: + def __init__(self, store, runner, config, metrics): + pass + + async def run(self, stop): + await stop.wait() + + class Registry: + def freeze(self): + return {} + + timer_runtime = object.__new__(ExpiryRuntime) + timer_runtime._storage_started = True + timer_runtime._state = ExpiryRuntimeState() + timer_runtime._config = ExpiryConfig.model_construct() + timer_runtime._roles = ExpiryRuntimeRoles(publish=True) + timer_runtime._registry = Registry() + timer_runtime._stop = asyncio.Event() + timer_runtime._store = object() + timer_runtime._shard_runner = object() + timer_runtime._metrics = object() + timer_runtime._publisher = None + timer_runtime._consumers = {} + spawned = {} + + def spawn_role(name, operation): + spawned[name] = operation + + timer_runtime._spawn_role = spawn_role + monkeypatch.setattr(runtime_module, "ExpiryPublisher", Publisher) + + await timer_runtime.start_roles() + + assert timer_runtime._publisher is not None + assert set(spawned) == {"publisher"} + spawned["publisher"].close() + + @pytest.mark.asyncio async def test_stop_roles_clears_cooperative_tasks_before_closing_dependencies(): timer_runtime = runtime() @@ -283,7 +553,7 @@ def __init__(self): self.adapter = object() self.adapter_updates = [] - def set_session_timer_adapter(self, adapter): + def set_session_expiry_adapter(self, adapter): self.adapter_updates.append(adapter) self.adapter = adapter diff --git a/tests/scheduling/expiry/test_session_timer.py b/tests/scheduling/expiry/test_session_expiry.py similarity index 62% rename from tests/scheduling/expiry/test_session_timer.py rename to tests/scheduling/expiry/test_session_expiry.py index da63431b42..5239802237 100644 --- a/tests/scheduling/expiry/test_session_timer.py +++ b/tests/scheduling/expiry/test_session_expiry.py @@ -10,18 +10,20 @@ ExpiryMode, ItemIdentity, ) -from openviking.service.session_timer import ( +from openviking.service.session_expiry import ( + SessionExpiryAdapter, SessionTimeoutHandler, - SessionTimerAdapter, effective_idle_timeout, + install_session_expiry, ) -from openviking.session.session import SessionTimerSnapshot +from openviking.session.session import SessionExpirySnapshot -class Timer: +class Scheduler: def __init__(self): self.reports = [] self.unregistered = [] + self.handlers = [] async def report(self, report): self.reports.append(report) @@ -30,11 +32,36 @@ async def report(self, report): async def unregister(self, identity): self.unregistered.append(identity) + def register_handler(self, item_type, handler, *, options=None): + self.handlers.append((item_type, handler, options)) + + +def test_install_session_expiry_uses_session_defaults(): + scheduler = Scheduler() + sessions = SimpleNamespace(set_session_expiry_adapter=lambda adapter: None) + + install_session_expiry(scheduler, sessions) + + assert len(scheduler.handlers) == 1 + item_type, _, options = scheduler.handlers[0] + assert item_type == "session" + assert options.concurrency == 8 + assert options.metadata_enabled is False + + +def test_install_session_expiry_uses_configured_concurrency(): + scheduler = Scheduler() + sessions = SimpleNamespace(set_session_expiry_adapter=lambda adapter: None) + + install_session_expiry(scheduler, sessions, concurrency=32) + + assert scheduler.handlers[0][2].concurrency == 32 + def test_adapter_builds_due_from_last_message_and_current_policy(): - timer = Timer() - adapter = SessionTimerAdapter(timer) - snapshot = SessionTimerSnapshot( + scheduler = Scheduler() + adapter = SessionExpiryAdapter(scheduler) + snapshot = SessionExpirySnapshot( "a", "u", "s", @@ -48,8 +75,8 @@ def test_adapter_builds_due_from_last_message_and_current_policy(): def test_disabled_policy_produces_no_report(): - adapter = SessionTimerAdapter(Timer()) - snapshot = SessionTimerSnapshot("a", "u", "s", "2026-01-01T00:00:00Z", None) + adapter = SessionExpiryAdapter(Scheduler()) + snapshot = SessionExpirySnapshot("a", "u", "s", "2026-01-01T00:00:00Z", None) assert adapter.build_report(snapshot) is None assert effective_idle_timeout({"idle_timeout_seconds": 0}) is None @@ -57,7 +84,7 @@ def test_disabled_policy_produces_no_report(): @pytest.mark.asyncio @pytest.mark.parametrize("reason", ["idle_cooldown", "idle_activity"]) async def test_handler_reports_deferred_before_completing_old_event(reason): - timer = Timer() + scheduler = Scheduler() class Sessions: async def commit_on_timeout(self, session_id, ctx): @@ -69,9 +96,9 @@ async def commit_on_timeout(self, session_id, ctx): } identity = ItemIdentity("session", "a", "u", "s") - result = await SessionTimeoutHandler(Sessions(), timer).handle(CallbackRequest(identity)) + result = await SessionTimeoutHandler(Sessions(), scheduler).handle(CallbackRequest(identity)) assert result.status is CallbackStatus.COMPLETED - assert timer.reports[0].due_at_ms == 999 + assert scheduler.reports[0].due_at_ms == 999 @pytest.mark.asyncio @@ -80,7 +107,7 @@ class Sessions: async def commit_on_timeout(self, session_id, ctx): return {"status": "skipped", "reason": "not_found"} - result = await SessionTimeoutHandler(Sessions(), Timer()).handle( + result = await SessionTimeoutHandler(Sessions(), Scheduler()).handle( CallbackRequest(ItemIdentity("session", "a", "u", "s")) ) assert result.status is CallbackStatus.COMPLETED @@ -98,17 +125,17 @@ async def commit_on_timeout(self, session_id, ctx): "timeout_seconds": 30, } - timer = Timer() + scheduler = Scheduler() error = RuntimeError("reschedule unavailable") - timer.report = AsyncMock(spec=timer.report, side_effect=error) + scheduler.report = AsyncMock(spec=scheduler.report, side_effect=error) identity = ItemIdentity("session", "a", "u", "s") with pytest.raises(RuntimeError, match="reschedule unavailable") as caught: - await SessionTimeoutHandler(Sessions(), timer).handle(CallbackRequest(identity)) + await SessionTimeoutHandler(Sessions(), scheduler).handle(CallbackRequest(identity)) assert caught.value is error - timer.report.assert_awaited_once() - report = timer.report.await_args.args[0] + scheduler.report.assert_awaited_once() + report = scheduler.report.await_args.args[0] assert report.identity == identity assert report.due_at_ms == 999 assert report.timeout_seconds == 30 @@ -124,10 +151,10 @@ async def commit_on_timeout(self, session_id, ctx): calls.append((session_id, ctx)) return {"status": "accepted", "task_id": "task"} - timer = Timer() + scheduler = Scheduler() identity = ItemIdentity("session", "account", "user", "s") - result = await SessionTimeoutHandler(Sessions(), timer).handle(CallbackRequest(identity)) + result = await SessionTimeoutHandler(Sessions(), scheduler).handle(CallbackRequest(identity)) assert result.status is CallbackStatus.COMPLETED assert len(calls) == 1 @@ -135,7 +162,7 @@ async def commit_on_timeout(self, session_id, ctx): assert session_id == "s" assert ctx.account_id == "account" assert ctx.user.user_id == "user" - assert timer.reports == [] + assert scheduler.reports == [] @pytest.mark.asyncio @@ -145,13 +172,13 @@ class Sessions: async def commit_on_timeout(self, session_id, ctx): return {"status": "skipped", "reason": reason} - timer = Timer() - result = await SessionTimeoutHandler(Sessions(), timer).handle( + scheduler = Scheduler() + result = await SessionTimeoutHandler(Sessions(), scheduler).handle( CallbackRequest(ItemIdentity("session", "a", "u", "s")) ) assert result.status is CallbackStatus.COMPLETED - assert timer.reports == [] + assert scheduler.reports == [] @pytest.mark.asyncio @@ -160,25 +187,25 @@ class Sessions: async def commit_on_timeout(self, session_id, ctx): raise RuntimeError("source commit failed") - timer = Timer() + scheduler = Scheduler() with pytest.raises(RuntimeError, match="source commit failed"): - await SessionTimeoutHandler(Sessions(), timer).handle( + await SessionTimeoutHandler(Sessions(), scheduler).handle( CallbackRequest(ItemIdentity("session", "a", "u", "s")) ) - assert timer.reports == [] + assert scheduler.reports == [] @pytest.mark.asyncio async def test_adapter_append_reports_current_snapshot_identity_and_policy(): - timer = Timer() - snapshot = SessionTimerSnapshot( + scheduler = Scheduler() + snapshot = SessionExpirySnapshot( "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 120} ) - await SessionTimerAdapter(timer).report_after_append(snapshot) + await SessionExpiryAdapter(scheduler).report_after_append(snapshot) - assert len(timer.reports) == 1 - report = timer.reports[0] + assert len(scheduler.reports) == 1 + report = scheduler.reports[0] assert report.identity == ItemIdentity("session", "a", "u", "s") assert report.due_at_ms == 1767225720000 assert report.timeout_seconds == 120 @@ -187,7 +214,7 @@ async def test_adapter_append_reports_current_snapshot_identity_and_policy(): @pytest.fixture def adapter_logs(caplog): - from openviking.service.session_timer import logger + from openviking.service.session_expiry import logger logger.addHandler(caplog.handler) try: @@ -198,26 +225,28 @@ def adapter_logs(caplog): @pytest.mark.asyncio async def test_adapter_append_report_failure_is_best_effort(adapter_logs): - timer = Timer() - timer.report = AsyncMock(spec=timer.report, side_effect=RuntimeError("redis unavailable")) - snapshot = SessionTimerSnapshot( + scheduler = Scheduler() + scheduler.report = AsyncMock( + spec=scheduler.report, side_effect=RuntimeError("redis unavailable") + ) + snapshot = SessionExpirySnapshot( "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 60} ) - await SessionTimerAdapter(timer).report_after_append(snapshot) + await SessionExpiryAdapter(scheduler).report_after_append(snapshot) - timer.report.assert_awaited_once() + scheduler.report.assert_awaited_once() assert "report_failed" in adapter_logs.text @pytest.mark.asyncio async def test_adapter_append_report_cancellation_propagates(): - timer = Timer() - timer.report = AsyncMock(spec=timer.report, side_effect=asyncio.CancelledError()) - snapshot = SessionTimerSnapshot( + scheduler = Scheduler() + scheduler.report = AsyncMock(spec=scheduler.report, side_effect=asyncio.CancelledError()) + snapshot = SessionExpirySnapshot( "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 60} ) with pytest.raises(asyncio.CancelledError): - await SessionTimerAdapter(timer).report_after_append(snapshot) - timer.report.assert_awaited_once() + await SessionExpiryAdapter(scheduler).report_after_append(snapshot) + scheduler.report.assert_awaited_once() diff --git a/tests/scheduling/expiry/test_session_timeout_service.py b/tests/scheduling/expiry/test_session_timeout_service.py index 589a4f11c4..712b70bb6c 100644 --- a/tests/scheduling/expiry/test_session_timeout_service.py +++ b/tests/scheduling/expiry/test_session_timeout_service.py @@ -87,7 +87,10 @@ async def read_file(self, uri, *, ctx, **kwargs): async def write_file(self, uri, content, *, ctx, **kwargs): assert uri == self.meta_uri assert self.locked - assert kwargs.get("lease_ref") == "lease", "Metadata writes must reuse the held lease" + assert kwargs.get("lease_ref") is None, ( + "Child .meta.json must take its own auto-pathlock under the held " + "session-directory lock, not reuse the directory lease" + ) assert self.exists, "Recording acceptance must not recreate a deleted session" self.events.append("write") if self.write_error is not None: @@ -603,11 +606,11 @@ async def test_default_legacy_backend_starts_scheduler_without_expiry(legacy_ser assert legacy_service._session_auto_commit_scheduler is not None assert legacy_service._session_auto_commit_scheduler._running is True assert legacy_service._expiry_runtime is None - assert legacy_service.sessions._session_timer_adapter is None + assert legacy_service.sessions._session_expiry_adapter is None @pytest.fixture -def enable_timer_without_business_work(monkeypatch): +def configure_expiry_without_enabling_redis_backend(monkeypatch): from openviking.service import core original_get_config = core.get_openviking_config @@ -617,7 +620,6 @@ def enabled_config(): config.memory.session_auto_commit = SessionAutoCommitConfig(enabled=False) config.expiry = ExpiryConfig( service_name="unused", - reporting_enabled=True, redis={"endpoints": ["127.0.0.1:1"]}, ) return config @@ -626,12 +628,12 @@ def enabled_config(): @pytest.fixture -def timer_without_business_work_service(enable_timer_without_business_work, service): +def expiry_without_redis_backend_service(configure_expiry_without_enabling_redis_backend, service): return service @pytest.mark.asyncio -async def test_enabled_timer_without_business_work_does_not_start_runtime( - timer_without_business_work_service, +async def test_expiry_config_without_redis_backend_does_not_start_runtime( + expiry_without_redis_backend_service, ): - assert timer_without_business_work_service._expiry_runtime is None + assert expiry_without_redis_backend_service._expiry_runtime is None diff --git a/tests/server/test_agent_evolution_global_setting.py b/tests/server/test_agent_evolution_global_setting.py index 9abb816441..c37ddda874 100644 --- a/tests/server/test_agent_evolution_global_setting.py +++ b/tests/server/test_agent_evolution_global_setting.py @@ -7,6 +7,7 @@ import httpx import pytest import pytest_asyncio +from pydantic import ValidationError from openviking.pyagfs import AGFSNotFoundError from openviking.server.app import create_app @@ -200,9 +201,7 @@ async def test_server_agent_evolution_updates_initialized_runtime_baseline(): OpenVikingConfigSingleton.reset_instance() -def test_server_default_memory_policy_is_configured_on_session_service( - fake_viking_fs, monkeypatch -): +def test_server_default_memory_policy_is_configured_on_session_service(fake_viking_fs, monkeypatch): monkeypatch.setattr( "openviking.service.session_service.get_default_registry", lambda: SimpleNamespace(list_names=lambda **_: ["profile"]), @@ -250,6 +249,22 @@ async def test_auto_commit_defaults_apply_to_new_sessions(fake_viking_fs, monkey assert session.meta.auto_commit_policy == expected +def test_deployment_auto_commit_default_rejects_idle_timeout_below_five_minutes(): + with pytest.raises(ValidationError, match="must be 0 or at least 300"): + ServerConfig(user_config_defaults={"auto_commit_policy": {"idle_timeout_seconds": 299}}) + + +@pytest.mark.parametrize("idle_timeout_seconds", [0, 300]) +def test_deployment_auto_commit_default_accepts_valid_idle_timeout(idle_timeout_seconds): + config = ServerConfig( + user_config_defaults={"auto_commit_policy": {"idle_timeout_seconds": idle_timeout_seconds}} + ) + + policy = config.user_config_defaults.auto_commit_policy + assert policy is not None + assert policy["idle_timeout_seconds"] == idle_timeout_seconds + + async def test_existing_session_observes_updated_account_value(fake_viking_fs): runtime_config = _FakeRuntimeConfig() service = SessionService(viking_fs=fake_viking_fs) @@ -316,9 +331,7 @@ async def test_account_settings_admin_api_reads_and_updates_effective_value( "agent_evolution": {"enabled": True}, "acl": {"enabled": True}, } - assert ( - await service.runtime_config_manager.get_account("default", "agent_evolution") - ).enabled + assert (await service.runtime_config_manager.get_account("default", "agent_evolution")).enabled assert (await service.runtime_config_manager.get_account("default", "acl")).enabled assert await service.viking_fs.acl_manager.is_enabled("default") @@ -380,9 +393,7 @@ async def test_account_configuration_exposes_three_state_layer(settings_http): json={"settings": {"acl": None}}, ) assert removed.status_code == 200, removed.text - assert removed.json()["result"]["settings"] == { - "github": {"token": "account-token"} - } + assert removed.json()["result"]["settings"] == {"github": {"token": "account-token"}} from openviking.server.auth import get_request_context @@ -423,9 +434,7 @@ async def test_cluster_agent_evolution_override_is_account_fallback(settings_htt json={"settings": {"agent_evolution": {"enabled": True}}}, ) assert updated.status_code == 200, updated.text - assert updated.json()["result"]["settings"] == { - "agent_evolution": {"enabled": True} - } + assert updated.json()["result"]["settings"] == {"agent_evolution": {"enabled": True}} assert await service.sessions.get_agent_evolution_enabled("default") await service.runtime_config_manager.patch_account( @@ -433,9 +442,7 @@ async def test_cluster_agent_evolution_override_is_account_fallback(settings_htt ) assert not await service.sessions.get_agent_evolution_enabled("default") - await service.runtime_config_manager.patch_account( - "default", {"agent_evolution": None} - ) + await service.runtime_config_manager.patch_account("default", {"agent_evolution": None}) assert await service.sessions.get_agent_evolution_enabled("default") @@ -448,9 +455,9 @@ async def test_legacy_admin_configuration_routes_are_deprecated(settings_http): legacy = schema["paths"]["/api/v1/admin/accounts/{account_id}/settings"] assert legacy["get"]["deprecated"] assert legacy["patch"]["deprecated"] - assert not schema["paths"][ - "/api/v1/admin/accounts/{account_id}/configuration" - ]["patch"].get("deprecated", False) + assert not schema["paths"]["/api/v1/admin/accounts/{account_id}/configuration"]["patch"].get( + "deprecated", False + ) assert "/api/v1/admin/configuration" in schema["paths"] assert "/api/v1/admin/settings" not in schema["paths"] @@ -485,9 +492,7 @@ async def test_agent_evolution_endpoint_keeps_name_and_uses_account_settings( assert response.status_code == 200, response.text assert response.json()["result"]["enabled"] is True - assert ( - await service.runtime_config_manager.get_account("default", "agent_evolution") - ).enabled + assert (await service.runtime_config_manager.get_account("default", "agent_evolution")).enabled def test_deprecated_user_agent_evolution_config_is_not_persisted(): diff --git a/tests/server/test_api_sessions.py b/tests/server/test_api_sessions.py index 9226df929d..fc4ff53985 100644 --- a/tests/server/test_api_sessions.py +++ b/tests/server/test_api_sessions.py @@ -676,6 +676,75 @@ async def test_auto_commit_policy_rejects_null_fields(client: httpx.AsyncClient) assert "auto_commit_policy=null" in resp.json()["error"]["message"] +@pytest.mark.parametrize("idle_timeout_seconds", [-1, 1, 299]) +async def test_auto_commit_policy_rejects_new_idle_timeout_below_five_minutes( + client: httpx.AsyncClient, + idle_timeout_seconds: int, +): + resp = await client.post( + "/api/v1/sessions", + json={"auto_commit_policy": {"idle_timeout_seconds": idle_timeout_seconds}}, + ) + + assert resp.status_code == 400 + assert "must be 0 or at least 300" in resp.json()["error"]["message"] + + +@pytest.mark.parametrize("idle_timeout_seconds", [0, 300]) +async def test_auto_commit_policy_accepts_disabled_or_five_minute_idle_timeout( + client: httpx.AsyncClient, + idle_timeout_seconds: int, +): + resp = await client.post( + "/api/v1/sessions", + json={"auto_commit_policy": {"idle_timeout_seconds": idle_timeout_seconds}}, + ) + + assert resp.status_code == 200 + assert ( + resp.json()["result"]["auto_commit_policy"]["idle_timeout_seconds"] == idle_timeout_seconds + ) + + +async def test_update_auto_commit_policy_rejects_idle_timeout_below_five_minutes( + client: httpx.AsyncClient, +): + created = await client.post("/api/v1/sessions", json={}) + session_id = created.json()["result"]["session_id"] + + updated = await client.patch( + f"/api/v1/sessions/{session_id}/config", + json={"auto_commit_policy": {"idle_timeout_seconds": 299}}, + ) + + assert updated.status_code == 400 + assert "must be 0 or at least 300" in updated.json()["error"]["message"] + + +async def test_updating_other_policy_field_preserves_legacy_idle_timeout( + client: httpx.AsyncClient, + service, +): + created = await client.post("/api/v1/sessions", json={}) + session_id = created.json()["result"]["session_id"] + ctx = RequestContext(user=DEFAULT_USER, role=Role.ADMIN) + session = await service.sessions.get(session_id, ctx) + await session.update_config( + auto_commit_policy={"idle_timeout_seconds": 60}, + update_auto_commit_policy=True, + ) + + updated = await client.patch( + f"/api/v1/sessions/{session_id}/config", + json={"auto_commit_policy": {"message_count_threshold": 25}}, + ) + + assert updated.status_code == 200 + policy = updated.json()["result"]["auto_commit_policy"] + assert policy["idle_timeout_seconds"] == 60 + assert policy["message_count_threshold"] == 25 + + async def test_auto_created_session_uses_default_policy_when_server_default_enabled( client: httpx.AsyncClient, service, diff --git a/tests/server/test_bootstrap.py b/tests/server/test_bootstrap.py index 9241411ec2..e2da91c2d2 100644 --- a/tests/server/test_bootstrap.py +++ b/tests/server/test_bootstrap.py @@ -37,10 +37,9 @@ def test_main_keeps_config_host_when_cli_host_is_omitted(monkeypatch): lambda: None, ) monkeypatch.setattr( - bootstrap, - "OpenVikingConfigSingleton", - SimpleNamespace(initialize=lambda config_path: None), - raising=False, + OpenVikingConfigSingleton, + "initialize", + classmethod(lambda cls, config_path: None), ) monkeypatch.setattr( bootstrap.argparse.ArgumentParser, @@ -91,10 +90,9 @@ def test_main_coerces_cli_host_all_to_none(monkeypatch): lambda: None, ) monkeypatch.setattr( - bootstrap, - "OpenVikingConfigSingleton", - SimpleNamespace(initialize=lambda config_path: None), - raising=False, + OpenVikingConfigSingleton, + "initialize", + classmethod(lambda cls, config_path: None), ) monkeypatch.setattr( bootstrap.argparse.ArgumentParser, diff --git a/tests/session/test_session_commit.py b/tests/session/test_session_commit.py index 946f7b4498..5b046d0aed 100644 --- a/tests/session/test_session_commit.py +++ b/tests/session/test_session_commit.py @@ -34,7 +34,7 @@ async def _marker_exists(session, archive_uri: str, name: str) -> bool: return False -class _RecordingTimerHook: +class _RecordingExpiryReporter: def __init__(self) -> None: self.snapshots = [] self.failures = [] @@ -52,8 +52,8 @@ class TestCommit: async def test_append_reports_repaired_meta_after_authoritative_reload_failure( self, session_with_messages: Session, monkeypatch ): - hook = _RecordingTimerHook() - session_with_messages._timer_hook = hook + reporter = _RecordingExpiryReporter() + session_with_messages._expiry_reporter = reporter session_with_messages.meta.auto_commit_policy = {"idle_timeout_seconds": 60} await session_with_messages._save_meta() original_read_file = session_with_messages._viking_fs.read_file @@ -72,10 +72,10 @@ async def fail_meta_reload_once(uri, *args, **kwargs): await session_with_messages.add_message_async("user", [TextPart("latest activity")]) assert failed is True - assert len(hook.snapshots) == 1 - assert hook.snapshots[0].last_message_at == session_with_messages.meta.last_message_at - assert hook.snapshots[0].auto_commit_policy == {"idle_timeout_seconds": 60} - assert hook.failures == [] + assert len(reporter.snapshots) == 1 + assert reporter.snapshots[0].last_message_at == session_with_messages.meta.last_message_at + assert reporter.snapshots[0].auto_commit_policy == {"idle_timeout_seconds": 60} + assert reporter.failures == [] async def test_service_timeout_archives_append_after_check_without_idle_phase1_extensions( self, session_with_messages: Session, service, request_context, monkeypatch diff --git a/tests/unit/session/test_auto_commit_policy.py b/tests/unit/session/test_auto_commit_policy.py index c24256c526..e94895030a 100644 --- a/tests/unit/session/test_auto_commit_policy.py +++ b/tests/unit/session/test_auto_commit_policy.py @@ -14,7 +14,9 @@ MAX_IDLE_TIMEOUT_SECONDS, MAX_MESSAGE_COUNT_THRESHOLD, MAX_PENDING_TOKEN_THRESHOLD, + MIN_IDLE_TIMEOUT_SECONDS, AutoCommitPolicy, + validate_idle_timeout_input, ) from openviking_cli.exceptions import InvalidArgumentError @@ -83,6 +85,24 @@ def test_from_dict_clamps_negatives_to_zero(): } +def test_from_dict_preserves_legacy_idle_timeout_below_write_minimum(): + policy = AutoCommitPolicy.from_dict({"idle_timeout_seconds": 60}) + + assert policy.idle_timeout_seconds == 60 + + +@pytest.mark.parametrize("value", [0, 300, 604800]) +def test_validate_idle_timeout_input_accepts_zero_or_at_least_minimum(value): + assert validate_idle_timeout_input(value) == value + assert MIN_IDLE_TIMEOUT_SECONDS == 300 + + +@pytest.mark.parametrize("value", [-1, 1, 299]) +def test_validate_idle_timeout_input_rejects_nonzero_values_below_minimum(value): + with pytest.raises(ValueError, match="must be 0 or at least 300"): + validate_idle_timeout_input(value) + + def test_from_dict_rejects_unknown_keys(): with pytest.raises(InvalidArgumentError): AutoCommitPolicy.from_dict({"enabled": True}) diff --git a/tests/unit/test_server_bootstrap_expiry_publisher.py b/tests/unit/test_server_bootstrap_expiry_publisher.py new file mode 100644 index 0000000000..288a612a67 --- /dev/null +++ b/tests/unit/test_server_bootstrap_expiry_publisher.py @@ -0,0 +1,200 @@ +import os +import signal +import subprocess +import sys +import textwrap +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import openviking.server.bootstrap as bootstrap + + +class FakeProcess: + pid = 4321 + + def __init__(self, *, returncode=None): + self.returncode = returncode + self.terminated = False + self.killed = False + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + return 0 if self.returncode is None else self.returncode + + def terminate(self): + self.terminated = True + self.returncode = 0 + + def kill(self): + self.killed = True + self.returncode = -9 + + +class FakeThread: + def __init__(self, *, target, args, name, daemon): + self.target = target + self.args = args + self.name = name + self.daemon = daemon + self.started = False + + def start(self): + self.started = True + + def join(self, timeout=None): + pass + + +def test_start_expiry_publisher_uses_internal_module_and_resolved_config(monkeypatch): + captured = {} + process = FakeProcess() + + def popen(command, **kwargs): + captured["command"] = command + captured["kwargs"] = kwargs + return process + + monkeypatch.setattr(bootstrap.subprocess, "Popen", popen) + monkeypatch.setattr(bootstrap, "_wait_for_expiry_publisher_ready", lambda *_: None) + monkeypatch.setattr(bootstrap.threading, "Thread", FakeThread) + + managed = bootstrap._start_expiry_publisher(Path("/tmp/ov.conf")) + + assert captured["command"][1:3] == ["-m", "openviking_cli.expiry_bootstrap"] + assert captured["command"][captured["command"].index("--config") + 1] == "/tmp/ov.conf" + assert "--startup-status" in captured["command"] + assert captured["kwargs"]["start_new_session"] == (bootstrap.os.name != "nt") + assert managed.process is process + assert managed.monitor_thread is not None + assert managed.monitor_thread.started is True + + +def test_monitor_stops_parent_when_publisher_exits_unexpectedly(monkeypatch): + process = FakeProcess(returncode=7) + managed = bootstrap.ExpiryPublisherProcess(process) + signals = [] + monkeypatch.setattr(bootstrap.os, "kill", lambda pid, signum: signals.append((pid, signum))) + + bootstrap._monitor_expiry_publisher(managed) + + assert signals == [(bootstrap.os.getpid(), bootstrap.signal.SIGTERM)] + + +def test_monitor_ignores_expected_publisher_shutdown(monkeypatch): + process = FakeProcess(returncode=0) + managed = bootstrap.ExpiryPublisherProcess(process) + managed.stopping.set() + signals = [] + monkeypatch.setattr(bootstrap.os, "kill", lambda pid, signum: signals.append((pid, signum))) + + bootstrap._monitor_expiry_publisher(managed) + + assert signals == [] + + +def test_readiness_surfaces_child_error(tmp_path): + status_path = tmp_path / "status.json" + status_path.write_text('{"status":"failed","error":"redis unavailable"}', encoding="utf-8") + process = SimpleNamespace(poll=lambda: None) + + try: + bootstrap._wait_for_expiry_publisher_ready( + process, + status_path, + timeout_seconds=0.1, + ) + except RuntimeError as exc: + assert "redis unavailable" in str(exc) + else: + raise AssertionError("readiness failure was not propagated") + + +@pytest.mark.skipif(os.name == "nt", reason="SIGTERM lifecycle is POSIX-specific") +def test_sigterm_runs_outer_finally_and_stops_detached_process(tmp_path): + ready_path = tmp_path / "ready" + cleanup_path = tmp_path / "cleaned" + child_pid_path = tmp_path / "child.pid" + script = textwrap.dedent( + """ + import subprocess + import sys + from contextlib import asynccontextmanager + from pathlib import Path + + import uvicorn + from fastapi import FastAPI + + from openviking.server.bootstrap import _sigterm_runs_finally + + ready_path = Path(sys.argv[1]) + cleanup_path = Path(sys.argv[2]) + child_pid_path = Path(sys.argv[3]) + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(300)"], + start_new_session=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + child_pid_path.write_text(str(child.pid), encoding="utf-8") + + @asynccontextmanager + async def lifespan(_app): + ready_path.write_text("ready", encoding="utf-8") + yield + + app = FastAPI(lifespan=lifespan) + try: + with _sigterm_runs_finally(): + uvicorn.run(app, host="127.0.0.1", port=0, log_level="error") + finally: + cleanup_path.write_text("cleaned", encoding="utf-8") + if child.poll() is None: + child.terminate() + child.wait(timeout=5) + """ + ) + process = subprocess.Popen( + [ + sys.executable, + "-c", + script, + str(ready_path), + str(cleanup_path), + str(child_pid_path), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + child_pid = None + output = "" + try: + deadline = time.monotonic() + 60 + while not ready_path.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.05) + assert ready_path.exists(), "Uvicorn child did not enter application lifespan" + child_pid = int(child_pid_path.read_text(encoding="utf-8")) + + process.send_signal(signal.SIGTERM) + output, _ = process.communicate(timeout=20) + + assert process.returncode == 128 + signal.SIGTERM, output + assert cleanup_path.exists(), output + with pytest.raises(ProcessLookupError): + os.kill(child_pid, 0) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + if child_pid is not None: + try: + os.kill(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass From 4050269e6568f36bb30d731f8377f526204d77b2 Mon Sep 17 00:00:00 2001 From: lujiakun Date: Wed, 14 Oct 2026 14:46:37 +0800 Subject: [PATCH 3/5] chore: remove design docs and rename expiry config test - Remove docs/zh/design session-idle-auto-commit scheduler design docs - Rename tests/scheduling/expiry/test_config.py to test_expiry_config.py to avoid pytest module-name collision with tests/unit/crypto/test_config.py Co-authored-by: TRAE CLI --- ...-idle-auto-commit-scheduler-code-design.md | 2795 ----------------- ...n-idle-auto-commit-scheduler-vibecoding.md | 453 --- .../{test_config.py => test_expiry_config.py} | 0 3 files changed, 3248 deletions(-) delete mode 100644 docs/zh/design/session-idle-auto-commit-scheduler-code-design.md delete mode 100644 docs/zh/design/session-idle-auto-commit-scheduler-vibecoding.md rename tests/scheduling/expiry/{test_config.py => test_expiry_config.py} (100%) diff --git a/docs/zh/design/session-idle-auto-commit-scheduler-code-design.md b/docs/zh/design/session-idle-auto-commit-scheduler-code-design.md deleted file mode 100644 index f09367f522..0000000000 --- a/docs/zh/design/session-idle-auto-commit-scheduler-code-design.md +++ /dev/null @@ -1,2795 +0,0 @@ -# Session Redis 计时调度：逐文件代码设计 - -本文配套 [VibeCoding 实施文档](session-idle-auto-commit-scheduler-vibecoding.md)，把实现方案展开为可审阅的代码骨架。业务仍只使用三个接口；骨架展示签名和关键控制流，不代替可运行实现或行为验收。 - -本轮以用户修正为准：报告 R→M，无伴随状态或资源更新版本；耗尽仅日志；逐 shard 临时抢占，TTL 120 秒、20 秒续租，失败停止旧任务。冷却未满足经统一 report 重新计时。元数据仍可缺失，跨槽未决请求仍有残余风险，不能用取消任务宣称完全隔离旧写入。 - -代码基线为 `7d77e3d68fea82070a0b700a9acd64c40e7f4a9a`，本轮更新时间为2026-10-11。Redis optional、默认legacy不变，Redis协议与业务三个接口沿用已确认设计。最新服务层方案覆盖旧idle专用提交/恢复设计：外层加载和纯检查通过即调用原 `Session.commit_async`，不做持锁最终idle检查，期间新消息允许一起提交。accepted后服务层锁内重读最新meta，仅best-effort patch接受时间与updated_at；失败仍accepted、删除不重建。没有idle归档指针或可修复缓存，不承诺严格空闲或零重复。原通用提交的ready、QueueFS、恢复与WM契约继续保留。 - -## 1. 阅读方式、依赖和文件目录 - -- 新文件列出计划中的全部类、数据字段、公开接口及主要私有函数，包括 Lua 入口。 -- 已有文件只列本次新增或需要修改的函数；未涉及的已有函数继续保留，不在此复制整个内核。 -- `...` 表示实现体待编码，但参数、返回值、异常、锁/线程边界由注释约定。关键交接流程给出具体控制流。 -- 片段中的既有类型沿用项目 import；新增类型来自本文指定模块。本文做语法和接口一致性检查，不宣称这些骨架已经可以运行。 -- 字段和函数名以本附录为具体建议；如与主文档的简略伪代码不同，以这里展开的签名为准，业务约束仍以主文档为准。 - -```text -openviking/scheduling/expiry/ - __init__.py # 只导出 api.py 的业务契约，不导出实现类 - api.py # 三个业务接口、业务 DTO、处理器协议及注册选项 - models.py # 调度/消费内部数据结构、校验和纯计算函数 - config.py # 轻量配置定义，独立进程不经过聚合配置包 - keys.py # 身份编码、hash、key 和注册项寻址 - metrics.py # timer 指标的轻量包装 - service.py # 私有 facade 实现，只由 runtime 构造 - registry.py # 内部处理器注册表及不可变注册快照 - redis_store.py # Redis 数据操作；所有网络 I/O 经专属 loop - publisher.py # 分页扫描、分钟构建和发布 - consumer.py # 已注册桶发现、claim、业务回调、条件 ACK - leases.py # 仅 try_acquire/renew/release：Redis token 操作 - shard_runner.py # 单 shard 执行、后台 heartbeat、撤销/有界 join/pending 强引用 - runtime.py # 构造共享 runner、Redis loop 与有界启停 - lua/ - _common.lua - ensure_metadata.lua - upsert_resource.lua - source_time.lua - read_source_page.lua - publish_minute.lua - read_registry_page.lua - claim.lua - ack.lua - acquire_owner.lua - renew_owner.lua - release_owner.lua -openviking/service/session_expiry.py # session 业务适配 -openviking/session/session.py # append reporter/接受时间字段，原通用commit不拆分 -openviking/service/session_service.py # 加载、纯检查、原commit调用及best-effort局部patch -openviking/service/core.py # 启停顺序与旧 backend 选择 -openviking_cli/utils/config/expiry_config.py -openviking_cli/utils/config/open_viking_config.py -openviking_cli/utils/config/memory_config.py -openviking_cli/expiry_bootstrap.py -pyproject.toml -``` - -模块分层从底向上为 `api → models/keys → store/registry/service → leases → shard_runner → publisher/consumer → runtime`，上层引用下层，api 不反向 import 实现模块。Coordinator 只封装 token 操作；runner 只负责一段单 shard 工作的租约执行生命周期；遍历 shard 和调度节奏由具体 worker 自己持有，不引入通用 worker 框架。业务模块通过包入口只依赖 ExpiryScheduler 协议和业务 DTO，启动组装层才依赖 runtime。session 适配另外依赖自身业务的 SessionService。Session 本身只持有自己定义的 reporter 协议，不 import session_expiry。runtime 不 import SessionService，内部 publisher 子进程不加载业务 handler。 - -两个 loop 的责任明确分开：Redis 客户端、连接池和原始 Redis 命令只在 Redis I/O loop；业务 handler、session commit、scheduler/consumer 的控制流在它们的运行 loop（API 中为 API loop）。report 可以从不同调用 loop 发起，store 负责桥接网络 I/O，不能将整个 session commit 丢进 Redis loop。 - -## 2. api.py 与 models.py：业务契约和内部模型分开 - -### 2.1 api.py：唯一稳定的业务契约 - -业务只接收 ExpiryScheduler 实例，不构造实现类。下面的三个方法是全部业务操作；没有启动、freeze、Redis client、队列领取或 ACK 接口。 - -```python -from __future__ import annotations - -from dataclasses import dataclass -from enum import Enum -from typing import Protocol - - -SESSION_ITEM_TYPE = "session" -SUPPORTED_ITEM_TYPES: frozenset[str] = frozenset({SESSION_ITEM_TYPE}) - - -class ExpiryMode(str, Enum): - IDLE_ONESHOT = "idle_oneshot" - FIXED = "fixed" # 本期预留；尚不支持时明确报错。 - - -class CallbackStatus(str, Enum): - COMPLETED = "completed" - RETRY = "retry" - - -class ExpiryError(RuntimeError): - """公开错误只含业务可理解的错误码和是否可重试，不提供内部执行凭据。""" - - def __init__(self, code: str, message: str, *, retryable: bool = False) -> None: - """参数/注册错误不可重试；暂时不可用由调用者执行自己的失败策略。""" - super().__init__(message) - self.code = code - self.retryable = retryable - - -@dataclass(frozen=True) -class ItemIdentity: - item_type: str - account_id: str - user_id: str - local_id: str - - -@dataclass(frozen=True) -class ExpiryRegistration: - identity: ItemIdentity - due_at_ms: int - timeout_seconds: int - mode: ExpiryMode = ExpiryMode.IDLE_ONESHOT - - -@dataclass(frozen=True) -class ExpiryRegistrationResult: - accepted_due_at_ms: int # 服务接受的到期时间；不是实际入队时间，不返回内部执行凭据。 - - -@dataclass(frozen=True) -class CallbackRequest: - identity: ItemIdentity # 业务用此身份回源；不携带元数据缓存是否命中的标志。 - - -@dataclass(frozen=True) -class CallbackResult: - status: CallbackStatus - reason: str - - -@dataclass(frozen=True) -class HandlerOptions: - concurrency: int = 4 # 通用默认值；各业务在启动注册时可指定，运行中不可动态更改。 - max_attempts: int = 3 # 包含首次；session 采用 3。相同类型各回调副本必须一致。 - - -class ExpiryHandler(Protocol): - async def handle(self, request: CallbackRequest) -> CallbackResult: - """只执行业务操作；返回完成或重试。队列重领、超时和清理由框架负责。""" - ... - - -class ExpiryScheduler(Protocol): - async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: - """写入或更新计时；业务提供身份、到期时间和 timeout，等待本次操作结果。""" - ... - - async def unregister(self, identity: ItemIdentity) -> None: - """注销当前计时；重复调用幂等，已交付处理的任务按业务当前状态完成。""" - ... - - def register_handler(self, item_type: str, handler: ExpiryHandler, *, - options: HandlerOptions | None = None) -> None: - """业务模块启动时注册类型及处理器；重复注册或运行中修改明确报错。""" - ... -``` - -HandlerOptions 表达业务处理能力，不暴露队列实现。首期只支持 session，`SUPPORTED_ITEM_TYPES` 是上报、Handler 注册和调度恢复共用的代码清单。report 不做请求顺序比较，接口不再返回重复或陈旧状态。 - -### 2.2 models.py：不向业务导出的内部类型 - -这些类型仅供框架内部使用，包入口不会导出 Claim、BucketRef、OwnerToken、游标或 Redis 状态。内部模块明确从 api.py 引用公开 DTO，不再把公开/私有模型混放。 - -```python -from __future__ import annotations - -import asyncio -import time -from dataclasses import dataclass, field -from enum import Enum -from typing import Awaitable, Callable, Literal, Protocol, TypeVar -from .api import ItemIdentity, ExpiryRegistration, ExpiryError, ExpiryMode - -T = TypeVar("T") -MINUTE_MS = 60_000 -MIN_METADATA_TTL_MS = 24 * 60 * 60 * 1000 -SCORE_TIME_BASE = 1 << 40 -MAX_ATTEMPT = 255 -MAX_EXACT_INTEGER = (1 << 53) - 1 - - -class StoreOperationError(ExpiryError): - """内部存储错误；执行结果是否未知只供调度/重试恢复使用。""" - - def __init__(self, code: str, message: str, *, retryable: bool = False, - outcome_unknown: bool = False) -> None: - """保留内部执行状态；facade 会映射为公开错误，不直接透传此类型。""" - super().__init__(code, message, retryable=retryable) - self.outcome_unknown = outcome_unknown - - -class OwnerRole(str, Enum): - SCHEDULER = "scheduler" - CALLBACK = "callback" - - -@dataclass(frozen=True) -class ItemMetadata: - identity: ItemIdentity - - -@dataclass(frozen=True) -class NamespaceSpec: - shard_count: int - hash_name: str - hash_seed: int - identity_encoding: str # 固定实际规则名称：urlsafe_base64，无独立版本号。 - - -@dataclass(frozen=True) -class OwnerToken: - role: OwnerRole - shard_id: int - token: str # 每次成功领取的随机 token；不复用上次进程 token。 - expires_at_ms: int # 两个角色均有 TTL；仅用于观测，本地截止使用 monotonic。 - - -@dataclass -class ShardLease: - owner: OwnerToken - lost: asyncio.Event # ownership 已不可信；一旦置位，后续迟到成功也不可复活。 - valid_until_monotonic: float - revoked: asyncio.Event = field(default_factory=asyncio.Event) # 本地执行已撤销，不等于失锁。 - - def ensure_active(self) -> None: - """动作前检查 ownership 截止和本地撤销；不能撤回已发外部请求。""" - if self.lost.is_set() or time.monotonic() >= self.valid_until_monotonic: - self.lost.set() - raise StoreOperationError("owner_lost", "Shard lease is no longer valid") - if self.revoked.is_set(): - raise StoreOperationError("owner_lost", "Shard lease execution is revoked") - - -@dataclass(frozen=True) -class ShardWindow: - progress_ms: int # C：最后发布的分钟上界。 - upper_ms: int # U：当前源 TIME 去掉秒和毫秒；只是本轮局部快照。 - redis_now_ms: int - - -@dataclass(frozen=True) -class SourceCursor: - due_at_ms: int - item_id: str # score 相同按 Redis member 的二进制字节顺序续读。 - - -@dataclass(frozen=True) -class SourceItem: - item_id: str - due_at_ms: int - - -@dataclass(frozen=True) -class SourcePage: - items: tuple[SourceItem, ...] - next_cursor: SourceCursor | None # None 表示本次分钟范围已读完。 - - -@dataclass(frozen=True) -class BucketRef: - item_type: str - shard_id: int - minute_ms: int # 窗口 (minute_ms - 60000, minute_ms] 的结束边界。 - - -@dataclass(frozen=True) -class RegistryCursor: - minute_ms: int - member: str - - -@dataclass(frozen=True) -class RegistryPage: - buckets: tuple[BucketRef, ...] - next_cursor: RegistryCursor | None - - -@dataclass(frozen=True) -class PublishResult: - status: Literal["published", "already_published"] - progress_ms: int - - -@dataclass(frozen=True) -class Claim: - bucket: BucketRef - item_id: str - claim_score: int - - -@dataclass(frozen=True) -class ClaimBatch: - claims: tuple[Claim, ...] - next_retry_layer: int # 消费者下次从该层继续，避免每次只检查低 attempt。 - layers_scanned: int - exhausted: tuple[Claim, ...] # 最终机会已超时，尚未删除；供 consumer 写日志后条件 ACK。 - - -# 内部模型不承载 session policy、消息内容、event_id 或 group_id。 - - -class AsyncExecutor(Protocol): - async def run(self, operation: Callable[[], Awaitable[T]]) -> T: - """在固定 I/O loop 创建并执行 coroutine，调用方立即等待。 - - operation 必须是工厂，不能把在别的 loop 创建的 Redis Future 传进来。 - 取消/超时不能被解释为服务端已回滚；写入结果未知由 store 分类。 - """ - ... - - -def validate_item_type(item_type: str) -> None: - """只校验 [a-z][a-z0-9_]{0,63} 格式；不认识 session/watch 等具体业务类型。""" - ... - - -def validate_identity(identity: ItemIdentity) -> None: - """校验类型名及非空字符串身份，具体组件由 keys.py 可逆编码。""" - ... - - -def validate_report(task: ExpiryRegistration) -> None: - """校验类型名称格式、非空身份、正 timeout、毫秒整数范围。 - - 类型是否已注册由 facade/内部目录判断，这里没有业务类型白名单。 - FIXED -> unsupported_mode；坏身份 -> invalid_identity；参数错误不可重试。 - bool 不当作 int 接受；所有值必须处于 Redis Lua 的精确整数范围。 - 使用 metadata_ttl_ms 校验两倍 timeout 的毫秒计算；TTL 不由业务单独传入。 - """ - ... - - -def minute_start_ms(timestamp_ms: int) -> int: - """Unix 毫秒归一到分钟起点。""" - return timestamp_ms // MINUTE_MS * MINUTE_MS - - -def scan_cutoff_ms(now_ms: int) -> int: - """扫描包含式上界：源 TIME 向下取整到当前分钟，不减 1ms、不预取下一分钟。""" - return minute_start_ms(now_ms) - - -def bucket_minute_ms(due_at_ms: int) -> int: - """due 保留原精度；返回包含它的结束分钟，整分钟 due 属于该分钟窗口。""" - if type(due_at_ms) is not int or not 0 <= due_at_ms <= MAX_EXACT_INTEGER: - raise ExpiryError("invalid_report", "Due time is outside the supported range") - minute = ((due_at_ms + MINUTE_MS - 1) // MINUTE_MS) * MINUTE_MS - if minute > MAX_EXACT_INTEGER: - raise ExpiryError("invalid_report", "Due time is outside the supported range") - return minute - - -def metadata_ttl_ms(timeout_seconds: int) -> int: - """M 创建与续期统一使用 max(两倍 timeout,24小时)，返回 PEXPIRE 的毫秒值。""" - if type(timeout_seconds) is not int or not 0 < timeout_seconds <= MAX_EXACT_INTEGER // 2000: - raise ExpiryError("invalid_report", "Timeout is outside the supported range") - return max(timeout_seconds * 2000, MIN_METADATA_TTL_MS) - - -def encode_claim_score(attempt: int, started_at_seconds: int) -> int: - """与 Lua 算术完全一致；非法范围报错，不静默截断。""" - if type(attempt) is not int or not 0 <= attempt <= MAX_ATTEMPT: - raise ValueError("attempt out of range") - if type(started_at_seconds) is not int or not 0 < started_at_seconds < SCORE_TIME_BASE: - raise ValueError("claim timestamp out of range") - return attempt * SCORE_TIME_BASE + started_at_seconds - - -def decode_claim_score(score: int) -> tuple[int, int]: - """仅解码已领取记录；0 是 ready 哨兵，由调用者先处理。""" - if type(score) is not int or not 0 < score < (MAX_ATTEMPT + 1) * SCORE_TIME_BASE: - raise ValueError("invalid claim score") - attempt, started_at = divmod(score, SCORE_TIME_BASE) - if started_at == 0: - raise ValueError("invalid claim timestamp") - return attempt, started_at -``` - -公开错误码包括 invalid_identity、invalid_report、unsupported_mode、unregistered_type、duplicate_handler、invalid_handler、registration_closed、reporting_disabled、not_started、timer_unavailable。namespace、owner、progress、clock 及结果未知状态属于内部恢复错误，由 facade 映射后返回业务；业务 NotFound 由业务 handler 转成 COMPLETED。 - - -## 3. keys.py：编码和寻址 - -```python -from dataclasses import dataclass -from .api import ItemIdentity -from .models import BucketRef, OwnerRole, validate_identity - -HASH_SEED = 0 -IDENTITY_ENCODING = "urlsafe_base64" - - -def encode_component(value: str) -> str: - """UTF-8 -> URL-safe Base64，无 '=' 填充；身份字段不能为空。""" - ... - - -def decode_component(value: str) -> str: - """严格解码并重编码校验规范形式，拒绝别名、坏 UTF-8、非法字符。""" - ... - - -def encode_item_id(identity: ItemIdentity) -> str: - """类型名 + 三个编码组件；不能将用户原始花括号放进 Redis key。""" - validate_identity(identity) - return ":".join((identity.item_type, encode_component(identity.account_id), - encode_component(identity.user_id), encode_component(identity.local_id))) - - -def decode_item_id(item_id: str) -> ItemIdentity: - """严格四段，校验类型名格式；不查业务白名单，用于框架内部精确路由。""" - ... - - -def shard_for(item_id: str, shard_count: int) -> int: - """xxhash.xxh64(item_id.encode('utf-8'), seed=HASH_SEED).intdigest() % N。""" - ... - - -@dataclass(frozen=True) -class ExpiryKeys: - service_key: str # 构造时先 encode_component(service_name)，禁止直接传原始字符串。 - - def namespace(self) -> str: - """协议 manifest HASH；只保存固定的分片、哈希和身份编码配置。""" - return f"expiry:namespace:{self.service_key}" - - def metadata(self, item_id: str) -> str: - """M：单 item 身份缓存。""" - return f"expiry:metadata:{self.service_key}:{item_id}" - - def source(self, shard_id: int) -> str: - """R：不加真实 hash tag。""" - return f"expiry:publisher:source:{self.service_key}:{shard_id}" - - - - def control_tag(self, shard_id: int) -> str: - """返回带花括号的真实 tag，避免不同逻辑内核控制 key 扎堆。""" - return f"{{{self.service_key}:{shard_id}}}" - - def progress(self, shard_id: int) -> str: - """C：与 G/调度 owner 同槽。""" - return f"expiry:publisher:progress:{self.control_tag(shard_id)}" - - def registry(self, shard_id: int) -> str: - """G：只登记已经写完并发布的 B。""" - return f"expiry:consumer:registry:{self.control_tag(shard_id)}" - - def owner(self, role: OwnerRole, shard_id: int, *, item_type: str | None = None) -> str: - """不同角色用不同 key，scheduler 与 callback 不互抢。""" - ... - - def bucket(self, ref: BucketRef) -> str: - """B：无需与 G 同槽。""" - return (f"expiry:consumer:bucket:{self.service_key}:{ref.item_type}:" - f"{ref.shard_id}:{ref.minute_ms}") - - - - def registry_member(self, ref: BucketRef) -> str: - """序列化注册项，不带 namespace，因为 G 已按 service_name 隔离。""" - return f"{ref.item_type}:{ref.shard_id}:{ref.minute_ms}" - - def parse_registry_member(self, value: str, *, expected_shard: int) -> BucketRef: - """校验 type、shard 和分钟对齐，拒绝跨 shard/损坏注册项。""" - ... -``` - -## 4. metrics.py：可观测性接口 - -```python -from collections.abc import Callable, Mapping -from .models import Claim - - -class ExpiryMetrics: - """包装内核指标 sink；内部 publisher 进程可使用同一 sink 或结构化日志。 - - 标签限于固定 operation/item_type/outcome；不以 item_id/account_id 作标签。 - sink 失败不能影响业务结果，批量任务不逐项写成功日志。 - """ - - def __init__(self, sink: Callable[[str, float, Mapping[str, str]], None] | None = None): - """绑定可选指标 sink；不连接 Redis 或初始化业务服务。""" - self._sink = sink - - def count(self, name: str, value: int = 1, *, item_type: str = "", - outcome: str = "") -> None: - """计数：report/claim/ACK/回源/失败等；内部隔离 sink 异常。""" - ... - - def observe(self, name: str, value: float, *, item_type: str = "", - outcome: str = "") -> None: - """耗时、页大小、retry 层数等分布。单位在指标名中固定。""" - ... - - def log_exhausted(self, claim: "Claim", *, max_attempts: int, - reason: str, outcome_unknown: bool) -> None: - """输出结构化耗尽日志：身份/窗口/claim/机会数/原因，不含消息正文。 - - 日志进入现有日志设施，不写 Redis。失败向调用者抛错，不能声称已记录。 - 日志与随后 ACK 不是事务，允许重复或崩溃时丢失；不伪造恰好一次。 - """ - ... - - def gauge(self, name: str, value: float, *, item_type: str = "") -> None: - """当前 backlog、inflight、owner 数等；不可逐资源生成系列。""" - ... -``` - -## 5. redis_store.py：完整存储接口 - -这个类集中管理 Redis 数据操作，scheduler/consumer 不直接取得 Redis 客户端。以下所有公开 async 方法都通过 `_io.run(...)` 执行原始客户端操作；标记 `_on_loop` 的方法只能从该 I/O loop 调用。逻辑流程可以跨多次 I/O，不伪装成跨 slot 事务。 - -```python -from __future__ import annotations - -from typing import Any, Literal, Sequence -from .keys import ExpiryKeys -from .metrics import ExpiryMetrics -from .api import ExpiryRegistration, ExpiryRegistrationResult -from .models import ( - AsyncExecutor, BucketRef, Claim, ClaimBatch, NamespaceSpec, OwnerRole, - ItemMetadata, OwnerToken, PublishResult, RegistryCursor, RegistryPage, - ShardWindow, SourceCursor, SourcePage, StoreOperationError, metadata_ttl_ms, -) -from .config import ExpiryRedisConfig - -RetryPolicy = Literal["read", "idempotent", "resolve_by_caller"] - - -class RedisExpiryStore: - def __init__(self, config: ExpiryRedisConfig, keys: ExpiryKeys, - io: AsyncExecutor, metrics: ExpiryMetrics) -> None: - """仅保存配置；此处不创建绑定调用方 loop 的 Redis 连接。""" - self._config, self._keys, self._io, self._metrics = config, keys, io, metrics - self._client: Any = None # 仅 _on_loop 方法访问。 - self._scripts: dict[str, str] = {} - self._ready = False - - async def open(self) -> None: - """在 I/O loop 创建 standalone/cluster 客户端，PING 并检查所需命令。""" - await self._io.run(self._open_on_loop) - - async def close(self) -> None: - """先拒绝新请求，再在 I/O loop 关闭连接；loop 此时仍须存活。""" - await self._io.run(self._close_on_loop) - - async def ensure_namespace(self, spec: NamespaceSpec) -> None: - """新 namespace: INITIALIZING -> 幂等初始化 N 个 C=0 -> READY。 - - 所有 reporter 等待 READY。中断初始化只补缺项、不覆盖已有 C。 - READY namespace 的缺 C 是损坏，不按当前时间自动重置。 - 实际N/hash/编码配置不匹配 -> namespace_mismatch；没有独立格式版本字段。 - """ - ... - - async def ensure_metadata(self, item_id: str, task: ExpiryRegistration) -> None: - """内部计算 TTL 后调用 ensure_metadata.lua；创建和续期使用同一个毫秒值。 - - ttl_ms=max(2*timeout_seconds*1000,86_400_000)，无独立客户端 EXISTS。 - """ - identity = task.identity - ttl_ms = metadata_ttl_ms(task.timeout_seconds) - await self._eval( - "ensure_metadata", [self._keys.metadata(item_id)], - [identity.item_type, identity.local_id, identity.account_id, identity.user_id, ttl_ms], - retry="idempotent", - ) - - async def read_metadata(self, item_id: str) -> ItemMetadata | None: - """None 只表示 Redis key 不存在；网络失败抛错，身份冲突也抛错。""" - ... - - async def upsert_resource(self, shard_id: int, item_id: str, - task: ExpiryRegistration) -> ExpiryRegistrationResult: - """仅 report 调用：单 R Lua 用 TIME 计算 max(requested_due,now+1) 后 ZADD。 - - 不读 C，不写伴随状态，不做业务顺序比较；同 item_id 允许后到请求覆盖。 - 网络未知结果不自动无限重放，业务是否再次 report 按失败策略决定。 - """ - ... - - async def unregister_resource(self, shard_id: int, item_id: str) -> None: - """单 key ZREM R 后 DEL M；部分失败报错，不扫描 B，重复删除幂等。""" - ... - - async def read_progress(self, shard_id: int) -> int: - """读 C；READY namespace 中不存在则抛 namespace_corrupt。""" - ... - - async def read_scan_window(self, shard_id: int) -> ShardWindow: - """读 C，再通过 source_time.lua(R) 取得源 TIME，U=去掉秒和毫秒。 - - 只读，不更新 R 或 C。U=C 直接跳过；U SourcePage: - """调用原子分页脚本；after 为稳定 score/member 游标，不是 OFFSET。""" - ... - - async def reset_unpublished_minute(self, owner: OwnerToken, *, minute_ms: int, - expected_progress_ms: int, - item_types: Sequence[str]) -> None: - """先校验 owner/C，本地 lease 有效且 C 尚未覆盖，再 UNLINK 已知 type 的 B。 - - 仅允许 minute_ms > C；已发布分钟拒绝清空。 - 校验 C 与跨槽 UNLINK 不是事务；租约过期后的旧请求仍是主稿 10.3 的限制。 - """ - ... - - async def stage_items(self, owner: OwnerToken, bucket: BucketRef, - item_ids: Sequence[str]) -> None: - """未注册 B 上的有界 ZADD 0；只由该 shard 独占 scheduler 调用。 - - 返回意味着本批明确成功；超时返回 result_unknown，不在后台盲目重放。 - 整个分钟所有 stage 调用都结束后，才能 publish。 - """ - ... - - async def publish_minute(self, owner: OwnerToken, *, expected_progress_ms: int, - minute_ms: int, buckets: Sequence[BucketRef]) -> PublishResult: - """同槽 Lua 注册所有 B 并推进 C；owner/C 错误在任何写入前返回。 - - minute_ms>expected_progress_ms，允许跨过扫描已确认的空分钟；发布后 C=minute_ms。 - buckets 必须非空，空分钟跳过；源扫描完成由 scheduler 保证。 - """ - ... - - async def cleanup_source(self, shard_id: int, through_ms: int) -> int: - """ZREMRANGEBYSCORE R -inf through_ms；返回删除数，单次耗时要监控。 - - 只接受已确认 C；不执行任何 ZADD。是否换成有界批删需压测后另行决定。 - """ - ... - - async def read_registry_page(self, shard_id: int, *, after: RegistryCursor | None, - limit: int) -> RegistryPage: - """G 只包含已发布分钟，按 score/member 稳定分页；不全量 ZRANGE。 - - 页末成员可能被摘除，使用与资源分页相同的原子定位策略。 - """ - ... - - async def claim(self, bucket: BucketRef, *, limit: int, max_attempts: int, - processing_timeout_seconds: int, - retry_layer: int, retry_layers_per_poll: int) -> ClaimBatch: - """单 B Lua：有界领取，最后机会超时项只返回 exhausted，不删除或重领。 - - 每 claim 占一次机会；session 查询0/1/2层，超时检查不可被新项饿死。 - exhausted 由 consumer 日志后条件 ACK，无 Redis 失败存储。 - 响应未知不自动重放 claim，等超时按计数恢复。 - """ - ... - - async def ack(self, claim: Claim) -> bool: - """相同 score 才删除；False 表示已删除或被新领取，不再调用业务补偿。 - - 使用 retry=idempotent 在请求预算内重试同一条件 ACK，不能重跑 handler。 - """ - ... - - - - - - async def remove_registration_if_empty(self, bucket: BucketRef) -> bool: - """读 B 为空后摘 G；依赖发布后不再生产，允许遗留空注册项。""" - ... - - async def try_acquire_owner(self, role: OwnerRole, shard_id: int, token: str, - ttl_seconds: int) -> OwnerToken | None: - """SET NX PX，两个角色均使用120秒TTL；忙返回None。 - - 同token仅用于确认本次领取响应，不延长已失败lease；下一次领取必须新token。 - """ - ... - - async def renew_owner(self, owner: OwnerToken, ttl_seconds: int) -> OwnerToken | None: - """比较 token 后 PEXPIRE；缺失/不匹配返回None，异常即本次lease失败。 - - 不能重新创建锁，不能在丢锁后用同token恢复处理；两种角色均适用。 - """ - ... - - async def assert_owner(self, owner: OwnerToken) -> None: - """读取 token 验证；失败抛 owner_lost。非跨槽 B 写 fencing。""" - ... - - async def release_owner(self, owner: OwnerToken) -> bool: - """Lua 比较 token 后删除，不清除后继 owner。""" - ... - - async def _open_on_loop(self) -> None: - """延迟 import redis.asyncio；缺 extra 时给可操作的依赖错误。""" - ... - - async def _close_on_loop(self) -> None: - """关闭两个 Redis 模式下的客户端；清空缓存引用。""" - ... - - async def _initialize_shard(self, shard_id: int) -> None: - """仅 namespace INITIALIZING 时 SET C 0 NX，已有值不覆盖。 - - 全部初始化完成后置READY；report须等待READY，不按当前时间跳过历史源数据。 - """ - ... - - async def _command(self, name: str, *args: object, - retry: RetryPolicy = "read") -> Any: - """将客户端命令封装为 coroutine 工厂，经 _io.run 发送并分类错误。""" - ... - - async def _eval(self, script: str, keys: Sequence[str], args: Sequence[object], - *, retry: RetryPolicy) -> Any: - """非 I/O loop 入口；只调度 _eval_on_loop，业务锁不带入这里。""" - return await self._io.run(lambda: self._eval_on_loop(script, keys, args, retry=retry)) - - async def _eval_on_loop(self, script: str, keys: Sequence[str], args: Sequence[object], - *, retry: RetryPolicy) -> Any: - """EVALSHA，NOSCRIPT 则对实际目标 slot EVAL；MOVED/ASK 遵从客户端。 - - 网络超时不对 resolve_by_caller 自动重试；返回 result_unknown。 - Lua 输入 key 显式列在 KEYS；跨 slot 参数在发送前拒绝。 - """ - ... - - def _read_script(self, name: str) -> str: - """读取 lua/_common.lua + lua/name.lua 并拼接，进程内缓存源码/SHA。""" - ... - - def _translate_redis_error(self, exc: Exception, *, retry: RetryPolicy) -> StoreOperationError: - """区分协议错误、可重试读错误、服务端执行结果未知；保留原异常链。""" - ... -``` - -不提供 `get_client()`，也不提供通用“跨 slot pipeline 事务”。`stage_items/reset_unpublished_minute` 是结果未知时需要上层协调的操作；条件 ACK 与读取有各自可重试语义；report 无版本去重，不自动声称重复请求无影响。 - -## 6. registry.py、service.py 与 __init__.py：三个业务方法，内部自行管理 - -### 6.1 registry.py：内部处理器注册表 - -```python -from dataclasses import dataclass -from types import MappingProxyType -from typing import Mapping -from .api import HandlerOptions, ExpiryHandler, ExpiryError -from .models import validate_item_type - - -@dataclass(frozen=True) -class HandlerBinding: - handler: ExpiryHandler - options: HandlerOptions - - -class HandlerRegistry: - """只在框架内部使用；处理器和并发设置不写入 Redis。""" - - def __init__(self) -> None: - """创建空注册表，不硬编码任何业务类型。""" - self._bindings: dict[str, HandlerBinding] = {} - self._frozen = False - - def register(self, item_type: str, handler: ExpiryHandler, options: HandlerOptions) -> None: - """校验类型名、异步 handle、正整数并发及 max_attempts∈[1,256]（bool 非法）。 - - 坏 handler/options -> invalid_handler；重复类型 -> duplicate_handler； - freeze 后修改 -> registration_closed；均使用公开 ExpiryError。 - """ - ... - - def freeze(self) -> Mapping[str, HandlerBinding]: - """runtime 在开始消费前冻结，返回只读注册快照；不能从业务接口调用。""" - ... -``` - -### 6.2 service.py：不导出的 facade 实现 - -```python -from dataclasses import dataclass, field -from .api import ( - HandlerOptions, ItemIdentity, ExpiryHandler, ExpiryRegistration, ExpiryRegistrationResult, ExpiryError, -) -from .keys import encode_item_id, shard_for -from .metrics import ExpiryMetrics -from .models import StoreOperationError, validate_report -from .redis_store import RedisExpiryStore -from .registry import HandlerRegistry - - -@dataclass -class ExpiryRuntimeState: - """框架私有生命周期状态，runtime 持有；业务对象不提供状态修改入口。""" - storage_ready: bool = False - accepting_calls: bool = False - - -class _ExpirySchedulerFacade: - """实现 ExpiryScheduler；只有 runtime 构造，业务通过依赖注入得到协议引用。""" - - def __init__(self, store: RedisExpiryStore, registry: HandlerRegistry, - state: ExpiryRuntimeState, *, shard_count: int, - reporting_enabled: bool, metrics: ExpiryMetrics) -> None: - """保存内部依赖；业务不负责构造 store、keys、registry 或 state。""" - self._store, self._registry, self._state = store, registry, state - self._metrics, self._shard_count = metrics, shard_count - self._reporting_enabled = reporting_enabled - - async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: - """执行一次计时写入，将内部存储错误映射为业务错误。""" - self._require_ready() - if not self._reporting_enabled: - raise ExpiryError("reporting_disabled", "Timer reporting is disabled") - validate_report(task) - self._ensure_supported_type(task.identity.item_type) - try: - item_id = encode_item_id(task.identity) - shard_id = shard_for(item_id, self._shard_count) - result = await self._store.upsert_resource(shard_id, item_id, task) - await self._store.ensure_metadata(item_id, task) - except StoreOperationError as exc: - raise self._business_error(exc) from exc - self._metrics.count("timer_report", item_type=task.identity.item_type, - outcome="written") - return result - - async def unregister(self, identity: ItemIdentity) -> None: - """删除受支持类型的当前计时；不要求本进程仍有该类型 handler。""" - self._require_ready() - self._ensure_supported_type(identity.item_type) - item_id = encode_item_id(identity) - try: - await self._store.unregister_resource(shard_for(item_id, self._shard_count), item_id) - except StoreOperationError as exc: - raise self._business_error(exc) from exc - - def register_handler(self, item_type: str, handler: ExpiryHandler, *, - options: HandlerOptions | None = None) -> None: - """同步本地注册，允许在 storage 启动前调用；运行后由内部 registry 拒绝修改。""" - self._registry.register(item_type, handler, options or HandlerOptions()) - - @staticmethod - def _ensure_supported_type(item_type: str) -> None: - """按 SUPPORTED_ITEM_TYPES 校验；未知类型在任何 Redis 写入前失败。""" - ... - - def _require_ready(self) -> None: - """未初始化或关闭中抛 not_started；就绪状态只能由内部生命周期管理。""" - if not (self._state.storage_ready and self._state.accepting_calls): - raise ExpiryError("not_started", "Expiry scheduling is not ready") - - def _business_error(self, exc: StoreOperationError) -> ExpiryError: - """参数/身份错误保留公开错误码；内部不可用映射 timer_unavailable 并记内部指标。 - - 不向业务返回 owner、claim score、Lua 错误格式或 outcome_unknown 字段。 - """ - ... -``` - -### 6.3 __init__.py：限定业务可依赖的导出 - -```python -from .api import ( - CallbackRequest, CallbackResult, CallbackStatus, HandlerOptions, - ItemIdentity, ExpiryHandler, ExpiryRegistration, ExpiryScheduler, - ExpiryRegistrationResult, SESSION_ITEM_TYPE, ExpiryError, ExpiryMode, -) - -__all__ = [ - "ExpiryScheduler", "SESSION_ITEM_TYPE", "ItemIdentity", "ExpiryRegistration", "ExpiryRegistrationResult", - "ExpiryMode", "ExpiryError", "HandlerOptions", "ExpiryHandler", - "CallbackRequest", "CallbackResult", "CallbackStatus", -] -``` - -包入口不导出 facade、store、registry、runtime、Redis 配置、Claim 或 BucketRef。Python 模块路径不构成安全隔离，但业务模块只依赖此公开导出，内部类型不作为兼容接口。 - -### 6.4 独立调度器如何得到类型信息 - -`api.py` 中的 `SUPPORTED_ITEM_TYPES` 是类型信息的唯一来源。业务 `register_handler()` 只允许注册清单内类型；report 在写 Redis 前按同一清单校验。独立调度器直接引用清单，不需要加载业务 Handler，也不访问 Redis 类型目录；处理器对象和 HandlerOptions 只留在本地进程。 - -调度器先取得本轮源时间上界，再使用 `SUPPORTED_ITEM_TYPES` 定位并清理未发布分钟的临时 B。注册表 G 仍是唯一的“桶可以消费”的标志。类型下线前必须先排空 source、bucket 和 processing 数据，再从代码清单删除。 - -回调实例应加载其负责业务的注册模块；遇到自己没有 handler 的历史/新类型，保留任务并报警，不静默 ACK。首期不做运行中替换处理器或调整并发；新增业务在进程启动阶段完成注册。 - -当前 callback owner 按 shard 分配，同一 namespace 的回调副本应加载同一组业务处理器。缺少 handler 的 owner 不会自动把该类型让给其他副本；新增类型应先部署对应回调注册，再开放业务上报。按类型部署不同 worker 属于另一种路由模型，不在首期隐式支持。 - - -## 7. publisher.py：持锁清空到期积压，空范围跳过 - -```python -from __future__ import annotations - -import asyncio -import time -from collections import defaultdict -from typing import Sequence -from .keys import decode_item_id -from .metrics import ExpiryMetrics -from .models import ( - MINUTE_MS, BucketRef, OwnerRole, PublishResult, - StoreOperationError, bucket_minute_ms, -) -from .redis_store import RedisExpiryStore -from .config import ExpiryConfig -from .shard_runner import ShardExecution, ShardRunner - - -async def _wait_until_next_round(stop: asyncio.Event, delay: float) -> None: - """无租约时的轮间等待；stop可唤醒，不负责续租或失锁监视。""" - try: - await asyncio.wait_for(stop.wait(), delay) - except TimeoutError: - pass - - -class ExpiryPublisher: - def __init__(self, store: RedisExpiryStore, runner: ShardRunner, - config: ExpiryConfig, metrics: ExpiryMetrics) -> None: - """worker拥有shard遍历与分钟节奏，runner只执行一次单shard工作。""" - self._store, self._runner, self._config, self._metrics = store, runner, config, metrics - - async def run(self, stop: asyncio.Event) -> None: - """每轮递增遍历全部shard；shard_not_stopped必须上抛，不能继续抢锁。""" - while not stop.is_set(): - for shard_id in range(self._config.shard_count): - if stop.is_set(): - break - try: - await self._runner.run_if_acquired( - OwnerRole.SCHEDULER, shard_id, self.process_shard, stop, - ) - except StoreOperationError as exc: - if exc.code == "shard_not_stopped": - raise - self._metrics.count("timer_shard_round", outcome="failed") - except Exception: - self._metrics.count("timer_shard_round", outcome="failed") - if not stop.is_set(): - delay = max(0.05, 60.0 - time.time() % 60.0) - await _wait_until_next_round(stop, delay) - - async def process_shard(self, lease: ShardExecution, stop: asyncio.Event) -> None: - """持续处理全部到期积压，不因页数/分钟数预算交出锁。 - - 每步检查lease；取消后退出旧函数，新领取从C重建。 - _process_minute返回None表示重读后该分钟已空，本地扫描下界前进但不写C。 - """ - while not stop.is_set(): - lease.ensure_active() - window = await self._store.read_scan_window(lease.owner.shard_id) - progress = window.progress_ms - lease.ensure_active() - await self._store.cleanup_source(lease.owner.shard_id, progress) - if window.upper_ms <= progress: - return - after_ms = progress - published_any = False - while after_ms < window.upper_ms and not stop.is_set(): - lease.ensure_active() - page = await self._store.read_source_page( - lease.owner.shard_id, lower_exclusive_ms=after_ms, - upper_inclusive_ms=window.upper_ms, after=None, limit=1, - ) - if not page.items: - break - minute = bucket_minute_ms(page.items[0].due_at_ms) - result = await self._process_minute(lease, progress_ms=progress, minute_ms=minute) - if result is not None: - progress = result - published_any = True - after_ms = minute - if not published_any: - return - # 确认追平当前上界后再读一次新上界，不持锁忙等尚未到期资源。 - - async def _process_minute(self, lease: ShardExecution, *, progress_ms: int, - minute_ms: int) -> int | None: - """分页读完整个非空分钟 -> 写完B -> 原子发布G/C -> 清理R。""" - lease.ensure_active() - await self._store.reset_unpublished_minute( - lease.owner, minute_ms=minute_ms, expected_progress_ms=progress_ms, - item_types=tuple(sorted(SUPPORTED_ITEM_TYPES)), - ) - cursor = None - types_with_data: set[str] = set() - while True: - lease.ensure_active() - page = await self._store.read_source_page( - lease.owner.shard_id, lower_exclusive_ms=max(progress_ms, minute_ms - MINUTE_MS), - upper_inclusive_ms=minute_ms, after=cursor, limit=self._config.scan_batch_size, - ) - by_type: dict[str, list[str]] = defaultdict(list) - for item in page.items: - identity = decode_item_id(item.item_id) - if identity.item_type not in SUPPORTED_ITEM_TYPES: - raise StoreOperationError("unregistered_type", "Expiry item type is not supported") - by_type[identity.item_type].append(item.item_id) - for item_type, item_ids in by_type.items(): - lease.ensure_active() - ref = BucketRef(item_type, lease.owner.shard_id, minute_ms) - await self._store.stage_items(lease.owner, ref, item_ids) - types_with_data.add(item_type) - cursor = page.next_cursor - if cursor is None: - break - if not types_with_data: - return None - lease.ensure_active() - refs = tuple(BucketRef(t, lease.owner.shard_id, minute_ms) for t in sorted(types_with_data)) - result = await self._publish_completed_minute( - lease, expected_progress_ms=progress_ms, minute_ms=minute_ms, buckets=refs, - ) - lease.ensure_active() - await self._store.cleanup_source(lease.owner.shard_id, result.progress_ms) - return result.progress_ms - - async def _publish_completed_minute(self, lease: ShardExecution, *, expected_progress_ms: int, - minute_ms: int, buckets: Sequence[BucketRef]) -> PublishResult: - """同一有效lease中保留不可变发布参数，超时查C或重试同一publish。 - - C未前进不代表旧请求不会执行，不能重置B。每次I/O前检查lease。 - lease失败立即退出，不在后续新lease中恢复本函数；未决请求风险见主稿10.3。 - """ - ... -``` - -分页是内存/命令预算，持锁期间可持续处理任意多个分钟。source_time只读，R的score只有report更新；调度的R变更仅为按已确认C清理。 - -## 8. consumer.py：清空积压、三次机会、仅日志终结 - -```python -from __future__ import annotations - -import asyncio -from collections import deque -from dataclasses import dataclass, field -from typing import Mapping -from .api import CallbackRequest, CallbackStatus, ExpiryError -from .keys import decode_item_id -from .metrics import ExpiryMetrics -from .models import BucketRef, Claim, OwnerRole, StoreOperationError, decode_claim_score -from .registry import HandlerBinding -from .redis_store import RedisExpiryStore -from .config import ExpiryConfig -from .shard_runner import ShardExecution, ShardRunner - - -@dataclass -class _TypeState: - limit: int - reserved: int = 0 - buckets: deque[BucketRef] = field(default_factory=deque) - queued: set[BucketRef] = field(default_factory=set) - tasks: set[asyncio.Task[None]] = field(default_factory=set) - - -class ExpiryConsumer: - def __init__(self, store: RedisExpiryStore, runner: ShardRunner, - bindings: Mapping[str, HandlerBinding], config: ExpiryConfig, - metrics: ExpiryMetrics) -> None: - """类型并发不跨shard叠加；旧tasks未退出时不得重置共享state。""" - self._store, self._runner, self._bindings = store, runner, bindings - self._config, self._metrics = config, metrics - self._types = {t: _TypeState(b.options.concurrency) for t, b in bindings.items()} - self._retry_layers: dict[BucketRef, int] = {} - - async def run(self, stop: asyncio.Event) -> None: - """worker自己逐个竞争全部shard；轮后按callback_poll_seconds等待。""" - while not stop.is_set(): - for shard_id in range(self._config.shard_count): - if stop.is_set(): - break - try: - await self._runner.run_if_acquired( - OwnerRole.CALLBACK, shard_id, self.process_shard, stop, - ) - except StoreOperationError as exc: - if exc.code == "shard_not_stopped": - raise - self._metrics.count("timer_shard_round", outcome="failed") - except Exception: - self._metrics.count("timer_shard_round", outcome="failed") - if not stop.is_set(): - try: - await asyncio.wait_for(stop.wait(), self._config.callback_poll_seconds) - except TimeoutError: - pass - - async def process_shard(self, lease: ShardExecution, stop: asyncio.Event) -> None: - """消费业务流程不自建lost等待task；runner监视失锁/stop/grace并撤销子任务。""" - self._reset_shard_state() - try: - while not stop.is_set(): - lease.ensure_active() - found = await self._discover(lease) - for item_type in self._types: - await self._fill_slots(lease, item_type) - tasks = {task for state in self._types.values() for task in state.tasks} - has_queued = any(state.buckets for state in self._types.values()) - if not found and not has_queued and not tasks: - return - await lease.wait(self._config.callback_poll_seconds, stop) - # 正常stop等待已启动callbacks；外层runner仍续租并监视lost/有效截止， - # 到grace截止会取消process/children，故此处不是独立无界关闭路径。 - tasks = {task for state in self._types.values() for task in state.tasks} - if tasks: - await asyncio.wait(tasks) - lease.ensure_active() - finally: - if not any(not task.done() for state in self._types.values() for task in state.tasks): - self._reset_shard_state() - - def _reset_shard_state(self) -> None: - """仅在旧tasks全部终止后清队列/计数；每次单shard执行清retry层缓存。""" - for state in self._types.values(): - if any(not task.done() for task in state.tasks): - raise StoreOperationError("shard_not_stopped", "Previous callbacks are still running") - state.tasks.clear() - state.buckets.clear() - state.queued.clear() - state.reserved = 0 - self._retry_layers.clear() - - async def _discover(self, lease: ShardExecution) -> bool: - """lease检查后从G稳定分页，返回是否仍有可处理桶；每次发现有缓存预算。 - - 一页满不等于处理完；持续推进游标、绕回确认空。没有handler的类型不领取。 - """ - ... - - async def _fill_slots(self, lease: ShardExecution, item_type: str) -> None: - """计算空闲槽并claim；claim前后检查execution，retry层仅本次shard运行使用。""" - state = self._types[item_type] - available = state.limit - state.reserved - if available <= 0 or not state.buckets: - return - bucket = state.buckets.popleft() - state.queued.discard(bucket) - lease.ensure_active() - binding = self._bindings[item_type] - batch = await self._store.claim( - bucket, limit=available, max_attempts=binding.options.max_attempts, - processing_timeout_seconds=self._config.processing_timeout_seconds, - retry_layer=self._retry_layers.get(bucket, 0), - retry_layers_per_poll=self._config.retry_layers_per_poll, - ) - lease.ensure_active() - self._retry_layers[bucket] = batch.next_retry_layer - for claim in batch.exhausted: - await self._log_exhausted_and_ack(lease, claim, "processing_timeout") - for claim in batch.claims: - # 传工厂，不提前创建coroutine；runner强引用跟踪每个_execute子task。 - task = lease.spawn(lambda claim=claim: self._execute(lease, claim)) - state.reserved += 1 - state.tasks.add(task) - task.add_done_callback(state.tasks.discard) - lease.ensure_active() - removed = await self._store.remove_registration_if_empty(bucket) - if not removed and bucket not in state.queued: - state.queued.add(bucket) - state.buckets.append(bucket) - elif removed: - self._retry_layers.pop(bucket, None) - - async def _execute(self, lease: ShardExecution, claim: Claim) -> None: - """最多调用一次handler；COMPLETED后只重试同一ACK，不变成业务失败。""" - item_type = claim.bucket.item_type - completed = False - try: - lease.ensure_active() - request = await self._resolve_request(claim) - lease.ensure_active() - binding = self._bindings[item_type] - result = await asyncio.wait_for(binding.handler.handle(request), - timeout=self._config.processing_timeout_seconds) - lease.ensure_active() - if result.status is CallbackStatus.COMPLETED: - completed = True - await self._store.ack(claim) - lease.ensure_active() - await self._store.remove_registration_if_empty(claim.bucket) - else: - await self._finish_failed_attempt(lease, claim, "callback_retry") - except asyncio.CancelledError: - raise - except StoreOperationError as exc: - if exc.code == "owner_lost" or exc.outcome_unknown: - raise # 让runner保留失锁/未知结果语义，不吞掉后主动释放。 - self._record_callback_error(claim, exc) - if not completed: - lease.ensure_active() - await self._finish_failed_attempt(lease, claim, "callback_exception") - except Exception as exc: - self._record_callback_error(claim, exc) - if not completed: - lease.ensure_active() - await self._finish_failed_attempt(lease, claim, "callback_exception") - finally: - self._release_slot(item_type) - - async def _finish_failed_attempt(self, lease: ShardExecution, claim: Claim, reason: str) -> None: - """未耗尽留B等待超时；已耗尽先日志再条件ACK。lease失效不再执行清理。""" - attempt, _ = decode_claim_score(claim.claim_score) - if attempt + 1 >= self._bindings[claim.bucket.item_type].options.max_attempts: - await self._log_exhausted_and_ack(lease, claim, reason) - - async def _log_exhausted_and_ack(self, lease: ShardExecution, claim: Claim, reason: str) -> None: - """先输出耗尽日志，再ACK相同claim；错误保留B，之后可重复日志和清理。 - - 日志无Redis副本，与ACK不是事务。lease_expired/异常均保守标记结果未知。 - """ - try: - lease.ensure_active() - self._metrics.log_exhausted( - claim, max_attempts=self._bindings[claim.bucket.item_type].options.max_attempts, - reason=reason, outcome_unknown=True, - ) - lease.ensure_active() - await self._store.ack(claim) - lease.ensure_active() - await self._store.remove_registration_if_empty(claim.bucket) - except StoreOperationError as exc: - if exc.code == "owner_lost" or exc.outcome_unknown: - raise - self._record_callback_error(claim, exc) - except Exception as exc: - self._record_callback_error(claim, exc) - - async def _resolve_request(self, claim: Claim) -> CallbackRequest: - """M缺失可完整身份回源；M网络错误抛异常，身份冲突不可跨用户猜测。""" - identity = decode_item_id(claim.item_id) - if identity.item_type != claim.bucket.item_type: - raise ExpiryError("invalid_identity", "Bucket type differs from item ID") - metadata = await self._store.read_metadata(claim.item_id) - if metadata is not None and metadata.identity != identity: - raise ExpiryError("invalid_identity", "Metadata differs from item ID") - self._metrics.count("timer_metadata_lookup", item_type=identity.item_type, - outcome="miss" if metadata is None else "hit") - return CallbackRequest(identity) - - async def drain(self, timeout_seconds: float) -> None: - """兼容辅助入口，不替代runner关闭：有界观察，超时cancel后报未退出，不无界join。""" - tasks = {task for state in self._types.values() for task in state.tasks} - if not tasks: - return - _, pending = await asyncio.wait(tasks, timeout=max(0.0, timeout_seconds)) - for task in pending: - task.cancel() - if pending: - raise StoreOperationError("shard_not_stopped", "Callbacks did not drain within budget") - - def _release_slot(self, item_type: str) -> None: - """同一运行loop释放一次预留，任务完成/取消均执行。""" - ... - - def _record_callback_error(self, claim: Claim, exc: Exception) -> None: - """低基数错误指标和限频日志；不保存失败到Redis，不直接清成员。""" - ... - - # 持租约的poll等待使用ShardExecution.wait；consumer不维护单独lost task。 -``` - -任务claim超时与shard租约超时是两种机制；首期不续租单任务，条件ACK保护超时重领。第三次超时只日志清理，不再发起业务调用。 - -## 9. leases.py、shard_runner.py 与 runtime.py：token、执行和生命周期 - -### 9.1 leases.py：Coordinator 仅三种 token 操作 - -```python -from __future__ import annotations - -import asyncio -import time -import uuid -from .config import ExpiryConfig -from .metrics import ExpiryMetrics -from .models import OwnerRole, ShardLease, StoreOperationError -from .redis_store import RedisExpiryStore - - -class ShardLeaseCoordinator: - def __init__(self, store: RedisExpiryStore, config: ExpiryConfig, - metrics: ExpiryMetrics) -> None: - """只封装Redis token操作；不遍历shard，不创建process/heartbeat或等待轮间节奏。""" - self._store, self._config, self._metrics = store, config, metrics - - async def try_acquire(self, role: OwnerRole, shard_id: int) -> ShardLease | None: - """新随机token；截止=request开始monotonic+TTL，返回前检查未过期。""" - started = time.monotonic() - token = uuid.uuid4().hex - owner = await self._store.try_acquire_owner( - role, shard_id, token, self._config.owner_ttl_seconds, - ) - if owner is None: - return None - lease = ShardLease(owner, asyncio.Event(), started + self._config.owner_ttl_seconds) - lease.ensure_active() - return lease - - async def renew(self, lease: ShardLease) -> bool: - """任一失锁/异常/未知/取消或迟到成功使ownership永久不可信，旧lease不可复活。""" - try: - lease.ensure_active() - except StoreOperationError: - return False - started = time.monotonic() - previous_deadline = lease.valid_until_monotonic - try: - renewed = await self._store.renew_owner(lease.owner, self._config.owner_ttl_seconds) - except asyncio.CancelledError: - lease.lost.set() - raise - except Exception: - lease.lost.set() - return False - task = asyncio.current_task() - if (renewed is None or lease.lost.is_set() - or (task is not None and task.cancelling()) - or time.monotonic() >= previous_deadline): - lease.lost.set() - return False - if lease.revoked.is_set(): - return False - lease.owner = renewed - lease.valid_until_monotonic = started + self._config.owner_ttl_seconds - return True - - async def release(self, lease: ShardLease) -> None: - """runner确认本地工作/续租均终止且ownership可信后才调用；revoked本身不禁止释放。""" - if time.monotonic() >= lease.valid_until_monotonic: - lease.lost.set() - if lease.lost.is_set(): - return - try: - await self._store.release_owner(lease.owner) - except asyncio.CancelledError: - lease.lost.set() - raise - except Exception as exc: - if isinstance(exc, StoreOperationError) and exc.outcome_unknown: - lease.lost.set() - self._metrics.count("timer_owner_release", outcome="failed") -``` - -### 9.2 shard_runner.py：一次单 shard 执行，不替 worker 排轮次 - -```python -from __future__ import annotations - -import asyncio -import time -from typing import Awaitable, Callable -from .config import ExpiryConfig -from .leases import ShardLeaseCoordinator -from .metrics import ExpiryMetrics -from .models import OwnerRole, OwnerToken, ShardLease, StoreOperationError - -ShardProcessor = Callable[["ShardExecution", asyncio.Event], Awaitable[None]] - - -class ShardExecution: - """内部可撤销执行句柄；业务processor只见检查点、owner和受跟踪的spawn/wait。""" - - def __init__(self, lease: ShardLease) -> None: - self._lease = lease - self._children: set[asyncio.Task[None]] = set() - self._waiters: set[asyncio.Task[object]] = set() - self._error: BaseException | None = None - self._outcome_unknown = False - - @property - def owner(self) -> OwnerToken: - return self._lease.owner - - @property - def shard_id(self) -> int: - return self.owner.shard_id - - def ensure_active(self) -> None: - task = asyncio.current_task() - if task is not None and task.cancelling(): - # handler吞掉取消后也不能执行ACK；先同步标记unknown，不能等done callback。 - self._outcome_unknown = True - raise asyncio.CancelledError - self._lease.ensure_active() - - def spawn(self, operation: Callable[[], Awaitable[None]]) -> asyncio.Task[None]: - """先检查再创建task；传coroutine工厂，避免撤销后遗留未await的coroutine。""" - self.ensure_active() - - async def guarded() -> None: - self.ensure_active() - await operation() - - task = asyncio.create_task(guarded()) - self._children.add(task) - task.add_done_callback(self._child_done) - return task - - async def wait(self, delay_seconds: float, stop: asyncio.Event) -> None: - """等待poll/stop/lost/revoked；两侧检查，timeout不超过本地lease截止。 - - stop/lost/revoked的辅助waiters保持强引用，finally cancel；runner有界收集。 - """ - ... - - def _waiter_done(self, task: asyncio.Task[object]) -> None: - """摘除准确task并读取异常，不能泄漏辅助等待。""" - ... - - def _child_done(self, task: asyncio.Task[None]) -> None: - """摘除task并记录错误；cancelled/cancelling或outcome_unknown均保守标记unknown。""" - ... - - def _revoke(self) -> None: - self._lease.revoked.set() # 只撤销本地执行，不伪造ownership失锁。 - - -def _consume_exception(task: asyncio.Task[object]) -> None: - if not task.cancelled(): - task.exception() - - -class ShardRunner: - def __init__(self, leases: ShardLeaseCoordinator, config: ExpiryConfig, - metrics: ExpiryMetrics) -> None: - self._leases, self._config, self._metrics = leases, config, metrics - self._active: set[OwnerRole] = set() - self._pending: dict[OwnerRole, set[asyncio.Task[object]]] = {} - - @property - def has_pending(self) -> bool: - return any(not task.done() for tasks in self._pending.values() for task in tasks) - - async def wait_pending(self, timeout: float) -> bool: - """仅有界观察强引用保留的工作，不取消；返回是否全部终止。""" - tasks = {task for tasks in self._pending.values() for task in tasks if not task.done()} - if tasks: - await asyncio.wait(tasks, timeout=max(0.0, timeout)) - return not self.has_pending - - async def run_if_acquired(self, role: OwnerRole, shard_id: int, - process: ShardProcessor, stop: asyncio.Event) -> bool: - """同role已有active或pending时抛shard_not_stopped；stop/busy返回False。 - - acquire放入独立task并shield：调用者cancel后cancel请求task并有界_settle， - 不让抑制取消的Redis实现拖住join；未知领取只留TTL，不补release。 - 成功后创建全新ShardExecution并_run；返回True表示取得过租约，不代表业务成功。 - """ - ... - - async def _run(self, role: OwnerRole, execution: ShardExecution, - process: ShardProcessor, stop: asyncio.Event) -> None: - """创建processor/heartbeat/stop/lost tasks并独立监视有效截止。 - - 正常stop启动shutdown_timeout_seconds grace，heartbeat继续；grace期间仍监视 - lost/heartbeat/lease截止，不只等processor。失锁、grace超时、外层取消均forced。 - finally先revoke，再cancel processor/heartbeat/children/waiters，并有界_settle。 - 若renew请求正进行，同步置lost后再cancel，抑制取消后的成功不能复活lease。 - processor已返回却仍有children属于生命周期违例，forced且不得主动release。 - 所有task终止后读取processor/children异常，再决定release，最后向worker传播错误。 - 普通业务异常但已终止、无forced/unknown且ownership可信时可比较token释放； - forced/失锁/未知结果保守TTL。release本身也用asyncio.wait有界观察/保留pending。 - """ - ... - - async def _heartbeat(self, lease: ShardLease, renewing: asyncio.Event) -> None: - """后台每20秒调用Coordinator.renew；renewing标记请求在途，失败结束。""" - while True: - await asyncio.sleep(self._config.owner_renew_interval_seconds) - renewing.set() - try: - if not await self._leases.renew(lease): - return - finally: - renewing.clear() - - async def _settle(self, role: OwnerRole, tasks: set[asyncio.Task]) -> None: - """cancel后的join用asyncio.wait(timeout=request_budget_seconds)，不等取消确认。 - - await之前把未终止task放_pending强引用（包括acquire/release/heartbeat）， - done callback读取异常并只移除自己持有的准确集合，不清后轮同role的state。 - 预算后仍pending抛shard_not_stopped；再次取消cleanup也不能丢失强引用。 - """ - ... - - def _pending_done(self, role: OwnerRole, retained: set[asyncio.Task[object]], - task: asyncio.Task[object]) -> None: - """旧task真实退出才移除；保留集合identity检查避免误删后轮记录。""" - ... - - @staticmethod - def _not_stopped(role: OwnerRole) -> StoreOperationError: - return StoreOperationError("shard_not_stopped", f"The {role.value} shard still has active local work") -``` - -runner 的 pending 限制是**同角色的全部 shard**，不是只禁止重领原 shard；另一角色的生命周期独立。`asyncio.wait` 仅保证事件循环能运行时等待有界，不强杀协程；未合作取消的工作仍被强引用保留，worker必须停止同role轮次，依赖不能先关闭。只有正常终止、ownership仍可信且没有未知结果才可主动释放；普通业务异常本身不是“结果未知”，错误仍须传播。 - -这不改变Redis/Lua/protocol，不撤回已发Redis写入，不构成跨槽fencing；当前B不按token隔离的残余风险继续按主稿10.3保留。Session外层timeout检查和原commit设计不因runner重构而改变。 - -### 9.3 runtime.py：共享 runner 的组装与有界关闭 - -Redis I/O 的终止凭据是 `_RedisRequest.finished`，不是调用方 wrapper 被取消，也不是 `run_coroutine_threadsafe` 返回的 Future 状态。请求提交前登记强引用，Redis-loop task 及其原始 operation 真正退出后才完成 finished 并摘除登记；请求取消或超时后仍保留生命周期桥接。runner 的有界观察和 executor 的 quiesce/join 分别保护业务工作与远端 I/O，不能用 role task 的退出代替 Redis-loop 的真实退出。 - -关闭顺序为 `stop_roles → io.quiesce → io.run_cleanup(store.close) → io.close`。quiesce 拒绝新外部请求，等待已登记请求真实 done，预算耗尽抛 `shard_not_stopped`，不提前关 store。cleanup 在 Redis loop 运行 raw parent，不先占用 `_gate`；store.close 内部嵌套 `_io.run` 可在 quiesce 后执行并正常关闭 client，避免 max_inflight=1 时递归等待自身持有的 gate。io.close 再次 quiesce 后仅请求一次 loop.stop，以剩余预算 join 线程；线程仍活着则保留 loop/thread/gate 引用，后续 close 可重试，但 start 不能创建第二线程。store cleanup 成功即置 `_storage_closed=True`；若后续线程 join 超时，仍保留 `_storage_started/storage_ready`，重试 close 跳过已成功的 cleanup，只重新观察/等待线程退出，不向已 stop_requested 的 loop 再提交 store.close。只有线程真实退出后清 I/O 引用和 runtime 的 `_storage_started/storage_ready`；start_storage 成功重置 `_storage_closed=False`。core 保留 runtime/adapter 并传播未退出异常，阻止 QueueFS/AGFS 提前关闭。 - -这里的预算是调用方的有界观察，不是强杀保证：`run` 取消后的 join、`_execute` 的取消收尾和线程内部清理可能等待非合作协程；其生命周期必须仍可由 pending/requests/thread 引用观察。此机制不撤回服务端已收到的 Redis 写入，也不提供跨槽 fencing。Session 外层 timeout 与其余 core startup/optional backend 不变。 - -```python -from __future__ import annotations - -import asyncio -import concurrent.futures -import threading -import time -from dataclasses import dataclass, field -from typing import Awaitable, Callable, TypeVar -from .api import ExpiryScheduler -from .config import ExpiryConfig -from .consumer import ExpiryConsumer -from .keys import ExpiryKeys, encode_component -from .leases import ShardLeaseCoordinator -from .metrics import ExpiryMetrics -from .models import StoreOperationError -from .redis_store import RedisExpiryStore -from .publisher import ExpiryPublisher -from .registry import HandlerRegistry -from .service import _ExpirySchedulerFacade, ExpiryRuntimeState -from .shard_runner import ShardRunner - -T = TypeVar("T") - - -@dataclass(eq=False) -class _RedisRequest: - finished: concurrent.futures.Future = field(default_factory=concurrent.futures.Future) - cancelled: threading.Event = field(default_factory=threading.Event) - task: asyncio.Task | None = None # 仅Redis loop读写；finished只在实际task退出后完成。 - - -class RedisLoopExecutor: - """专属后台线程 + asyncio loop；跟踪真实Redis任务，不运行session commit。""" - - def __init__(self, *, max_inflight: int, request_budget_seconds: float) -> None: - self._max_inflight = max_inflight - self._request_budget_seconds = request_budget_seconds - self._loop: asyncio.AbstractEventLoop | None = None - self._thread: threading.Thread | None = None - self._gate: asyncio.Semaphore | None = None - self._accepting = False - self._ready = threading.Event() - self._requests: set[_RedisRequest] = set() - self._requests_lock = threading.Lock() - self._stop_requested = False - - async def start(self) -> None: - """已accepting时幂等；旧thread引用仍在则shard_not_stopped，不创建第二线程。 - - 清ready/stop_requested，启动线程并等待ready；gate仅在Redis loop创建。 - """ - ... - - async def run(self, operation: Callable[[], Awaitable[T]]) -> T: - """调用方取消后仍等待真实finished；多次取消不能切断wrapper的生命周期桥接。""" - loop = self._loop - if loop is not None and asyncio.get_running_loop() is loop: - # cleanup parent已登记，嵌套client关闭允许在quiesce之后执行。 - return await self._execute(operation) - if not self._accepting or loop is None: - raise StoreOperationError("not_started", "Redis expiry I/O loop is not running") - request = self._submit(operation) - finished = asyncio.wrap_future(request.finished) - try: - return await asyncio.shield(finished) - except asyncio.CancelledError: - self._cancel(request) - await self._join(finished) - raise - - async def quiesce(self, timeout_seconds: float) -> None: - """先拒绝新外部请求，以asyncio.wait有界等待登记请求的真实finished。 - - 包括尚未开始的请求；done读取异常，pending安装异常消费回调但不丢强引用。 - 未退出抛shard_not_stopped，不关闭client/loop/thread。 - """ - ... - - async def run_cleanup(self, operation: Callable[[], Awaitable[T]], - timeout_seconds: float) -> T: - """quiesce后提交tracked raw parent；不经_execute先占gate，避免nested run递归。 - - cleanup自身使用显式timeout而非普通请求预算；超时/取消发一次cancel， - 保留request并消费后续异常，超时报shard_not_stopped，不把取消当退出。 - """ - ... - - async def close(self, timeout_seconds: float) -> None: - deadline = time.monotonic() + max(0.0, timeout_seconds) - await self.quiesce(timeout_seconds) - loop, thread = self._loop, self._thread - if loop is not None and not self._stop_requested: - self._stop_requested = True - loop.call_soon_threadsafe(loop.stop) - if thread is not None: - await asyncio.to_thread(thread.join, max(0.0, deadline - time.monotonic())) - if thread.is_alive(): - raise self._not_stopped() # 保留引用，重试不重复stop；start不能新建线程。 - self._loop = None - self._thread = None - self._gate = None - - def _submit(self, operation: Callable[[], Awaitable[T]], *, - cleanup: bool = False) -> _RedisRequest: - """requests先登记后call_soon_threadsafe创建task，关闭须看到尚未开始的请求。 - - 普通请求task运行_execute；cleanup task直接await operation，不先占gate。 - task的done callback调用_request_done；预先取消的request启动时立即cancel。 - 无loop或已stop_requested拒绝；调度失败撤销登记。 - """ - ... - - def _cancel(self, request: _RedisRequest) -> None: - """threading.Event确保只向远端task发一次cancel；先取消后启动也不会漏掉。""" - ... - - def _request_done(self, request: _RedisRequest, task: asyncio.Task) -> None: - """真实task终止才将result/error/CancelledError写finished，然后摘除requests。""" - ... - - @staticmethod - def _consume_exception(future: asyncio.Future) -> None: - if not future.cancelled(): - future.exception() - - @staticmethod - async def _join(future: asyncio.Future) -> None: - while not future.done(): - try: - await asyncio.shield(future) - except (asyncio.CancelledError, Exception): - pass # 即使调用方再次cancel也保留等待；不重复cancel原始operation。 - if not future.cancelled(): - future.exception() - - @staticmethod - def _not_stopped() -> StoreOperationError: - return StoreOperationError( - "shard_not_stopped", "Redis expiry work or its I/O thread has not terminated", - ) - - def _thread_main(self) -> None: - """创建loop/gate并发布ready，run_forever结束后cancel/gather残余任务并关loop。 - - 线程内清理不冒充有界强杀：close在外层有界join，非合作清理仍alive则保留引用。 - """ - ... - - async def _execute(self, operation: Callable[[], Awaitable[T]]) -> T: - """gate内创建原始operation task，总请求预算含gate等待。 - - 超时cancel原始task后_join至真实done再报redis_timeout/outcome_unknown； - finally也join尚未退出的operation，防止外层task先done而远端工作仍存活。 - 预算触发取消并非硬终止，亦不证明服务端Redis写入撤销。 - """ - ... - - -@dataclass(frozen=True) -class ExpiryRuntimeRoles: - report: bool = False - publish: bool = False - consume: bool = False - - -class ExpiryRuntime: - def __init__(self, config: ExpiryConfig, *, roles: ExpiryRuntimeRoles, - metrics: ExpiryMetrics | None = None) -> None: - """构造 executor/store/service；不初始化 AGFS 或模型。""" - self._config = config - self._metrics = metrics or ExpiryMetrics() - self._keys = ExpiryKeys(encode_component(config.service_name)) - self._io = RedisLoopExecutor( - max_inflight=config.redis.max_inflight_requests, - request_budget_seconds=config.redis.request_budget_seconds, - ) - self._store = RedisExpiryStore(config.redis, self._keys, self._io, self._metrics) - self._registry = HandlerRegistry() - self._state = ExpiryRuntimeState() - self._facade = _ExpirySchedulerFacade( - self._store, self._registry, self._state, shard_count=config.shard_count, - reporting_enabled=roles.report, metrics=self._metrics, - ) - self._leases = ShardLeaseCoordinator(self._store, config, self._metrics) - self._shard_runner = ShardRunner(self._leases, config, self._metrics) - self._publisher: ExpiryPublisher | None = None - self._consumer: ExpiryConsumer | None = None - self._tasks: dict[str, asyncio.Task[None]] = {} - self._stop = asyncio.Event() - self._storage_started = False - self._storage_closed = False # store cleanup成功与I/O线程退出分别记录。 - - @property - def scheduler(self) -> ExpiryScheduler: - """仅供启动组装层取得协议引用，再注入业务；不把 runtime 交给业务。""" - return self._facade - - async def start_storage(self) -> None: - """依次 executor.start -> store.open -> ensure_namespace，置内部 storage_ready。 - - 成功时置_storage_started/storage_ready，并重置_storage_closed=False。 - 此时只开放本地 handler 注册，report 等 start_roles 完成才可用。 - 任一步失败按逆序释放已建资源，不启动 consumer。 - """ - ... - - async def start_roles(self) -> None: - """冻结本地 registry 后启动组装层选择的 publisher/consumer。 - - 同一_shard_runner注入scheduler/consumer，不把Coordinator交给worker。 - API 须先完成业务注册和 QueueFS 启动，再调用这里。 - 独立 publisher 没有本地 handler，直接使用 SUPPORTED_ITEM_TYPES。 - """ - bindings = self._registry.freeze() - self._stop.clear() - if self._roles.publish: - self._publisher = ExpiryPublisher( - self._store, self._shard_runner, self._config, self._metrics, - ) - self._spawn_role("publisher", self._publisher.run(self._stop)) - if self._roles.consume: - self._consumer = ExpiryConsumer( - self._store, self._shard_runner, bindings, self._config, self._metrics, - ) - self._spawn_role("callback", self._consumer.run(self._stop)) - self._state.accepting_calls = True - - async def wait(self) -> None: - """独立进程等待角色任务结束；任一角色非预期退出向 CLI 传播异常。""" - ... - - async def stop_roles(self) -> None: - """先拒绝新调用/set stop；grace后cancel，所有join均用asyncio.wait有界观察。 - - 保留未终止role task和runner pending；仅真实done才读异常并移除引用。 - 未退出则抛shard_not_stopped，保持store/I/O/业务依赖打开，允许后续再次观察。 - """ - self._state.accepting_calls = False - self._stop.set() - tasks = set(self._tasks.values()) - if tasks: - _, pending = await asyncio.wait(tasks, timeout=self._config.shutdown_timeout_seconds) - for task in pending: - task.cancel() - if pending: - await asyncio.wait(pending, timeout=self._config.redis.request_budget_seconds) - for name, task in list(self._tasks.items()): - if task.done(): - if not task.cancelled(): - task.exception() - self._tasks.pop(name, None) - settled = await self._shard_runner.wait_pending(self._config.redis.request_budget_seconds) - if self._tasks or not settled: - raise StoreOperationError("shard_not_stopped", "Expiry work is still running; dependencies remain open") - - async def close(self) -> None: - """依次停止role、等待真实I/O、同loop清理store、停止并join线程。 - - stop_roles/quiesce未退出不进入依赖清理；cleanup之后finally尝试io.close， - io.close若报未退出则保留storage标志，不能提前清状态或继续关业务依赖。 - 各阶段分别使用配置预算，非整个close共享一个总截止。 - """ - await self.stop_roles() - if not self._storage_started: - return - await self._io.quiesce(self._config.shutdown_timeout_seconds) - try: - if not self._storage_closed: - await self._io.run_cleanup(self._store.close, self._config.shutdown_timeout_seconds) - self._storage_closed = True - finally: - await self._io.close(self._config.shutdown_timeout_seconds) - self._storage_started = False - self._state.storage_ready = False - - def _spawn_role(self, name: str, operation: Awaitable[None]) -> None: - """保存 Task 强引用与退出结果；角色异常要可见，不能静默失去调度循环。""" - ... -``` - -## 10. lua/：精简后的全部脚本契约 - -仅保留12个Lua文件（含共享前缀）。不再有源状态初始化、源关闭水位、注销伴随状态或失败归档脚本。所有脚本在写前校验参数与key类型；脚本执行异常不保证回滚。 - -### 10.1 _common.lua - -```lua -local BASE = 1099511627776 -local MAX_INT = 9007199254740991 - -local function decimal(n) - -- 时间和claim score使用精确十进制字符串。 - return string.format('%.0f', n) -end - -local function now_ms() - -- 返回当前执行节点TIME，不能用另一slot节点或客户端时间替代源准入时间。 - local t = redis.call('TIME') - return tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000) -end - -local function integer_arg(value, min_value, max_value) - -- 严格校验规范十进制、整数和范围，不接受NaN/inf；错误发生在任何写入前。 -end - -local function expect_type(key, allowed) - -- TYPE必须为none或指定类型，否则写前报错。 -end - -local function require_owner(key, token) - -- GET必须等于token且PTTL>0，否则owner_lost；不重新创建或续活失效锁。 -end - -local function page_after(key, lower, upper, last_score, last_member, limit) - -- 稳定score/member分页：原游标仍在则rank快路径，否则rank二分定位。 - -- 有界读取limit+1，返回items及next，不使用客户端OFFSET，不扫描全部同score。 -end -``` - -### 10.2 ensure_metadata.lua - -```lua --- KEYS = M --- ARGV = item_type, local_id, account_id, user_id, ttl_ms --- RETURN = 'created' | 'refreshed' --- ttl_ms=max(2*timeout_seconds*1000,86400000)，由store统一计算。 --- 先校验HASH/none、全部身份和ttl范围；无schema字段。 --- 不存在则HSET身份；存在逐字段校验，冲突在任何写入前报错。 --- 最后PEXPIRE ttl_ms；只有前一步R明确成功后才调用本脚本。 -``` - -### 10.3 upsert_resource.lua - -```lua --- KEYS = R；ARGV = item_id, requested_due_ms --- RETURN = decimal(accepted_due_ms) --- 仅report调用；不存在V，不读C，没有revision/CAS/请求顺序比较。 -expect_type(KEYS[1], 'zset') -local requested = integer_arg(ARGV[2], 0, MAX_INT) -local current_time = now_ms() -local accepted = math.max(requested, current_time + 1) --- 写前检查item_id非空；向上对齐后的桶分钟必须仍在精确整数范围内。 -local max_supported_due = math.floor(MAX_INT / 60000) * 60000 -if accepted > max_supported_due then return redis.error_reply('invalid_report') end -redis.call('ZADD', KEYS[1], decimal(accepted), ARGV[1]) -return decimal(accepted) -``` - -这里校正的是写入时的未来到期时间，不能识别业务先后顺序。重试报告可能覆盖并发更新；接口不返回DUPLICATE/STALE。R已经成功但M失败时不补偿删除R。 - -### 10.4 source_time.lua - -```lua --- KEYS = R （用于路由到R所在Redis slot）；无ARGV。 --- RETURN = decimal(source_now_ms) --- 本脚本只读，不修改R，也不创建任何伴随key。 -return decimal(now_ms()) -``` - -store读C并使用该时间构造U=floor(T/60000)*60000。U=C或无数据则跳过；不执行“关闭窗口”写入。 - -### 10.5 read_source_page.lua - -```lua --- KEYS = R --- ARGV = lower_exclusive, upper_inclusive, after_score_or_empty, after_member, limit --- RETURN = cjson {items=[[item,score_string],...], next=[score_string,item] | false} --- 校验后调用page_after；整个范围为空返回空items，不修改C。 --- scheduler先limit=1定位下一个非空分钟，再分页读完整分钟，避免逐空分钟循环。 -``` - -### 10.6 publish_minute.lua - -```lua --- KEYS = G, C, scheduler_owner （同槽） --- ARGV = owner_token, expected_C, minute_ms, member_count, registry_member... --- RETURN = {'published'|'already_published', decimal(C)} --- 先验证key类型、C存在、分钟对齐、全部注册member与shard/minute一致。 --- 空分钟不调用，本脚本要求member_count>=1。 -require_owner(KEYS[3], ARGV[1]) -local expected = integer_arg(ARGV[2], 0, MAX_INT) -local minute = integer_arg(ARGV[3], 0, MAX_INT) -local count = integer_arg(ARGV[4], 1, 1024) -local current = tonumber(redis.call('GET', KEYS[2])) -if current >= minute then - return {'already_published', decimal(current)} -end -if current ~= expected or minute <= current then - return redis.error_reply('progress_conflict') -end --- 中间空分钟已由scheduler扫描证实；只要求单调，不要求minute=current+60000。 -for i = 1, count do - redis.call('ZADD', KEYS[1], decimal(minute), ARGV[4+i]) -end -redis.call('SET', KEYS[2], decimal(minute)) -return {'published', decimal(minute)} -``` - -owner比较与发布原子，但不能约束其他slot上已经发出的B写入；保留主文档10.3的限制。 - -### 10.7 read_registry_page.lua - -```lua --- KEYS = G；ARGV = after_minute_or_empty, after_member, limit --- RETURN = cjson {items=[[registry_member,minute_string],...], next=[minute,member] | false} --- 使用page_after；G只包含发布完成的桶，无需额外用客户端minute=max_attempts：仅加入exhausted，保持原score，不删除、不新增Redis记录。 --- score=0新项优先，但至少为已有可重试项保留一个槽；claims总数<=limit。 --- consumer对exhausted先日志再ack.lua，不调用handler；总检查/写入<=2*limit。 - -local function next_score(old_score, now_seconds, max_attempts) - -- now_seconds在外部通过范围检查。 - if old_score == 0 then return now_seconds end - local attempt = math.floor(old_score / BASE) - if attempt + 1 >= max_attempts then error('attempts_exhausted') end - local next_time = math.max(now_seconds, old_score % BASE + 1) - if next_time >= BASE then error('invalid_claim_time') end - return (attempt + 1) * BASE + next_time -end -``` - -### 10.9 ack.lua - -```lua --- KEYS = B；ARGV = item_id, exact_claim_score；RETURN = 0 | 1。 --- 成功回调ACK和耗尽日志后的清理共用；不删除R/M。 -local expected = integer_arg(ARGV[2], 1, 256 * BASE - 1) -local current = redis.call('ZSCORE', KEYS[1], ARGV[1]) -if current and tonumber(current) == expected then - return redis.call('ZREM', KEYS[1], ARGV[1]) -end -return 0 -``` - -### 10.10 acquire_owner.lua - -```lua --- KEYS = owner；ARGV = token, ttl_ms（本期120000） --- RETURN = {'owned', expires_at_ms} | {'busy', 0} --- SET NX PX；同token只确认本次请求已取得锁，不因此延长lease。 --- 新token未取得则busy；返回TIME/PTTL供观测，本地使用请求起点+TTL保守截止。 -``` - -### 10.11 renew_owner.lua - -```lua --- KEYS = owner；ARGV = token, ttl_ms（120000）；RETURN = expires_at_ms | 0。 --- GET==token且PTTL>0才PEXPIRE；不创建缺失锁，不延长其他token的锁。 --- 任意不匹配/异常/超时在调用方立即使当前lease失效，不等到下一个20秒周期。 -``` - -### 10.12 release_owner.lua - -```lua --- KEYS = owner；ARGV = token；RETURN = 0 | 1。 --- 本地续租task已经停止并await后调用，防止释放与自己的续租交错。 -if redis.call('GET', KEYS[1]) == ARGV[1] then - return redis.call('DEL', KEYS[1]) -end -return 0 -``` - -## 11. session_expiry.py：业务适配，不把 policy 塞进调度层 - -```python -from __future__ import annotations - -from datetime import datetime, timezone -from typing import TYPE_CHECKING -from openviking.server.identity import RequestContext, Role -from openviking.session.auto_commit_policy import AutoCommitPolicy -from openviking.session.session import SessionExpirySnapshot -from openviking.scheduling.expiry import ( - CallbackRequest, CallbackResult, CallbackStatus, HandlerOptions, ItemIdentity, - ExpiryRegistration, ExpiryScheduler, ExpiryMode, -) -from openviking_cli.session.user_id import UserIdentifier - -if TYPE_CHECKING: - from openviking.service.session_service import SessionService - - -def install_session_expiry( - scheduler: ExpiryScheduler, sessions: SessionService, *, concurrency: int = 8 -) -> None: - """业务模块自己的启动注册函数；core 只调用业务初始化，不认识处理器细节。""" - scheduler.register_handler("session", SessionTimeoutHandler(sessions, scheduler), - options=HandlerOptions(concurrency=concurrency, max_attempts=3)) - sessions.set_session_expiry_adapter(SessionExpiryAdapter(scheduler)) - - -def effective_idle_timeout(policy: dict | None) -> int | None: - """None 表示禁用；不能将 None 传给 from_dict 后意外启用默认 policy。""" - if policy is None: - return None - timeout = AutoCommitPolicy.from_dict(policy).idle_timeout_seconds - return timeout if timeout > 0 else None - - -def timestamp_ms(value: str) -> int: - """沿用内核时间格式解析规则，统一 UTC 毫秒；错误显式上报。""" - ... - - -class SessionExpiryAdapter: - """实现本业务的 SessionExpiryReporter；承接已成功持久化的消息快照。""" - - def __init__(self, scheduler: ExpiryScheduler) -> None: - """只依赖到期调度协议；不依赖 store、runtime 或框架内部指标类型。""" - self._scheduler = scheduler - - def build_report(self, snapshot: SessionExpirySnapshot) -> ExpiryRegistration | None: - """在业务层计算 due；禁用时返回 None，不回改已有历史 score。""" - timeout = effective_idle_timeout(snapshot.auto_commit_policy) - if timeout is None: - return None - return ExpiryRegistration( - identity=ItemIdentity("session", snapshot.account_id, - snapshot.user_id, snapshot.session_id), - due_at_ms=timestamp_ms(snapshot.last_message_at) + timeout * 1000, - timeout_seconds=timeout, - mode=ExpiryMode.IDLE_ONESHOT, - ) - - async def report_after_append(self, snapshot: SessionExpirySnapshot) -> None: - """消息已落盘，报告最终失败记录指标，不伪装消息回滚。 - - 立即 await，没有内存去抖；CancelledError 不被 except Exception 吞掉。 - """ - try: - report = self.build_report(snapshot) - if report is not None: - await self._scheduler.report(report) - except Exception as exc: - self.record_report_failure("report_failed", exc) - - async def unregister_after_delete(self, identity: ItemIdentity) -> None: - """业务删除已成功后尽力注销；失败记录日志/指标，不反向恢复业务数据。""" - ... - - def record_report_failure(self, reason: str, exc: Exception | None = None) -> None: - """使用业务自身的日志/指标记录无法报告计时；不接触框架内部队列或指标对象。""" - ... - - -class SessionTimeoutHandler: - def __init__(self, sessions: SessionService, scheduler: ExpiryScheduler) -> None: - """持有业务服务和公开scheduler协议；不持有Redis实现，冷却重投也走report。""" - self._sessions, self._scheduler = sessions, scheduler - - async def handle(self, request: CallbackRequest) -> CallbackResult: - """只按业务身份构造 ctx 并提交，不感知元数据缓存、调度桶或消费凭据。""" - ctx = self._make_context(request.identity) - # SessionService.commit_on_timeout 自行把明确业务 NotFound 转 skipped。 - # 存储/网络异常继续抛给 consumer，保留 Redis 成员。 - result = await self._sessions.commit_on_timeout(request.identity.local_id, ctx) - if result.get("status") == "deferred": - await self._report_deferred(request.identity, result) - return CallbackResult(CallbackStatus.COMPLETED, "idle_rescheduled") - return self._to_callback_result(result) - - async def _report_deferred(self, identity: ItemIdentity, result: dict) -> None: - """使用服务层纯检查生成的新due报告，不持有Session提交锁，不取旧score或旧timeout。 - - 必须await R→M成功后才返回。异常传播为旧事件RETRY，不吞错假装已重投。 - 只接受 idle_cooldown 和 idle_activity；disabled/无消息不生成deferred； - 重投不更新last_idle_commit_accepted_at。 - """ - if result.get("reason") not in {"idle_cooldown", "idle_activity"}: - raise ValueError("Unexpected deferred reason") - await self._scheduler.report(ExpiryRegistration( - identity=identity, due_at_ms=result["due_at_ms"], - timeout_seconds=result["timeout_seconds"], mode=ExpiryMode.IDLE_ONESHOT, - )) - - def _make_context(self, identity: ItemIdentity) -> RequestContext: - """使用资源自身 account/user，沿用业务用户权限，不提升到 root。""" - return RequestContext( - user=UserIdentifier(account_id=identity.account_id, user_id=identity.user_id), - role=Role.USER, - ) - - def _to_callback_result(self, result: dict) -> CallbackResult: - """accepted 或明确 skipped -> COMPLETED；未知返回结构 -> RETRY/抛协议错误。 - - 不把HTTP 200或任意truthy字典当作accepted；原commit API返回accepted即成功，不等后台任务。 - """ - status = result.get("status") - if status == "accepted": - return CallbackResult(CallbackStatus.COMPLETED, "accepted") - if status == "skipped" and result.get("reason") in { - "not_found", "idle_disabled", "no_messages", - }: - return CallbackResult(CallbackStatus.COMPLETED, result["reason"]) - return CallbackResult(CallbackStatus.RETRY, "unexpected_commit_result") -``` - -`effective_idle_timeout` 只用于上报适配器；超时判断由SessionService复用自身的AutoCommitPolicy与时间工具完成。Session提交内部不做idle判断、不掌握deferred或Redis，也不反向import此service文件。 - -## 12. session.py：新增时间字段与追加 reporter，保留原通用提交 - -### 12.1 新增字段和内部类型 - -```python -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, Protocol - - -@dataclass(frozen=True) -class SessionExpirySnapshot: - account_id: str - user_id: str - session_id: str - last_message_at: str - auto_commit_policy: dict[str, Any] | None - # policy 必须深拷贝；后续 Session 对象更新不得改变这次报告。 - - -class SessionExpiryReporter(Protocol): - async def report_after_append(self, snapshot: SessionExpirySnapshot) -> None: - """持久化成功、释放 session lock 后立即等待；reporter 自己处理上报失败策略。""" - ... - - def record_report_failure(self, reason: str, exc: Exception | None = None) -> None: - """消息仍然成功，但无法构造可信计时快照时调用。""" - ... -``` - -在现有 `SessionMeta` 追加字段，不新建第二份 metadata 文件： - -```python -@dataclass -class SessionMeta: - # 现有全部字段原样保留；这里只展示新增字段和要修改的方法。 - last_idle_commit_accepted_at: str = "" # 服务层best-effort记录，不是ready原子记录或修复缓存。 - - def to_dict(self) -> dict[str, Any]: - """只加入接受时间字段，保留现有全部字段与输出兼容性，不新增idle归档指针。""" - ... - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> SessionMeta: - """缺字段使用空串；时间解析沿用内核工具，不从归档查证或恢复此字段。""" - ... -``` - -### 12.2 Session 新增/修改的完整方法目录 - -以下 class 是现有类的**增量视图**，不是替换整个类。构造函数保留原参数，仅在末尾加可选 reporter： - -```python -class Session: - def __init__( - self, viking_fs: "VikingFS", vikingdb_manager: "VikingDBManager | None" = None, - session_compressor: "SessionCompressor | None" = None, - user: "UserIdentifier | None" = None, ctx: "RequestContext | None" = None, - session_id: str | None = None, session_uri: str | None = None, - auto_commit_threshold: int = 8000, - tool_output_externalization_config: "ToolOutputExternalizationConfig | None" = None, - agent_evolution_enabled: bool = True, usage_reporter: "UsageReporter | None" = None, - agent_evolution_enabled_provider: "Callable[[], bool | Awaitable[bool]] | None" = None, - memory_policy_provider: "MemoryPolicyProvider | None" = None, - vlm_resolver: "VLMResolver | None" = None, - expiry_reporter: SessionExpiryReporter | None = None, - ) -> None: - """原初始化保留，仅增加 self._expiry_reporter = expiry_reporter。""" - ... - - async def _append_messages_authoritatively(self, messages: "list[Message]") -> None: - """修改现有方法：锁内追加并保存业务meta，锁外立即 await reporter。 - - 原写消息失败时不报告；无 VikingFS 的原内存分支不宣称持久化成功。 - 元信息读取失败时维持既有消息处理规则，但不伪造可信的policy/时间快照。 - """ - ... - - def _build_expiry_snapshot_locked(self) -> SessionExpirySnapshot: - """新增：只在最新消息/meta 写入成功后调用；深拷贝 policy 和身份。 - - 仅深拷贝当前时间、policy和身份，不分配资源更新版本。 - """ - ... - - # commit/commit_async及原通用提交内部实现保持不变，不为timer新增或拆分提交函数。 - # 服务层直接调用原commit_async；Session内部没有idle判断、deferred、Redis调用。 - # 原返回结构、参数校验、锁、ready、QueueFS及恢复能力继续存在。 - # 各既有meta更新路径须保留接受时间字段，但不生成或恢复该字段。 -``` - -### 12.3 追加函数内部放置位置 - -下面是已有 `_append_messages_authoritatively()` 中的插入结构，省略的行保留原写消息逻辑，不是另新增一个消息入口： - -```python -# 已持有 session path lock，且最新 meta 读取状态已知。 -# 原有：reload -> _apply_appended_messages_to_state -> append/write messages.jsonl。 -snapshot = None -snapshot_error = None -metadata_reload_error = None -await self._save_meta() -if self._expiry_reporter is not None: - try: - # save成功后，self._meta已是本次路径锁内刚持久化的权威状态。 - snapshot = self._build_expiry_snapshot_locked() - except Exception as exc: - snapshot_error = exc -# 原 finally 释放 session path lock。 -# 以下放到 try/finally 外，只在持久化成功后执行： -if self._expiry_reporter is not None: - if snapshot is not None: - await self._expiry_reporter.report_after_append(snapshot) - else: - self._expiry_reporter.record_report_failure("snapshot_build_failed", snapshot_error) -``` - -实现时处理“消息文件已写成，但 meta 写失败”的既有业务结果，不额外声称它们是事务。追加前 meta 重读失败不再单独阻止 report：只要随后 `_save_meta()` 成功，就必须用刚持久化的状态构造 snapshot；只有 snapshot 本身无法构造时才不 report。`update_config()` 已在 session lock 内重读 meta，序列化时保留新增的接受时间字段；`update_event_search_tags()` 继续委托它，无需新建计时更新逻辑。 - -### 12.4 原通用 commit、QueueFS 与 Working Memory 契约 - -以下是保留清单，不新增一套提交实现： - -| 位置/函数 | 必须保留的原行为 | -|---|---| -| Session.commit / commit_async | 原公开签名、参数校验、锁和返回结构不变；enable_working_memory的None/True/False完整透传 | -| 原通用ready与归档恢复 | 原能力和兼容协议不删除、不添加idle标记、接受时间或额外严格判断 | -| MemoryPolicy.from_dict / to_dict | enable_working_memory缺省False；序列化保留working_memory.enabled的真实布尔值 | -| SessionCommitMsg.to_dict / from_dict | 新消息memory_policy_version=1；旧队列沿用既有WM默认值兼容，显式False不覆盖 | -| Session.resume_queued_commit / _run_memory_extraction | 使用队列策略及原后台任务流程，不因timer重新解析当前用户策略 | -| Session._write_done_file | enable_working_memory参数和同名.done字段继续保留 | -| ArchiveStore.normalize_done / read_done / scan_states | 保留旧working_memory_enabled兼容；WM关闭且无摘要仍可正常完成 | - -`memory_policy_version` 和原归档格式版本是既有持久业务协议，与Redis M.schema_version不同，不删除、不复制到R/M/B。无需因本方案修改memory_policy.py、archive_store.py或session_commit_msg.py。 - -idle服务层调用不覆盖WM，原始归档、QueueFS及符合policy的长期记忆提取继续存在。原commit的accepted及skipped字段（包括`effective_enable_working_memory`）原样返回；deferred只在SessionService/外层handler之间使用，不扩展Session公开commit返回语义。timer以原API返回accepted为成功，不等后台任务，不用ready恢复idle接受记录。 - -## 13. session_service.py：外层纯检查、原提交与 best-effort 接受记录 - -以下是服务层增量骨架；使用现有factory/get、路径锁、身份和时间工具，不将检查迁入Session。`get(..., auto_create=False)`必须加载最新session/meta，明确不存在才转skipped，其他读取错误不能退回旧内存状态。 - -```python -from __future__ import annotations - -import json -import logging -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Optional -from openviking.core.namespace import canonical_session_uri -from openviking.server.identity import RequestContext -from openviking.session import Session -from openviking.session.archive_store import is_storage_not_found -from openviking.session.auto_commit_policy import AutoCommitPolicy -from openviking.utils.time_utils import get_current_timestamp, parse_iso_datetime -from openviking.scheduling.expiry import ExpiryScheduler - -if TYPE_CHECKING: - from .session_expiry import SessionExpiryAdapter - -# 复用现有logger与meta锁超时配置，不新增另一套时间解析或锁策略。 -logger = logging.getLogger(__name__) -_SESSION_META_LOCK_TIMEOUT_SECONDS = 30.0 - - -class SessionService: - # __init__ 原签名不变，初始化 self._session_expiry_adapter = None。 - - def configure_expiry(self, scheduler: "ExpiryScheduler") -> None: - """业务初始化入口：由 session 模块注册自己的 handler 和适配器。""" - from .session_expiry import install_session_expiry - install_session_expiry(scheduler, self) - - def set_session_expiry_adapter(self, adapter: "SessionExpiryAdapter | None") -> None: - """新增：初始化阶段注入；关闭时摘除，不扫描已经创建的 Session 对象。""" - self._session_expiry_adapter = adapter - - def session(self, ctx: RequestContext, session_id: str | None = None, *, - session_uri: str | None = None) -> Session: - """原 factory 参数/行为保留；构造 Session 时增加 expiry_reporter=adapter。 - - 原 memory_policy_provider、usage_reporter、vlm_resolver 等参数全部继续传递。 - """ - ... - - async def commit_on_timeout(self, session_id: str, ctx: RequestContext) -> dict[str, Any]: - """加载最新状态、检查一次，直接复用原commit；不自动创建session。 - - 不在Session提交锁内最终复检idle，期间追加的新消息允许一起提交。 - 原API返回accepted即成功，不等待后台任务；返回结构原样保留。 - 入口加载/检查/commit异常传播，只有明确NotFound转skipped。 - """ - try: - session = await self.get(session_id, ctx, auto_create=False) - except Exception as exc: - if not is_storage_not_found(exc): - raise - return {"status": "skipped", "reason": "not_found"} - decision = self._check_timeout_commit(session, datetime.now(timezone.utc)) - if decision is not None: - return decision # skipped/deferred不记录接受时间，不调用commit。 - result = await session.commit_async( - keep_recent_count=0, - persist_keep_recent_count=False, - record_auto_commit_success=True, - ) - if result.get("status") == "accepted": - await self._record_idle_commit_time(session_id, ctx, get_current_timestamp()) - # 原生命周期/归档指标保留；不把记录失败改成业务RETRY。 - return result - - @staticmethod - def _check_timeout_commit(session: Session, now: datetime) -> Optional[dict]: - """纯检查：policy/timeout -> no_messages -> cooldown -> last_message_due。 - - 只读最新加载状态，不取提交锁，不读归档，不写meta/Redis。 - None表示直接调用原commit；deferred只由服务层/外层handler掌握。 - 时间解析使用parse_iso_datetime；缺失last_message_at或解析异常向外抛。 - """ - policy_data = session.meta.auto_commit_policy - if policy_data is None: - return {"status": "skipped", "reason": "idle_disabled"} - timeout = AutoCommitPolicy.from_dict(policy_data).idle_timeout_seconds - if timeout <= 0: - return {"status": "skipped", "reason": "idle_disabled"} - if not session.messages: - return {"status": "skipped", "reason": "no_messages"} - if now.tzinfo is None: - now = now.replace(tzinfo=timezone.utc) - now_ms = int(now.timestamp() * 1000) - accepted_at = session.meta.last_idle_commit_accepted_at - if accepted_at: - accepted = parse_iso_datetime(accepted_at) - if accepted.tzinfo is None: - accepted = accepted.replace(tzinfo=timezone.utc) - if (now - accepted).total_seconds() < timeout: - return { - "status": "deferred", "reason": "idle_cooldown", - "timeout_seconds": timeout, "due_at_ms": now_ms + timeout * 1000, - } - if not session.meta.last_message_at: - raise RuntimeError("Session last_message_at is missing") - last_message = parse_iso_datetime(session.meta.last_message_at) - if last_message.tzinfo is None: - last_message = last_message.replace(tzinfo=timezone.utc) - activity_due_ms = int(last_message.timestamp() * 1000) + timeout * 1000 - if now_ms < activity_due_ms: - return { - "status": "deferred", "reason": "idle_activity", - "timeout_seconds": timeout, "due_at_ms": activity_due_ms, - } - return None - - async def _record_idle_commit_time( - self, session_id: str, ctx: RequestContext, accepted_at: str, - ) -> None: - """accepted之后的best-effort记录，失败日志后仍返回accepted。 - - 服务层取session root exact路径锁，重读最新raw meta字典，只patch两字段； - 保留并发append/config更新及未知字段，不调用旧Session._save_meta。 - 新旧有效接受时间只取max，旧时间不回写；不读idle归档或修复缓存。 - meta不存在时跳过，不创建目录/文件，不重建已删除session。 - helper内部读取/锁/写失败只记录日志；取消沿用既有异常语义。 - """ - try: - self._ensure_initialized() - fs = self._viking_fs - session_uri = canonical_session_uri(ctx, session_id) - path = fs._uri_to_path(session_uri, ctx=ctx) - lease = await fs._async_agfs.pathlock_acquire_exact( - path, timeout_secs=_SESSION_META_LOCK_TIMEOUT_SECONDS, - ) - try: - # 在与既有delete协作的session锁内直接读取；缺失不能以旧meta补建。 - metadata = json.loads(await fs.read_file(f"{session_uri}/.meta.json", ctx=ctx)) - if not isinstance(metadata, dict): - raise ValueError("Session metadata must be a JSON object") - current = parse_iso_datetime(accepted_at) - if current.tzinfo is None: - current = current.replace(tzinfo=timezone.utc) - previous = metadata.get("last_idle_commit_accepted_at") - if previous: - try: - previous_dt = parse_iso_datetime(previous) - if previous_dt.tzinfo is None: - previous_dt = previous_dt.replace(tzinfo=timezone.utc) - except (TypeError, ValueError): - logger.warning("Replacing invalid idle commit time for %s", session_id) - else: - if previous_dt >= current: - return # max合并：已有时间更晚/相同，不写旧值。 - metadata["last_idle_commit_accepted_at"] = accepted_at - metadata["updated_at"] = get_current_timestamp() - await fs.write_file( - f"{session_uri}/.meta.json", json.dumps(metadata, ensure_ascii=False), - ctx=ctx, lease_ref=lease, - ) - finally: - await fs._async_agfs.pathlock_release(lease) - except Exception: - logger.warning("Failed to record idle commit time for %s", session_id, exc_info=True) - - async def delete(self, session_id: str, ctx: RequestContext) -> bool: - """原业务删除成功后，await adapter.unregister_after_delete(identity)。 - - identity 取 ctx 的 account/user 和传入 session_id；不依赖可能已过期的 M。 - 原删除失败则不注销；不搜寻已投递 B；注销失败由适配器记录。 - """ - ... -``` - -`create/get/update_config/commit/commit_async/maybe_schedule_auto_commit/run_auto_commit` 的最新外部签名保留。SessionService.commit/commit_async已有的 `enable_working_memory: Optional[bool] = None` 不删除；继续以 `is not None` 筛选后传给Session，False必须保留。HTTP CommitRequest的StrictBool校验和router透传不变，新增idle handler不提供一次性WM覆盖。只有 factory 注入影响它们返回的 Session；Redis idle 回调不依赖旧自动提交的 has_running 扫描。 - -## 14. 配置文件：字段、校验和兼容入口 - -### 14.1 scheduling/expiry/config.py：轻量配置定义（新增拆分） - -实际读取发现 `openviking_cli.utils.config.__init__` 会聚合导入总配置、模型和解析器配置。配置类定义放入这个轻量文件，CLI 配置目录只保留重导出入口。store/publisher/consumer/runtime 均从 `.config` 导入；内部 publisher 启动模块仅在校验完整 `ov.conf` 时加载聚合配置。 - -```python -from __future__ import annotations - -import os -from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, model_validator - - -class ExpiryRedisConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - mode: Literal["standalone", "cluster"] = "standalone" - endpoints: list[str] = Field(default_factory=list) - username: str = "" - password_env: str = "" - password: str = Field(default="", repr=False) - tls: bool = False - connect_timeout_seconds: float = Field(default=1.0, gt=0) - socket_timeout_seconds: float = Field(default=2.0, gt=0) - request_budget_seconds: float = Field(default=5.0, gt=0) - max_connections: int = Field(default=64, ge=1) - max_inflight_requests: int = Field(default=128, ge=1) - - @model_validator(mode="after") - def validate_connection(self) -> ExpiryRedisConfig: - """校验 endpoint 格式、独立凭证字段和 standalone 地址数量。 - - endpoints 为空允许 Redis idle backend 未启用；启用时由聚合配置拒绝。 - endpoint 不允许携带凭证、path、query 或 fragment。 - 不尝试网络连接，不读密码值，不在配置对象构建时启动客户端。 - """ - ... - - def resolve_password(self) -> str | None: - """优先按 password_env 读取；变量不存在时失败，不静默降级为无密码。""" - ... - - -class ExpiryConfig(BaseModel): - model_config = ConfigDict(extra="forbid") - service_name: str = "" - publisher_separate_process: bool = True - shard_count: int = Field(default=128, ge=1, le=512) - session_consumer_concurrency: int = Field(default=8, ge=1) - precision_seconds: Literal[60] = 60 - owner_ttl_seconds: Literal[120] = 120 - owner_renew_interval_seconds: Literal[20] = 20 - scan_batch_size: int = Field(default=128, ge=1, le=4096) - cleanup_batch_size: int = Field(default=128, ge=1, le=4096) - registry_page_size: int = Field(default=128, ge=1, le=4096) - processing_timeout_seconds: int = Field(default=120, ge=1) - callback_poll_seconds: float = Field(default=0.5, gt=0) - shutdown_timeout_seconds: float = Field(default=30.0, gt=0) - redis: ExpiryRedisConfig = Field(default_factory=ExpiryRedisConfig) - - def validate_for_runtime(self) -> None: - """Redis idle backend 启用时要求 service_name/endpoints 并校验预算。 - - publisher 与 consumer 自动遍历0..N-1，每次持有一个shard；无静态候选清单。 - TTL固定120秒、续租20秒；任务处理超时独立配置。 - """ - ... -``` - -TTL 120秒和20秒续租来自用户约定，其余数值为压测起点。持续积压会长期持有当前shard，不承诺按资源量或实例严格均衡。 - -### 14.2 openviking_cli/utils/config/expiry_config.py - -```python -# 兼容内核配置目录的入口；类型定义只保留一份，避免默认值/校验分叉。 -from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig - -__all__ = ["ExpiryConfig", "ExpiryRedisConfig"] -``` - -### 14.3 memory_config.py - -```python -class SessionAutoCommitConfig(BaseModel): - # 原 enabled/check_interval_seconds/scan_rate_limit_files_per_second 保留。 - idle_backend: Literal["legacy", "redis"] = "legacy" - # enabled 的 description 同步修改：默认 policy 开关和所选 idle backend 的启用条件。 - # 不再写成一定启动 AGFS 扫描器。没有新增方法。 -``` - -### 14.4 open_viking_config.py - -```python -class OpenVikingConfig(BaseModel): - # 原全部字段保留。 - expiry: ExpiryConfig = Field(default_factory=ExpiryConfig) - - @model_validator(mode="after") - def _validate_expiry_backend(self) -> OpenVikingConfig: - """Redis idle backend 启用时校验完整 expiry 运行配置。""" - idle = self.memory.session_auto_commit - if idle.enabled and idle.idle_backend == "redis": - self.expiry.validate_for_runtime() - return self - - @classmethod - def from_dict(cls, config: dict[str, Any]) -> OpenVikingConfig: - """原加载逻辑保留，新增字段走 Pydantic 嵌套解析。 - - 现有实现先构造 instance，再单独设置 memory：因此 memory 应用完成后， - 必须再次显式调用 instance._check_expiry_backend()，不能只依赖首次构造的 validator。 - """ - ... - - def to_dict(self) -> dict[str, Any]: - """原 model_dump 已自动包含 expiry；保留签名，增加往返检查。""" - ... -``` - -既有配置仍默认 legacy。选择 Redis backend 后，report、publisher、consumer 作为一个完整能力启用，不再提供可独立组合的用户角色开关。 - -## 15. core.py：初始化位置必须放对 - -```python -class OpenVikingService: - # __init__ 原参数保留；初始化 _expiry_runtime = None。 - - async def initialize(self) -> None: - """在原初始化流程中插入： - - 1. SessionService 依赖/policy 已设置后，判断是否启用 Redis expiry。 - 2. 根据 idle_backend 选择是否启动原 SessionAutoCommitScheduler。 - 3. 原 QueueFS handler 注册、TaskTracker 恢复、queue_manager.start()。 - 4. Redis backend 启用时创建 API runtime，注册业务后统一 start_storage/start_roles。 - 5. 再完成其余初始化并对外提供服务。 - - publisher_separate_process=false 时 API runtime 额外运行 publisher。 - 中途失败须清理已建 expiry storage；不能留下空转 callback。 - """ - ... - - async def _initialize_expiry(self) -> None: - """启动组装层先调用所有已启用业务的注册入口，再统一启动 runtime。 - - 仅在 session 选择启用 Redis idle backend 时，调用 - self._session_service.configure_expiry(runtime.scheduler)。 - API runtime 固定使用 ExpiryRuntimeRoles(report=True, consume=True)， - publisher 是否同进程由 publisher_separate_process 决定。 - timer 为其他业务启用时不能顺带给 legacy session 安装 Redis 上报适配器。 - 新业务由其自己的初始化入口完成注册，core 不 import 各业务 handler 类。 - Redis idle backend 未启用时不 import redis，不创建线程。 - """ - ... - - async def _start_expiry_roles(self) -> None: - """所有业务注册完成后依次 start_storage/start_roles；无 runtime 则直接返回。""" - ... - - async def _stop_expiry(self) -> None: - """保持runtime强引用，await runtime.close；成功后才清runtime并摘除adapter。 - - runtime.close抛shard_not_stopped时原样向外传播，保留runtime和adapter， - 禁止新calls由runtime.accepting_calls=False负责，不能提前detach造成状态不一致。 - 不能继续关闭QueueFS/AGFS或其他未退出工作依赖；允许后续重试关闭/观察。 - """ - runtime = self._expiry_runtime - if runtime is None: - return - await runtime.close() - self._expiry_runtime = None - self._session_service.set_session_expiry_adapter(None) - - async def close(self) -> None: - """在QueueManager.stop之前await _stop_expiry；未退出错误传播并中止依赖关闭。 - - 其余startup/optional backend和既有无timer关闭路径保持不变。 - """ - ... -``` - -旧 scanner 创建条件具体改为： - -```python -use_legacy_idle = ( - session_auto_commit_config.enabled - and session_auto_commit_config.idle_backend == "legacy" -) -# 仅 use_legacy_idle 为 True 时走原 SessionAutoCommitScheduler 构造/start 分支。 -# enabled 的 policy 默认值逻辑继续传给 SessionService，不因为 Redis backend 被关闭。 -``` - -## 16. expiry_bootstrap.py：内部 publisher 子进程 - -```python -from __future__ import annotations - -import argparse -import asyncio -import json -import logging -import os -import signal -from pathlib import Path -from typing import Any - - -def parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """解析父进程传入的配置路径与 readiness 状态文件。""" - parser = argparse.ArgumentParser(description="Internal OpenViking expiry publisher") - parser.add_argument("--config", required=True, type=Path) - parser.add_argument("--startup-status", required=True, type=Path) - return parser.parse_args(argv) - - -def load_expiry_config(path: Path) -> "ExpiryConfig": - """读取完整 ov.conf，确认 Redis backend 和独立 publisher 均已启用。""" - from openviking_cli.utils.config.open_viking_config import OpenVikingConfig - raw = json.loads(os.path.expandvars(path.expanduser().read_text(encoding="utf-8-sig"))) - config = OpenVikingConfig.from_dict(raw) - ... - return config.expiry - - -def install_signal_handlers(stop: asyncio.Event) -> None: - """在主线程 loop 上把 SIGINT/SIGTERM 转为 stop.set，信号处理器不做 I/O。""" - ... - - -async def run_publisher(config: "ExpiryConfig", stop: asyncio.Event, - startup_status: Path) -> None: - """创建 runtime，启动 storage/roles，同时等待 stop 或角色故障。 - - 任一角色异常立即结束命令并返回失败；finally await runtime.close。 - storage/role 启动成功后写 ready；不能仅 await stop.wait 而让 publisher - task 异常后进程继续假存活。 - """ - ... - - -def main(argv: list[str] | None = None) -> int: - """解析/配置失败返回非零；正常信号退出走有界关闭；不打印密码值。""" - ... -``` - -该模块不注册公开 CLI。`openviking-server` 通过 `python -m` 拉起它，readiness 失败则拒绝启动 API；运行期异常退出会触发父进程退出。publisher 自动遍历全部 shard 逐个抢锁，只有成功取得 lease 才处理；不依赖 Pod ID 或静态分片列表。 - -## 17. pyproject.toml 与包资源 - -这里只列需要合并进现有 TOML table 的字段，不重复创建已有 table： - -```toml -[project.optional-dependencies] -# 新增到原 table；redis 版本是待验证的候选范围。 -expiry = ["redis>=5,<6"] - -[tool.setuptools.package-data] -# 保留原全部 package-data；增加独立 package 的 Lua 数据。 -"openviking.scheduling.expiry" = ["lua/*.lua"] -``` - -新增 Python 文件沿用仓库许可证头。`lua/` 仅保存数据文件，无需 Python `__init__.py`；`scheduling/expiry/__init__.py` 要存在。验证构建后的 wheel 包含全部 Lua；publisher 只作为 server 内部模块运行。 - -## 18. 启动与四条运行链路 - -```text -启动 -core汇总已启用Expiry业务 -> 无业务则跳过 - -> 创建runtime -> SessionService.configure_expiry(runtime.scheduler) - -> session及其他业务逐个注册handler/adapter - -> start_storage -> namespace实际配置/C初始0 - -> start_roles -> 冻结注册/开放report - -报告 -Session._append_messages_authoritatively - -> 锁内消息/meta保存，构造无revision快照 -> 释放锁 - -> SessionExpiryAdapter.report_after_append -> ExpiryScheduler.report - -> upsert_resource.lua(R)：TIME + 未来due + ZADD - -> ensure_metadata.lua(M)：身份 + max(2*timeout,24小时) TTL - -> 两步成功返回；R成功M失败报错但保留R - -逐shard调度 -ExpiryPublisher.run：0..N-1递增遍历，本轮后等待下一分钟边界 - -> ShardRunner.run_if_acquired(SCHEDULER,shard_id,process,stop) - -> Coordinator.try_acquire取得120秒lease，runner启动每20秒heartbeat - -> ExpiryPublisher.process_shard：读C/源TIME，U截断至分钟 - -> U=C/无数据跳过；有数据定位下一非空分钟M - -> read_source_page(R) -> stage_items(B)，读完整分钟 - -> publish_minute.lua(G,C,owner)，C=M -> cleanup_source(R) - -> 持锁追平全部到期积压 -> revoke/停heartbeat/有界join -> 可信终止时token释放 -> 更大shard - -> 续租失败：lost/revoke/cancel process与children，有界join；pending阻止同role再抢 - -> 旧tasks真实退出后，下次新token/新execution从C重新开始 - -回调 -ExpiryConsumer.run：全部shard递增遍历，轮后callback_poll_seconds - -> ShardRunner.run_if_acquired(CALLBACK,shard_id,process,stop) -> process_shard - -> reset state/retry缓存（旧tasks必须已退出），ShardExecution.wait执行poll等待 - -> 分页读G -> claim.lua(B,max_attempts=3) - -> claims：execution.spawn工厂创建runner跟踪子task，解析完整身份、读M（可缺失） -> SessionTimeoutHandler - -> SessionService.commit_on_timeout -> get(auto_create=False)加载最新session/meta - -> _check_timeout_commit：外层policy/timeout/no_messages/cooldown/last_message_due纯检查 - -> 检查通过：直接原Session.commit_async(keep_recent_count=0, - persist_keep_recent_count=False, record_auto_commit_success=True) - -> 不做持锁最终idle检查，期间新消息允许一起提交 - -> 原API返回accepted -> _record_idle_commit_time：服务层锁内重读最新meta - -> 仅patch接受时间(max前后)/updated_at；失败日志仍accepted，删除不重建 - -> handler返回后ensure_active先检查current_task.cancelling；吞取消也同步unknown/CancelledError，不ACK - -> 正常返回原accepted且检查通过 -> ack.lua(B) -> 空B摘G；不删除未来R/M - -> 未耗尽失败：留B；最终失败/claim.exhausted：日志 -> 条件ACK - -> 当前shard积压/在途清空后释放；仅旧tasks全部退出后清共享state，重试等待期间继续续租 - -关闭（四条业务链路之外的生命周期） -core保持runtime强引用 -> runtime.stop_roles拒绝新调用/set stop - -> runner grace期间heartbeat继续，仍监视lost/截止；失锁立即revoke/cancel - -> grace超时cancel，有界join role tasks/runner pending - -> 未退出：shard_not_stopped向外传播，保留runtime/tasks/pending，不关闭QueueFS/AGFS/store/I/O - -> role/pending退出 -> io.quiesce拒绝外部请求，等真实_RedisRequest.finished - -> quiesce超时：保留requests和依赖，shard_not_stopped；run多次取消仍桥接至实际退出 - -> 未关闭store：run_cleanup(store.close,timeout)提交Redis loop raw parent，nested io.run关client - -> cleanup成功置_storage_closed=True -> io.close再次quiesce -> 一次loop.stop -> 剩余budget join - -> 线程alive：保留loop/thread及_storage_started/storage_ready，重试跳过成功cleanup、不重复stop - -> 线程真实退出：清I/O引用和storage就绪标志；core成功后才清runtime/adapter并关业务依赖 - -冷却/活动重投 -SessionService外层纯检查：冷却未满足或最新last_message_due未到 - -> deferred：冷却due=检查now+timeout，活动due=last_message_at+timeout - -> SessionTimeoutHandler._report_deferred，不进入Session提交内部 - -> 统一report，R成功再M成功 -> COMPLETED -> ACK旧B，不删除未来R/M - -> report失败：RETRY，保留旧B；skipped/deferred不更新时间 -``` - -## 19. 检验范围与证据边界 - -静态检查覆盖 Markdown 围栏、Python 骨架语法、runtime/shard_runner 相关签名与两文档控制流一致性；省略体和模拟任务不能作为真实 Redis 行为证据。 - -生命周期回归须区分以下边界： - -- stop grace 期间仍监视失锁/截止；迟到 renew 不复活，普通可信业务异常可释放，forced/失锁/unknown 留 TTL。 -- handler 吞取消返回 COMPLETED 时，ensure_active 同步检查 cancelling、置 unknown、抛 CancelledError，阻止 ACK，不等待 done callback。 -- Redis operation 抑制取消及调用方多次取消时，wrapper/requests 保留到真实 finished；quiesce 拒绝外部新请求，超时保留依赖。 -- run_cleanup 的 raw parent 不先占 gate，nested io.run 在 quiesce 后可关 client；timeout/cancel 不冒充 cleanup 已退出。 -- io.close quiesce 后只发一次 stop，以剩余预算 join；线程 alive 保留引用、禁止 start 第二线程。 -- `test_runtime_retries_thread_join_without_resubmitting_closed_storage_cleanup` 覆盖 cleanup 成功后 join 超时：_storage_closed=True，_storage_started/storage_ready 保留，重试不重复提交 store.close，只重新观察 join。 -- runner pending 在 cleanup 再次取消时不丢；同 role 全 shard 禁重领，旧 callback finally 不污染新 state；core 保留 runtime/adapter 并传播未退出异常，阻止业务依赖提前关闭。 - -自动化完整回归结果：**235 passed，8 条既有 deprecation warnings，40.68s**（2026-10-11，日志 `/tmp/OpenViking_shard_refactor_final3.log`）。该结果包含线程 join 重试回归，不等同于真实 Redis/Lua、Cluster 故障注入或吞吐验证。 - -真实 Redis/Lua、Cluster 接管故障和吞吐测试未执行。Lua 共12个文件，R→M、源 TIME、空范围、未决跨槽请求、仅日志终结等协议限制保持不变；本地取消和真实 task done 仍不证明服务端写入已撤回，不构成跨槽 fencing。 - -Session 外层纯检查、原 commit 参数、best-effort 两字段 patch、deferred report 成功才 ACK 及 WM/QueueFS 兼容验收继续保留。检查与 commit 不原子，接受时间可能记录失败，不承诺严格 idle 边界、零重复或每次成功可恢复；原通用 ready/QueueFS 恢复不用于恢复 idle 记录。其他 Redis 可靠性限制以主文档为准。 diff --git a/docs/zh/design/session-idle-auto-commit-scheduler-vibecoding.md b/docs/zh/design/session-idle-auto-commit-scheduler-vibecoding.md deleted file mode 100644 index b0972316a6..0000000000 --- a/docs/zh/design/session-idle-auto-commit-scheduler-vibecoding.md +++ /dev/null @@ -1,453 +0,0 @@ -# Session 空闲超时自动提交调度器 VibeCoding 实施文档 - -- 状态：**采用临时 shard 租约、真实 Redis I/O 生命周期跟踪和有界关闭观察；可靠性与检验边界见第 10、13、14 节。** -- 来源：[内核超时自动触发技术方案](https://bytedance.my.larkoffice.com/docx/KUbYdV4SJoxxsyxhgz1mW1wMypb)，最近读取 revision 3059；与文档不同之处以本轮用户明确修正为准。 -- 代码基线：OpenViking `7d77e3d68fea82070a0b700a9acd64c40e7f4a9a`；本轮对比原基线 `6e1030d9951c4c19048e4758ef130be5da4abfe0` 后的 7 个提交。 -- 更新时间：2026-10-11。 -- 附录：[逐文件代码骨架](session-idle-auto-commit-scheduler-code-design.md)。主稿说明行为，附录列出字段、函数签名及 Lua 契约；省略体只省去实现细节，不作为行为验收证据。 - -## 1. 当前结论与本轮变更 - -采用业务统一接入、Redis 资源桶、调度桶、注册表及游标的结构，首期支持 session 的 IDLE_ONESHOT。调度和回调实例均通过 Redis 临时领取 shard，持锁处理全部积压后释放，轮内按 shard_id 递增。 - -本轮明确移除资源伴随状态 V、资源更新 revision、Redis 失败记录及元数据格式版本字段。报告按 R → M 执行；调度按读 R → 写 B → 原子写 G/C 执行。仅 report 写入/更新 R 的 score，调度只按已发布 C 删除源成员，不把调度读取与报告更新混为同一条调用链。 - -**有两个保证不能由这些步骤直接推出：** R 写完、M 尚未写完时退出会留下缺失元数据；续租失败后取消本地任务不能撤销已发出的跨槽 Redis 请求。前者由完整 item_id 回源兜底；后者仍是当前共享调度桶在自动接管时的残余风险，见第 10.3 节。本稿不把这两点写成已证明安全，也不擅自加入新的桶隔离标识。 - -### 1.1 最新内核适配与本轮放宽保证 - -Redis 是可选能力，默认 `idle_backend=legacy` 与既有提交能力保持不变。本轮覆盖旧的锁内 idle 检查及 idle 归档恢复方案：`SessionService.commit_on_timeout` 加载最新 session/meta，在服务层完成纯检查；通过后直接调用原 `Session.commit_async`。Session 的通用提交内部不增加 Redis 或 idle 判断，也不为 timer 拆分提交实现。 - -检查与提交不在同一把业务锁内原子执行。检查通过后、原提交读取消息前追加的新消息允许一起提交；policy 或 timeout 在此窗口变化也不触发一次新的最终 idle 检查。`last_idle_commit_accepted_at` 仅为外层 best-effort 接受记录，写失败不改变 accepted，不提供归档查证或修复缓存。这一方案不承诺严格空闲边界、零重复提交或每次 accepted 都能持久记录。 - -| 最新代码契约 | 本方案保留方式 | -|---|---| -| Session.commit/commit_async、SessionService.commit/commit_async 的 enable_working_memory | 原签名与三态透传保持不变，False 不能当作未传；不新增 idle 专用 commit API | -| MemoryPolicy.enable_working_memory 默认 False | idle 调用不覆盖 WM，继承当前 session/user policy；保留显式启用策略 | -| accepted 和原有 skipped 结果包含 effective_enable_working_memory | 直接返回原 commit 结果，不重建丢字段的精简结果 | -| QueueFS 的 memory_policy_version 及完整 policy 快照 | 沿用原序列化、兼容读取和恢复，不把它当成 Redis M.schema_version | -| .done 的 enable_working_memory 与旧字段兼容；关闭 WM 可无 summary | 原 ArchiveStore 能力保持不变；timer 只以原 commit API 返回 accepted 为成功，不等待后台任务或摘要 | - -原通用提交已有的 ready、QueueFS、恢复及 Working Memory 契约均继续保留，不成为 idle 接受时间的存储或恢复机制。 - -## 2. 已确认约束 - -1. 业务只使用 report、unregister、register_handler 三个方法，通过依赖注入取得协议；调度、Redis、消费、生命周期均封装在内部。 -2. 各业务自行注册处理器，通用模块不硬编码 session 类型、policy 或并发映射。 -3. 首期 IDLE_ONESHOT；FIXED 仅预留，调用时明确不支持。 -4. 消息成功追加后直接 await report，无内存去抖；按先 R 后 M 的顺序写 Redis。 -5. 不使用 V、源关闭水位或更新顺序号；接受同 item_id 近间隔并发写入时旧请求覆盖新请求的误差。 -6. 只处理实际到期数据：U 为当前时间去掉秒和毫秒，扫描 `(C,U]`；U=C 或没有数据就跳过。 -7. 不使用 group_id。分钟调度桶全部写完后，G/C 同槽 Lua 注册并推进游标；已发布桶不再生产。 -8. 元数据 TTL 为 `max(2×timeout,24小时)`；完整 item_id 承载 type/account/user/local_id，M 缺失时仍可回源。 -9. 已投递任务由服务层检查加载时的最新状态；未到最后消息到期时间则 deferred。检查通过后不再做持锁最终 idle 校验，期间追加的新消息允许一起提交；新活动的未来计时独立保留。 -10. idle 冷却统一使用 `last_idle_commit_accepted_at`，它是服务层 best-effort 接受记录，不使用通用自动提交统计时间作判断，也不建立 idle 归档指针或修复缓存。 -11. timeout 修改不批量重写旧 R；回调冷却校验不通过时 report 新计时，due 为本次检查时间加当前 timeout；report 成功后完成旧事件。 -12. 同一调度事件最多三次调用机会，包含首次；耗尽只写日志并条件清除 B，不在 Redis 保存失败记录。 -13. 每个实例每个角色一次领取一个 shard，处理全部积压后释放，再尝试更大的 shard；一轮结束从头开始。 -14. 锁 TTL 120 秒，每 20 秒续租；续租失败立即停止本次处理，不续跑旧任务，后续重新领取必须使用新 token。 -15. 不处理同名资源删除重建；不增加 incarnation、持久待注册集合、outbox 或零漏触发承诺。旧扫描器仅参考业务处理步骤。 - -## 3. 业务契约与内部边界 - -```python -from typing import Protocol -from openviking.scheduling.expiry import ( - HandlerOptions, ItemIdentity, ExpiryHandler, ExpiryRegistration, ExpiryRegistrationResult, -) - - -class ExpiryScheduler(Protocol): - async def report(self, task: ExpiryRegistration) -> ExpiryRegistrationResult: - """先写入资源计时，再创建或刷新元数据；等待本次结果。""" - ... - - async def unregister(self, identity: ItemIdentity) -> None: - """幂等注销当前计时及元数据；已投递事件按业务当前状态结束。""" - ... - - def register_handler(self, item_type: str, handler: ExpiryHandler, *, - options: HandlerOptions | None = None) -> None: - """业务启动时注册自己的处理器及处理能力。""" - ... -``` - -| 类型 | 字段及约定 | -|---|---| -| ItemIdentity | item_type、account_id、user_id、local_id | -| ExpiryRegistration | identity、due_at_ms、timeout_seconds、mode=IDLE_ONESHOT；无 revision | -| ExpiryRegistrationResult | accepted_due_at_ms；不再返回需要版本比较才能判断的 DUPLICATE/STALE | -| HandlerOptions | concurrency 默认 4、max_attempts 默认 3（含首次），同类型各副本保持一致 | -| CallbackRequest | identity；框架不透出 claim、缓存命中、游标或 Redis 状态 | -| CallbackResult | COMPLETED / RETRY 与原因码 | -| ExpiryHandler | async handle(request) -> CallbackResult | -| ExpiryError | 业务错误码、错误信息、retryable | - -`api.py` 定义公开契约，包入口只重导出上述协议/类型。业务不构造 store、keys、runtime，不调用启动、关闭、冻结或 ACK。SessionService.configure_expiry 委托 session_expiry.install_session_expiry，业务模块自行安装上报适配器并注册 handler；core 只组装、注入和启停。 - -`api.py` 中的 `SUPPORTED_ITEM_TYPES` 是类型信息的唯一来源。内部 HandlerRegistry 只允许注册清单内类型，并在启动消费前冻结；report 在写 Redis 前按同一清单校验。独立调度器直接使用代码清单定位未发布桶，不访问 Redis 类型目录；处理器和选项只在本地。只有 G 注册后的桶可消费。 - -同一 namespace 的回调副本加载相同的业务处理器集合；缺 handler 的类型不领取，限频报警，不能假设其他实例会自动接管该类型。session 只有选择 Redis idle backend 时才安装本业务接入，其他业务启用 timer 不应顺带启动 session 的 Redis 计时。 - -完整 item_id 使用 `item_type:encoded_account:encoded_user:encoded_local_id`，身份组件 UTF-8 后用无填充 URL-safe Base64，严格解码并重编码验证，禁止花括号或冒号改变 key 结构。编码规则和 xxhash64 seed 固定，不增加逐资源格式版本字段。 - -## 4. Redis 数据结构与 report - -`<...>` 是占位符，只有明确列出的 `{...}` 才是 Redis hash tag；service_key 为编码后的共享 service_name。 - -| 名称 | Key 示意 | 内容 | -|---|---|---| -| R 发布源 | `expiry:publisher:source::` | ZSET，member=完整 item_id，score=due_at_ms | -| M 元数据 | `expiry:metadata::` | HASH，item_type、local_id、account_id、user_id；带 TTL | -| B 消费桶 | `expiry:consumer:bucket::::` | ZSET，初始 score=0；minute 为窗口结束边界 | -| G 注册表 | `expiry:consumer:registry:{:}` | ZSET，member=type:shard:minute，score=minute | -| C 发布进度 | `expiry:publisher:progress:{:}` | 最后已发布的包含式分钟上界，UTC 毫秒 | -| owner | `expiry:publisher:owner:{:}` 或 `expiry:consumer:owner::{:}` | 当前领取 token，TTL=120秒；publisher/consumer 分开 | -| namespace | `expiry:namespace:` | 初始化状态、固定 N/hash/编码配置及内部类型名称目录 | - -仅 G/C/publisher owner 需要同槽发布；R、M、B 各自的单 key 操作不必同槽。没有 V 或 Redis 失败存储 key。 - -### 4.1 元数据与格式字段 - -M 只保存身份字段。原设计的 `schema_version` 用于区分序列化格式；当前没有多版本解析或迁移流程，字段本身不能完成兼容处理，**本期删除**。配置仍比较实际的 shard_count、hash_name、hash_seed、identity_encoding，防止不同实例使用不同寻址规则；这属于命名空间配置校验，不是给每条资源增加版本。 - -TTL 由框架内部统一计算，业务不另传 TTL： - -```text -metadata_ttl_ms = max(timeout_seconds × 2000, 86_400_000) -``` - -创建、续期使用同一公式，从 Redis 执行时开始计时。timeout≤12小时取24小时，超过12小时取两倍 timeout。ensure_metadata.lua 在单 key 内检查身份、必要时 HSET、最后 PEXPIRE，避免 EXISTS/EXPIRE 的临界过期竞态。 - -### 4.2 report 的真实顺序和错误语义 - -1. 校验身份、模式、正 timeout 和整数范围，确认任务类型已发布。 -2. 调用只访问 R 的 upsert_resource.lua，用源节点 TIME 校正为未来 due，然后 ZADD 覆盖 score。 -3. R 明确成功后，调用 ensure_metadata.lua 写 M 或续 TTL。 -4. 两步均成功才返回 accepted_due_at_ms。R 失败不继续写 M；R 成功、M 失败则接口报错，但不回滚已写 R，避免覆盖或删除并发的新报告。 - -先 R 后 M **不能保证 R 中每个成员都有 M**：两步间退出、M 写失败或后来过期都会造成缺失。完整 item_id 回源是必要正常路径。M 的 Redis 网络错误与明确缺失仍区分，前者按可重试错误处理。 - -本期接受上报乱序覆盖，不做 CAS、不保留更新 revision、不识别重复请求。报告重试可能覆盖较新 due，或在原任务已处理后建立新计时；不声称资源级无限幂等。删除 R/M 可重复执行，但并发删除与新上报的生命周期顺序由业务负责，同名重建不在范围内。 - -### 4.3 report 与调度并行时的时间边界 - -report 不读取 C；调度读取 C 时也不通过该调用更新 R。不同请求仍可以同时执行 report 和调度，不能因为代码属于两个阶段就认为全局互斥。 - -保留一个无需伴随状态的单 R Lua 约束： - -```text -accepted_due = max(requested_due, source_redis_now_ms + 1) -ZADD R accepted_due item_id -``` - -TIME 与 ZADD 在同一脚本中执行；调度的 U 也取该 R 所在 Redis 节点的 TIME。节点时间正常前进时，调度取得 U 后到达的 report 会写到 U 之后；之前写入的未来项随时间到期后可扫描。此设计不比较业务请求先后，仅确保写入时 due 在未来。 - -调度取得 U 的脚本只读时间、不修改 R。若改用另一个节点时间或在客户端算完时间后长时间滞留再 ZADD，这个保证不成立。未保留历史关闭水位时，也不承诺任意时钟回退下无遗漏；UC 时，按 score/member 分页查找下一条到期成员，确定下一个有数据的分钟 M。没有成员即结束本轮范围。 -3. 使用 `SUPPORTED_ITEM_TYPES`，确定 C 未覆盖 M 后清理该分钟未注册的临时 B，按当前 R 重建；不恢复旧内存分组。 -4. 分页读该分钟所有成员，按 type 批量 ZADD 0 到 B。每页有限，但不因页数/分钟数达到某个配额而交出 shard。 -5. M 全部读完且 B 写入结果明确后，单条 Lua 检查 owner token 和预期 C，注册实际有数据的 B 并设 C=M；随后按 C 清理 R。 -6. 继续后续有数据分钟。扫描追平本轮 U 后重新检查当前上界和新增到期积压；确认无可处理数据，结束并释放锁，继续更大的 shard。 - -扫描期间成员可能续期或被删除。若选定分钟在真正分页时已经变空，只跳过该分钟，不发布；本地查询下界向前移动以避免空转，持久 C 保持不变。若已读成员随后活跃，则本次事件仍可投递，新 due 留在 R。 - -G/C Lua 允许跨过已经扫描确认为空的分钟，不再要求 M=C+60000，只要求 M>C 且已完成中间扫描。分钟结束前不推进 C。读取 U、读 R、写 B 都不写入或更新 R 的 score；源删除是发布后单独的清理步骤。 - -### 6.3 分页与恢复 - -源页使用 `(last_score,last_item_id)`，不使用递增 OFFSET。单 key Lua 根据游标当前 rank 定位；末项续期或被删时可按 score/member 二分定位，约 O(log²N+page_size)，不全量扫描同 score 集合。 - -重领后丢弃旧内存状态，重新读 C。C 已覆盖的分钟不清桶、不重投，G 已被消费摘除也不能重新发布。确定未注册的分钟按当前 R 重做。空范围不推进 C，曾写入但未注册、成员后来全部续期的 B 可能成为临时遗留 key；不注册它，不给已注册 B 设置短 TTL，必要时离线核对清理。 - -发布响应未知时，租约仍有效可读 C 查证：C 已推进则确认；C 未推进不证明旧请求永远不会执行，保留已写完 B 并重试同一个发布，不能回到 reset/stage。租约失败立即取消本次处理，不在重新领取后接着执行旧的挂起函数。已发出跨槽命令的残余影响见第 10.3 节。 - -源清理使用 ZREMRANGEBYSCORE R -inf C，保留正常未来 due。复杂度 O(logN+K)，大积压和集中到期需测单命令耗时；若过大再改为单 key 有界批删，不把分页投递等同于删除已无大 key 风险。 - -## 7. Lua 与故障处理 - -| 脚本 | 原子域 | 职责 | -|---|---|---| -| upsert_resource | R | TIME + 未来 due + ZADD，不读 C、不写伴随状态 | -| source_time | R 所在 slot 的只读脚本 | 返回源节点 TIME，不修改 R | -| ensure_metadata | M | 校验身份、创建/刷新和 TTL | -| read_source_page | R | score/member 有界分页 | -| publish_minute | G/C/scheduler owner | 比较 token、预期 C，注册非空分钟并推进 | -| read_registry_page | G | 已发布桶有界发现 | -| claim | B | 原子领取并计数，最后一次超时只返回待终结项，不再领取 | -| ack | B | 按 claim_score 条件删除，成功和耗尽终结共用 | -| acquire/renew/release_owner | 对应 owner | NX 领取、比较 token 续期、比较 token 释放 | - -脚本写前检查参数、key 类型；Lua 运行异常不提供跨命令回滚。删除 R 使用单 key ZREM，删除 M 使用 DEL，不再需要注销伴随状态脚本。 - -| 故障 | 处理 | -|---|---| -| R 成功、M 失败 | report 报错，保留 R；完整身份仍可回源 | -| M 缺失 | 解码完整身份，正常调用业务 | -| 写部分 B 后取消/退出 | 未发布不消费；重新领取后读 C 和当前 R | -| 发布后响应丢失 | 查 C 或在同一有效租约下重试原发布，不重写已发布 B | -| 清理失败 | 以后只补清理，不重投已发布分钟 | -| 续租失败 | 本次租约立即失效，取消处理；旧 token 不复活 | -| callback 未耗尽且失败 | 保留 B，超时后重领 | -| 第三次失败/超时 | 写耗尽日志，然后条件 ACK；无 Redis 失败记录 | -| report 重新计时失败 | 旧 B 不 ACK，按剩余机会重试 | - -## 8. 回调、三次机会与仅日志终结 - -回调实例持有 shard 后，分页发现 G，按 type 的进程级并发处理全部可执行积压。发现循环需继续遍历，不能被最老页或单个慢类型阻塞。重试等待期间仍续租；已注册桶清空、在途任务收尾后释放 shard。未知 handler 的桶保留并报警，不无限持锁等待不存在的处理器。 - -```text -score = attempt × 2**40 + started_at_seconds -initial_score = 0 -session attempt = 0,1,2 → 最多三次机会 -``` - -claim 在 B 内原子占用一次机会，每个 claim 至多调用一次 handler。进程可能领取后、调用前退出，因此实际调用可能少于三次；不退还结果不明的机会。新事件包括冷却重投产生的事件有独立预算。各类型上限由业务 HandlerOptions 注册，session 固定采用3。 - -按 attempt 层有界查询超时项；新项优先但为重试保留预算。第三次超时的成员不生成第四次 claim，而是返回 exhausted 列表，由 consumer 写日志后条件 ACK。脚本不直接删除这些项，避免正常路径中先删后无法输出日志。 - -明确 RETRY/异常且最后机会耗尽时，consumer 同样先写结构化日志，再条件 ACK。日志包含资源身份、桶分钟、claim_score、机会数、失败原因及是否结果未知，不记录消息正文。Redis 不保存失败 HASH、索引、死信或重放记录。ACK 失败时留 B 以后重复终结，日志可能重复；日志输出与 Redis 删除不构成事务，不能承诺故障下日志必达或恰好一次。 - -COMPLETED 后 ACK 出错，只重试同一条件 ACK，不再调用 handler、不算一次新的业务失败。最后一次 accepted 后 ACK 是否生效无法查清时，最多记录结果未知并清理，不额外调用业务来突破三次上限。条件 ACK 不匹配则不删除新 claim。B 空后单独摘 G，不删新活动 R/M。 - -回调处理超时与 shard 租约是独立参数；前者决定任务何时可重领，后者每20秒续租，不以任务耗时长为由主动释放。 - -## 9. Session idle 时间与冷却重投 - -### 9.1 服务层 best-effort 接受记录 - -SessionMeta 仅增加 `last_idle_commit_accepted_at`，缺字段按空串处理；现有普通统计字段保持兼容但不参与 idle 决策。此字段不是原通用提交 ready 的原子记录，不存储 idle archive 指针，不提供 idle readiness 查证或恢复。 - -原 `Session.commit_async` 返回 `status=accepted` 即为本次 timer 业务成功，不等待后台任务。之后服务层 await `_record_idle_commit_time(session_id, ctx, accepted_at)`：复用既有 session root exact 路径锁，锁内直接重读最新 raw meta 字典，仅 patch `last_idle_commit_accepted_at` 和 `updated_at`，写入传 `lease_ref=lease`，不以检查时的旧 Session 对象全量保存。接受时间只以有效新旧值的 `max` 合并：已有有效更大/相等时间时不写，坏旧值记录日志后可覆盖，避免较早并发调用迟到回写旧成功时间。 - -记录更新失败只写日志，仍原样返回 accepted；若 session 已删除，跳过记录，不创建目录或重建 meta。该 helper 的读取/锁/写异常属于 best-effort 记录失败；入口加载、纯检查和原 commit 的异常则向外传播，不能伪装成无消息或 accepted。无消息等 skipped 和 deferred 不更新时间。 - -### 9.2 timeout 变更、外层检查与重新报告 - -timeout 更新不批量改已有 score。`SessionService.commit_on_timeout` 加载最新 session/meta 后，以本次 `now` 调用纯检查 `_check_timeout_commit(session: Session, now: datetime) -> Optional[dict]`，顺序如下： - -1. 明确 session 已删除：skipped/not_found，handler 转 COMPLETED；其他加载/读取异常向外传播。 -2. policy 禁用或 timeout≤0：skipped/idle_disabled，不重新计时。 -3. 没有可提交消息：skipped/no_messages，不更新时间，避免无内容 session 反复重投。 -4. 有消息且 `now-last_idle_commit_accepted_at < current_timeout`：返回 deferred/idle_cooldown，due 为本次检查 `now + 当前 timeout`。 -5. 冷却通过但 `now < last_message_at + current_timeout`：返回 deferred/idle_activity，due 为 `last_message_at + 当前 timeout`。 -6. deferred 仅由 SessionService 和 SessionTimeoutHandler 掌握，不修改 meta 或调用 commit。handler 统一 report；R→M 两步成功后才返回 COMPLETED、ACK 旧 B，失败传播以保留旧 B。旧 B 的 ACK 不删除未来 R/M。 -7. 检查返回 None：直接 `await session.commit_async(keep_recent_count=0, persist_keep_recent_count=False, record_auto_commit_success=True)`，不新增 idle 专用提交入口，不在提交锁内进行最终 idle 判断。原结果为 accepted 才执行 best-effort 接受时间记录，然后返回原结果；commit 异常传播。 - -例如最近 idle accepted 为12:00，旧事件12:10到期，而 timeout 已调大为30分钟。12:10回调因冷却未满足，report 新 due=12:40；不是回填12:30，也不复用旧 timeout。这是“当前时间+当前 timeout”语义。最后消息尚未到期的活动延后则使用最后消息时间加当前 timeout。 - -服务层复用 `parse_iso_datetime` 和 `get_current_timestamp` 处理时间，纯检查只读加载状态，不获取提交锁、不写 Redis 或 meta。重投不是成功提交，不更新接受时间；重复重投可能顺延 R，仍接受无版本顺序保护的覆盖行为。首次无接受时间不构造虚假冷却窗口。 - -检查完成后的追加、policy/timeout 变化不触发最终复检，允许新消息一起提交；检查与 commit 之间的并发以及接受时间记录失败都可能弱化冷却、导致重复调用。这是本轮明确放宽的保证，不要求从原 ready 反查或补写该字段。 - -### 9.3 与 Working Memory 策略的关系 - -idle timeout 的外层触发检查与 Working Memory 策略独立。直接调用原 `commit_async` 时不传一次性 WM 覆盖，沿用默认 `enable_working_memory=None`，继承 session/user 设置，未配置时采用默认 False。原始归档、QueueFS 及符合 policy 的长期记忆提取继续存在。 - -原手工提交的 `enable_working_memory=True/False` 只覆盖本次有效 policy，不写回用户或 SessionMeta 默认 policy；原队列快照、`memory_policy_version` 及恢复兼容契约不变。本方案移除的仅是 Redis 元数据中的 schema_version，不删除原通用提交能力。 - -WM 关闭时没有 `.overview.md` 是正常结果，不据此重试或推迟 timer ACK。后台任务继续使用既有失败/恢复流程，不回流为 timer 的三次回调重试；accepted 后 timer 只处理自身 ACK。 - -## 10. 实例抢占、续租与退出 - -### 10.1 一次一个 shard，轮内递增 - -每次抢占生成独立随机 owner token，不依赖进程身份或 Kubernetes。每个角色独立遍历 `0..N-1`，一次最多持有一个 shard。抢不到继续尝试更大的编号；抢到则处理全部积压，完成才释放；遍历和轮间节奏由worker自己控制：scheduler本轮后等下一分钟边界，callback本轮后等callback_poll_seconds，再从0开始。示例编号1..256与代码0..255仅是编号约定差异。 - -不使用成员名单、Rendezvous Hash、固定 shard 分配或每批强制移交。分页和 pipeline 有界只是控制内存与单命令工作量，不是释放锁的依据。热点 shard 若持续产生积压，可能长期由同一实例处理；其余实例继续争抢其他 shard，不承诺严格轮转公平或按资源量均衡。 - -### 10.2 token 操作、单 shard 执行与 worker 节奏 - -职责只拆到必要边界，不引入通用 worker 框架： - -| 对象/文件 | 职责 | -|---|---| -| ShardLeaseCoordinator / leases.py | 仅 try_acquire、renew、release，调用既有 Redis token 操作；不遍历 shard、不创建业务/heartbeat任务、不决定轮询节奏 | -| ShardLease / models.py | owner、保守 monotonic 截止、lost 与 revoked 两种单向状态；ensure_active 检查两者 | -| ShardExecution / shard_runner.py | owner、shard_id、ensure_active、spawn 工厂、wait；强引用跟踪 processor 子任务和辅助 waiters，汇总异常/unknown | -| ShardRunner / shard_runner.py | run_if_acquired 一次执行一个 shard；后台 heartbeat、stop grace、失锁撤销取消、有界 join、同 role pending 保护 | -| publisher.py / consumer.py | 自己遍历所有 shard，分别控制 publisher 下一分钟边界、callback poll 节奏；只把单 shard process 交给 runner | -| runtime.py | 构造同一个 runner 并注入两个 worker，保留角色任务，协调有界停止和依赖关闭 | - -1. `run_if_acquired(role, shard_id, process, stop)` 在 stop 时不领取、busy 返回 False；成功取得新随机 token 后构造全新 execution/process。每 role 只允许一个 active，**同 role 任一 shard 尚有 pending 都禁止再抢锁**，抛 `shard_not_stopped`；另一角色独立。 -2. Coordinator 的 try_acquire 使用 SET NX PX 120000；renew 比较 token，runner 后台每20秒调用。请求异常、超时、取消、token 不同/缺失或响应晚于之前本地截止，均保守置 lost；迟到成功也不能复活 lease。截止为请求发起 monotonic+TTL，不按收到响应重新延长。 -3. lost 表示 ownership 已不可信；revoked 表示本地执行已撤销，正常清理也会设置，**不把正常撤销冒充失锁**。`ShardExecution.ensure_active` 首先检查 `asyncio.current_task().cancelling()`，若存在取消请求，**同步置 outcome_unknown 并抛 CancelledError**，再检查 lease 的 lost、revoked 和截止。即使 handler 吞掉取消并返回 COMPLETED，后续检查点也不能放行 ACK，不依赖较晚执行的 child done callback 才标记未知。spawn 先检查，再创建执行工厂的 task，业务 processor 不取得续租/释放接口。 -4. runner 监视 processor、heartbeat、lost 和有效截止。正常 stop 开始 shutdown grace，heartbeat 继续；**grace 期间仍监视失锁及有效截止**，失锁立即 revoke/cancel，不只等待慢 processor。grace 超时、外层取消或 processor 返回却仍有 children，均按 forced 收尾。 -5. 收尾先 revoke，再 cancel process、heartbeat、children/waiters；若 renew 正在请求中，取消前同步置 lost，不能被抑制取消后的成功响应复活。join 用 `asyncio.wait(timeout=request_budget_seconds)` 有界观察，不能用取消后无界 await/gather，或把 wait_for 超时当作任务已经终止。 -6. 未终止 task 在等待前放入 runner pending 强引用（含 acquire/release/heartbeat），done callback 读取异常且只摘除自己的集合；二次取消 cleanup 也不能孤儿化 task。预算结束仍未退出则报 `shard_not_stopped`，worker 必须向 runtime 传播而非继续下一 shard。旧 tasks 真实退出后才允许新 token/新 execution；不恢复旧函数挂起点。 -7. process/children 普通业务异常但任务全部终止、无 forced/unknown 且 ownership 仍可信，允许停止 heartbeat 后比较 token 释放，再传播异常；**不是所有业务异常都等于失锁**。forced、lost、未知结果或本地工作未退出均保守停止续租、留 TTL，不主动释放。release 本身也有界观察，超时/未知不能假装已交接。 - -consumer 每个 callback 用 execution.spawn 工厂创建并被 runner 跟踪；poll 用 execution.wait，不自建旧式 lost task 等待。正常 stop 可等已启动 callback，期限/失锁由 runner 在外层持续监视。仅旧 tasks 全部退出后才清共享队列、reserved 和 task state，避免旧 finally 释放槽污染新轮；retry 层缓存在每次单 shard 执行边界清理，不跨租约累积。 - -所有有界等待都以事件循环能继续调度为前提。async task 不能消除进程暂停、阻塞事件循环，也**不能强杀抑制取消的协程**；pending 强引用和禁止同 role 重领是本地生命周期防护，不证明 Redis 请求已撤销。 - -### 10.3 必须保留的可靠性限制 - -本轮按用户指定的“续租失败停止、下次重新抢锁”实现，不加入按 token 隔离的 B。该措施可以停止本地后续动作，**不能撤回 Redis 已接收但尚未完成的 ZADD/UNLINK，也不能让跨槽的检查与写入原子化**。 - -因此仍有窗口：旧写入已发出或进程暂停在检查之后，新实例因 TTL 到期接管并发布同一个 B，旧请求随后执行。G/C/owner Lua 会拒绝旧发布，但无法拒绝另一个 slot 上的旧 B 写入。加长 TTL、取消 coroutine、换新 token 均不构成完整防护。 - -上线前需要故障注入验证这个边界，并决定是否接受残余风险。如果要求在暂停、分区和请求延迟下仍严格防止旧写入影响新 owner，需要桶隔离或 B 所在槽的写入版本校验等进一步变更；本轮不替用户自动选择。回调侧的 claim/条件 ACK 只能保护消费记录，不能抵御生产者后来无条件 ZADD 0。 - -## 11. 配置、启动与文件清单 - -顶层 `expiry` 配置保留 `service_name` 并增加 `publisher_separate_process=true`，删除 `reporting_enabled`、`scheduler_enabled`、`callback_enabled` 三个可组合角色开关。默认 `session_auto_commit.idle_backend=legacy`；只有 `enabled=true && idle_backend=redis` 时才整体启用 report、publisher、consumer。`service_name` 是所有实例共享且稳定的文档服务名。`shard_count` 默认 128、范围为 1 到 512，身份编码/hash seed 统一；共享同一 namespace 的实例必须使用相同 N。Session consumer 并发默认 8，可通过 `session_consumer_concurrency` 配置为任意正整数。Redis 扩物理节点由 slot 迁移处理。 - -新增公共角色参数 `owner_ttl_seconds=120`、`owner_renew_interval_seconds=20`，删除静态 scheduler_shards、callback_shards、领取数量上限及处理若干分钟就移交的参数。保留 scan_batch_size、registry_page_size、类型并发、processing_timeout_seconds、callback_poll_seconds、shutdown_timeout_seconds。 - -API 顺序：配置检查 → Redis backend 未启用则跳过 Runtime → 创建 API Runtime → 注册 session handler/reporter → 统一启动 storage/namespace → 冻结处理器并启动 consumer；`publisher_separate_process=false` 时同一 Runtime 也启动 publisher。默认值为 `true`，此时 `openviking-server` 在启动 API 前拉起并监督唯一 publisher 子进程。关闭先停止 expiry 角色及在途业务调用，再关闭QueueFS/AGFS/Redis。runtime用asyncio.wait有界等待role tasks和runner pending，仅真实done才移除引用；未退出抛shard_not_stopped并保持相关依赖打开，不能在finally强关。core._stop_expiry保持runtime强引用，runtime.close成功后才清runtime并detach adapter；pending失败时runtime和adapter均保留，禁止新calls由runtime.accepting_calls=False负责；shard_not_stopped向外传播，中止后续QueueFS/AGFS关闭，允许后续再观察/重试关闭。其余core启动与optional backend不变。只启动storage不代表report已开放。API Runtime 内部固定允许report，以便 consumer 完成冷却重投。 - -runtime组装为 `store → ShardLeaseCoordinator(store, config, metrics) → ShardRunner(leases, config, metrics)`，同一个runner注入 `ExpiryPublisher(store, runner, config, metrics)` 与 `ExpiryConsumer(store, runner, bindings, config, metrics)`。 - -RedisLoopExecutor 以 `_RedisRequest` 强引用登记实际 Redis-loop task；`finished` 仅在该 task 及其原始 operation 真正终止时完成。`run` 使用 shield 等待 finished，取消时只向远端发一次取消，再通过 `_join` 保留调用方 wrapper 至实际退出；多次取消也不能切断桥接。因此 runner 能观察未终止 I/O，而不是把跨线程 Future 的 cancelled 状态当作远端退出。`quiesce(timeout_seconds)` 拒绝新外部请求，以有界等待观察所有登记请求（含尚未开始的请求）的真实 finished；未退出抛 `shard_not_stopped`，保留请求和相关依赖。 - -`ExpiryRuntime.close` 依次执行 `stop_roles → io.quiesce → io.run_cleanup(store.close) → io.close`。`run_cleanup(operation, timeout_seconds)` 在 Redis loop 提交受跟踪的 raw parent，不先占用 gate；store.close 内部可嵌套 io.run 关闭 client，既允许 quiesce 后清理，也避免 max_inflight=1 时递归等待 gate。cleanup 超时/取消不等于退出，仍保留 request。io.close 再次 quiesce 后仅发一次 loop.stop，以本次 close 的剩余预算 join 线程；线程仍 alive 则保留 loop/thread/gate 引用并报未退出，后续 close 可重试、不重复 stop，start 不得创建第二线程。runtime 增加 `_storage_closed=False`，start_storage 成功重置；仅在其为 False 时 run_cleanup，成功后置 True。若随后线程 join 超时，仍保留 `_storage_started/storage_ready`，重试 close 跳过已成功的 store cleanup，只重新观察/等待 join，不向已 stop_requested 的 loop 再提交 cleanup。只有线程真实退出才清 I/O 引用及 `_storage_started/storage_ready`。 - -各关闭阶段分别使用配置预算，不能宣称整个 runtime.close 只有一个总预算；run 取消后的 join、原始 operation 收尾及线程内清理也不提供强杀保证。外层有界观察失败时保留实际工作引用与依赖，不把取消、超时或 join 返回误写成已关闭。 - -内部 `openviking_cli.expiry_bootstrap` 模块读取完整 `ov.conf` 后只运行 publisher，不初始化 AGFS、模型或 SessionService。它通过 readiness 文件与 server 父进程握手，自动遍历全部 N 竞争，不要求显式分片列表；启动失败阻止 API 启动，运行期异常退出会带动父进程退出。 - -| 文件 | 计划职责 | -|---|---| -| scheduling/expiry/api.py、__init__.py | 三个业务方法、DTO、处理器选项及受限导出 | -| scheduling/expiry/models.py | 内部桶/页/claim/租约、TTL与时间计算，无资源版本 | -| scheduling/expiry/config.py | 轻量 expiry 配置 | -| scheduling/expiry/keys.py | 完整身份、hash、R/M/B/G/C/owner寻址 | -| scheduling/expiry/metrics.py | 低基数指标和耗尽日志入口 | -| scheduling/expiry/registry.py、service.py | 本地处理器注册、私有facade、R→M调用链 | -| scheduling/expiry/redis_store.py、lua/ | 单key操作、G/C发布、抢锁续租、结果未知分类 | -| scheduling/expiry/leases.py | 仅try_acquire/renew/release，Redis token操作，不持有worker循环 | -| scheduling/expiry/shard_runner.py | ShardExecution检查点/spawn/wait；单shard heartbeat、grace/失锁撤销、有界join、pending强引用 | -| scheduling/expiry/publisher.py | 自遍历全部shard/分钟节奏，到期积压、空范围、发布和清理 | -| scheduling/expiry/consumer.py | 自遍历全部shard/poll节奏，runner跟踪子tasks、旧tasks退出后清state、三次机会及日志终结 | -| scheduling/expiry/runtime.py | 共享runner组装、I/O loop、有界启停，未退出工作保留引用/依赖 | -| service/session_expiry.py | 消息上报、业务handler及冷却重投 | -| session/session.py | 追加reporter、SessionMeta接受时间字段；原通用提交不加idle/Redis判断 | -| service/session_service.py | 最新状态加载、纯超时检查、直接原commit调用、best-effort时间patch | -| service/core.py | 注入、正确启停与backend选择 | -| openviking_cli/utils/config/*.py | timer配置集成及idle_backend兼容 | -| openviking_cli/expiry_bootstrap.py、server/bootstrap.py、pyproject.toml | 内部 publisher 子进程、父进程监督、Redis可选依赖及Lua包资源 | - -上述内核路径均以 openviking/ 为前缀。QueueFS/SessionCommitProcessor、原通用 ready/恢复、阈值提交和手工提交语义继续保留。 - -## 12. 实施顺序 - -| 阶段 | 交付与验证 | -|---|---| -| P1 | 公开契约、namespace实际配置、身份编码、store和单key Lua | -| P2 | R→M上报、TTL、注销、元数据缺失回源 | -| P3 | Coordinator三操作/ShardRunner单shard执行、heartbeat/grace失锁监视、lost/revoked不可复活、有界join/pending及可信异常释放 | -| P4 | 当前分钟扫描、非空分钟发布、分页、未知结果恢复 | -| P5 | runner跟踪回调子tasks/旧tasks退出后清state与每轮retry缓存；并发、三次机会、仅日志终结、旧ACK保护 | -| P6 | 服务层纯检查/直接原commit、锁内最新meta局部patch、记录失败仍accepted、冷却/活动重投 | -| P7 | CLI/多worker/legacy回归、Cluster故障注入和容量压测 | - -跨槽旧请求风险应在早期验证，不能到最后才用普通功能测试代替接管安全性检查。 - -## 13. 验收矩阵 - -| 场景 | 预期 | -|---|---| -| 同local_id跨account/user | 身份与回源隔离，绝不提升为root | -| R失败 / R成功M失败 / M过期 | 顺序明确；缺M可回源；无错误回滚R | -| timeout为1小时/12小时/13小时 | TTL分别24/24/26小时 | -| 旧A覆盖新B | 接受覆盖，不出现隐藏CAS或revision状态 | -| report与调度并行、延迟report到达 | 单R TIME+ZADD写未来due；正常时钟下不落入本轮旧范围 | -| 12:34:04.234、边界due±1ms | U=12:34:00；包含等值，晚1ms下轮处理 | -| U=C / 无到期成员 / 空分钟跨度很大 | 跳过，无空桶和空推进，无逐空分钟循环 | -| 分页末项续期/删除、大量同score | 不因OFFSET位移漏扫 | -| 处理大量积压 | 保持租约直到追平，不按页/分钟配额交出shard | -| 两个角色、多实例、轮内递增 | 每角色一次一个shard，锁分开，下一轮补扫跳过项 | -| 20秒续租失败 / 迟到成功 / 长暂停 | lost永久生效，不复活旧lease；revoke/cancel旧process/children，保留未决命令限制 | -| 正常stop grace期间失锁 | heartbeat继续且失锁/有效截止仍被监视，立即撤销，不等grace耗尽 | -| 抑制取消的process/child/renew/acquire/release | asyncio.wait有界join，pending强引用不丢；同role所有shard停止再抢，另一role独立；不宣称强杀 | -| 普通业务异常但任务已终止 | 无forced/unknown、ownership可信可按token释放，错误仍传播；revoked不等于lost | -| forced / 失锁 / 未知结果 | 不主动release，停止续租留TTL；本地取消不证明Redis写已撤回 | -| callback旧task的finally迟到 | 未退出前不清state/reserved、不重领同role；退出后再清，retry缓存每次单shard运行清理 | -| handler吞取消后返回COMPLETED | ensure_active先检查current_task.cancelling，同步unknown并抛CancelledError，不继续ACK，也不主动release | -| run取消及多次取消、远端operation抑制取消 | wrapper保留至真实finished；requests强引用不丢，quiesce超时报shard_not_stopped，依赖仍保留 | -| quiesce后新外部请求 / nested cleanup | 外部run拒绝；run_cleanup raw parent在Redis loop运行，nested io.run可关client，max_inflight=1无gate递归 | -| runtime/core关闭仍有工作 | stop_roles/quiesce未退出保留runtime/roles/pending/requests；shard_not_stopped传播，不提前关依赖 | -| store已关但I/O线程join超时 | _storage_closed=True而_storage_started/storage_ready保留；重试跳过cleanup、不重复stop，只观察join；真实退出才清引用，start不能建第二线程 | -| store cleanup成功后thread重试回归 | test_runtime_retries_thread_join_without_resubmitting_closed_storage_cleanup验证不向stop_requested loop重复提交cleanup | -| worker角色节奏 | 各自遍历0..N-1；scheduler等下一分钟边界，callback按poll，Coordinator无轮次逻辑 | -| publish响应未知 | 不盲目重做可能已发布的B | -| claim attempt=0/1/2 | 不生成第四次调用；最后失败日志后条件清理 | -| 日志输出/ACK故障 | 可重复日志，无Redis失败归档；不伪称日志与删除原子 | -| last_idle_commit_accepted_at记录 | 原commit accepted后外层best-effort patch；无idle归档指针/查证/修复要求 | -| accepted后并发append/config更新 | root exact锁内重读最新raw meta，仅patch接受时间/updated_at，写入传lease_ref=lease | -| 并发成功记录时间乱序或坏旧值 | 有效更大/相等时间不写，较旧调用不回写；坏旧值日志后可覆盖 | -| accepted后记录读取/锁/写失败或session删除 | 失败日志后仍accepted；missing不重建，不从ready恢复记录 | -| 检查通过后新消息到达 | 原commit允许一起提交，无锁内最终idle检查；不承诺严格空闲或零重复 | -| 入口读取/commit异常 | 向外传播供重试，不伪装skipped或accepted | -| 最新last_message_at尚未到期 | 外层deferred/idle_activity，report成功才ACK旧B且保留未来R/M | -| timeout调大、冷却未满足 | report due=检查now+当前timeout，成功后才ACK旧B | -| 重投report失败 | 旧B保留，未耗尽可重试；R成功M失败的部分结果可解释 | -| 无消息/禁用policy | skipped后完成，不更新时间，不产生无限空重投 | -| 新版 commit 参数 None/True/False | HTTP/Service/Session 透传，False 不丢失；非 bool 拒绝；一次性覆盖不持久化 | -| 默认 WM 关闭 / 业务 policy 显式开启 | 沿用原commit，accepted后ACK、不等后台任务；按有效policy决定摘要，保留effective_enable_working_memory | -| 新旧 QueueFS policy 快照、重启前后配置不同 | 新任务保留显式 WM 值；旧任务按 from_dict 兼容规则恢复，不重新套用当前用户 policy | -| 新旧 .done 字段、无 summary 的成功归档 | 原ArchiveStore兼容契约不变；无摘要不影响timer accepted判定，不修改原通用恢复 | -| 多loop/独立CLI/legacy | Redis连接不跨loop误用，CLI不加载业务依赖，原提交路径回归 | - -实施时复用最新版已有回归：`tests/session/test_working_memory_default_off.py`、`tests/unit/session/test_memory_policy.py`、`tests/unit/session/test_session_commit_resume.py`、`tests/session/test_session_commit.py`、`tests/server/test_api_sessions.py`，并在新增 idle 路径中覆盖上述交互。测试执行结论以最终 suite 输出为准，文档矩阵只定义预期。 - -## 14. 性能、容量与证据 - -报告至少一次R操作和一次M操作，通常跨槽两次往返；无V，无每资源更新版本成本。schema字段删除不代表取消实际namespace配置校验。延长M到至少24小时后，按最近一天内上报的去重资源量估算容量，不能按几分钟活跃量估算。 - -调度扫描量与到期数据相关，但每轮逐shard抢锁、读C/时间、空扫和释放存在固定成本。20秒续租约为每个持锁角色每分钟3次；需要测大量逻辑内核/实例同时轮询时的总负载。轮内递增减少回头竞争，不保证严格公平。空范围不推进C后，C年龄不能单独视为积压，应同时看最老到期R成员、G年龄和实际处理延迟。 - -B/G随积压增长；无Redis失败归档容量，但日志吞吐、保留时间和查询能力需要现有日志设施承担。每类型并发限制回调/原commit调用，不限制已接受后的后台任务总积压。批量范围删除、单type大分钟桶、热点shard和持续积压都需要真实Redis Cluster压测。 - -默认关闭 WM 可省去未显式启用时的摘要和 checkpoint 相关工作；长期记忆提取仍可能执行。不能把“没有摘要”等同于“没有后台成本”，也不能以此提高 timer 并发而跳过 QueueFS 积压与模型调用容量评估。本轮未提供新调度器的性能数据。 - -证据分为源码/骨架静态核对、自动化回归、真实 Redis/Lua 集成、Cluster 故障注入和吞吐压测，不能相互替代。静态核对重点为取消检查点、真实任务 finished 桥接、quiesce/cleanup/join 顺序及重试状态。2026-10-11 完整自动化回归为 **235 passed、8条既有 deprecation warnings、40.68s**，包含 store cleanup 成功后的线程 join 重试边界；日志及检验范围见附录第19节。 - -真实 Redis/Lua、Cluster 接管故障和吞吐测试未执行；自动化回归须以最终 suite 输出为准。不宣称已经解决第10.3节的残余风险，也不以文档骨架或模拟任务测试代表真实服务端行为已通过验收。 - -## 15. 工程约束与上线检查 - -以下约束与可靠性限制需在上线检查中保持一致： - -- 上报通过单R Lua校正过去due到源TIME之后，不读取C，不引入关闭水位；任意时钟回退仍不在无状态保证内。 -- C初始为0、非空发布允许跳过空分钟；完整身份及namespace寻址配置需全实例一致。 -- 三次预算按领取占用，耗尽仅日志；log与ACK非事务，允许重复或故障下缺失日志。 -- 冷却重投只有R/M均成功后才完成旧事件；失败不等于已经写入的R被撤销。 -- shard租约按120/20执行；取消本地工作不能撤回已发跨槽请求，自动接管安全性仍需明确接受范围或进一步修改协议。 -- 切换前已静默session是否做一次性导入、具体性能预算和灰度范围仍在上线计划中确认，不恢复周期全量扫描。 - -## 16. 逐文件代码附录 - -[代码骨架](session-idle-auto-commit-scheduler-code-design.md)与本文同步：删除V/更新revision/Redis失败记录/格式版本字段，同步leases.py三操作、shard_runner.py单shard执行、worker自有节奏、consumer子tasks与state、runtime/core未退出不关依赖；保留R→M、空范围跳过和Session外层冷却重投。新增/修改方法列出签名、注释及关键控制流；已有文件未涉及部分保留，省略体只省去实现细节。 diff --git a/tests/scheduling/expiry/test_config.py b/tests/scheduling/expiry/test_expiry_config.py similarity index 100% rename from tests/scheduling/expiry/test_config.py rename to tests/scheduling/expiry/test_expiry_config.py From bf7d6725391c54c8046f1380715f08ed16550dd7 Mon Sep 17 00:00:00 2001 From: qin-ctx Date: Wed, 14 Oct 2026 17:34:14 +0800 Subject: [PATCH 4/5] test(session): consolidate expiry coverage into existing contracts --- tests/scheduling/expiry/test_consumer.py | 734 ------------------ .../expiry/test_expiry_bootstrap.py | 156 ---- tests/scheduling/expiry/test_expiry_config.py | 223 ------ .../expiry/test_facade_and_registry.py | 177 ----- tests/scheduling/expiry/test_leases.py | 542 ------------- .../scheduling/expiry/test_models_and_keys.py | 106 --- tests/scheduling/expiry/test_publisher.py | 306 -------- tests/scheduling/expiry/test_redis_loop.py | 382 --------- tests/scheduling/expiry/test_redis_store.py | 680 ---------------- tests/scheduling/expiry/test_runtime.py | 608 --------------- .../scheduling/expiry/test_session_expiry.py | 252 ------ .../expiry/test_session_timeout_service.py | 639 --------------- .../test_agent_evolution_global_setting.py | 23 +- tests/server/test_api_sessions.py | 114 ++- tests/server/test_bootstrap.py | 76 +- tests/session/test_session_commit.py | 159 ++-- tests/unit/session/test_auto_commit_policy.py | 20 - .../test_server_bootstrap_expiry_publisher.py | 200 ----- 18 files changed, 159 insertions(+), 5238 deletions(-) delete mode 100644 tests/scheduling/expiry/test_consumer.py delete mode 100644 tests/scheduling/expiry/test_expiry_bootstrap.py delete mode 100644 tests/scheduling/expiry/test_expiry_config.py delete mode 100644 tests/scheduling/expiry/test_facade_and_registry.py delete mode 100644 tests/scheduling/expiry/test_leases.py delete mode 100644 tests/scheduling/expiry/test_models_and_keys.py delete mode 100644 tests/scheduling/expiry/test_publisher.py delete mode 100644 tests/scheduling/expiry/test_redis_loop.py delete mode 100644 tests/scheduling/expiry/test_redis_store.py delete mode 100644 tests/scheduling/expiry/test_runtime.py delete mode 100644 tests/scheduling/expiry/test_session_expiry.py delete mode 100644 tests/scheduling/expiry/test_session_timeout_service.py delete mode 100644 tests/unit/test_server_bootstrap_expiry_publisher.py diff --git a/tests/scheduling/expiry/test_consumer.py b/tests/scheduling/expiry/test_consumer.py deleted file mode 100644 index d0707e999b..0000000000 --- a/tests/scheduling/expiry/test_consumer.py +++ /dev/null @@ -1,734 +0,0 @@ -import asyncio - -import pytest - -from openviking.scheduling.expiry import consumer as consumer_module -from openviking.scheduling.expiry.api import ( - CallbackResult, - CallbackStatus, - HandlerOptions, - ItemIdentity, -) -from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig -from openviking.scheduling.expiry.consumer import ExpiryConsumer -from openviking.scheduling.expiry.keys import encode_item_id -from openviking.scheduling.expiry.leases import ShardLeaseCoordinator -from openviking.scheduling.expiry.metrics import ExpiryMetrics -from openviking.scheduling.expiry.models import ( - BucketItem, - BucketPage, - BucketRef, - Claim, - OwnerRole, - OwnerToken, - RegistryPage, - StoreOperationError, - claim_layer_bounds, - decode_claim_score, - encode_claim_score, - encode_ready_claim_score, -) -from openviking.scheduling.expiry.registry import HandlerBinding -from openviking.scheduling.expiry.shard_runner import ShardRunner - - -class Store: - def __init__( - self, - *, - empty=False, - attempt=0, - now_ms=200_000, - lose_on_renew=False, - ): - self.identity = ItemIdentity("session", "account", "user", "session-id") - self.bucket = BucketRef("session", 0, 60_000) - self.item_id = encode_item_id(self.identity) - self.score = None if empty else encode_claim_score(attempt, 123) - self.now_ms = now_ms - self.empty = empty - self.lose_on_renew = lose_on_renew - self.claim_requests = [] - self.claims = [] - self.advanced = [] - self.acked = [] - self.released = [] - self.metadata_reads = [] - self.renew_started = asyncio.Event() - self.allow_renew = asyncio.Event() - self.callback_finished = asyncio.Event() - self.deferred = [] - - async def try_acquire_owner(self, role, shard_id, token, ttl_seconds, *, item_type=None): - return OwnerToken(role, shard_id, token, 9999999999999, item_type) - - async def renew_owner(self, owner, ttl_seconds): - self.renew_started.set() - if self.lose_on_renew: - await self.allow_renew.wait() - return None - return owner - - async def release_owner(self, owner): - self.released.append(owner) - return True - - async def read_registry_page(self, shard_id, *, after, limit): - return RegistryPage(() if self.score is None else (self.bucket,), None) - - async def read_time_ms(self): - return self.now_ms - - async def read_bucket_page(self, bucket, *, attempt, limit): - if self.score is None: - return BucketPage(()) - lower, upper = claim_layer_bounds(attempt) - if lower <= self.score <= upper: - return BucketPage((BucketItem(self.item_id, self.score),)) - return BucketPage(()) - - async def claim(self, bucket, item, *, attempt, started_at_seconds): - self.claim_requests.append( - (bucket, item, {"attempt": attempt, "started_at_seconds": started_at_seconds}) - ) - if self.score != item.score: - return None - self.score = encode_claim_score(attempt, started_at_seconds) - claim = Claim(bucket, item.item_id, self.score) - self.claims.append(claim) - return claim - - async def advance_retry(self, claim): - if self.score != claim.claim_score: - return False - attempt, _ = decode_claim_score(claim.claim_score) - self.score = encode_ready_claim_score(attempt + 1) - self.advanced.append(claim) - return True - - async def read_metadata(self, item_id): - self.metadata_reads.append(item_id) - return None - - async def ack(self, claim): - if self.score != claim.claim_score: - return False - self.score = None - self.acked.append(claim) - return True - - async def remove_registration_if_empty(self, bucket): - return self.score is None - - async def defer_registration(self, owner, bucket, *, next_poll_at_ms): - self.deferred.append((owner, bucket, next_poll_at_ms)) - return True - - -class Handler: - def __init__(self, *, status=CallbackStatus.COMPLETED, error=None, resist_cancellation=False): - self.status = status - self.error = error - self.resist_cancellation = resist_cancellation - self.started = asyncio.Event() - self.cancelled = asyncio.Event() - self.release = asyncio.Event() - self.finished = asyncio.Event() - self.requests = [] - - async def handle(self, request): - self.requests.append(request) - self.started.set() - try: - while not self.release.is_set(): - try: - await self.release.wait() - except asyncio.CancelledError: - self.cancelled.set() - if not self.resist_cancellation: - raise - if self.error is not None: - raise self.error - return CallbackResult(self.status, "test callback") - finally: - self.finished.set() - - -class Runner: - def __init__(self, *, stop_on_shard=0, error=None): - self.calls = [] - self.stop_on_shard = stop_on_shard - self.error = error - - async def run_if_acquired(self, role, shard_id, process, stop, *, item_type=None): - self.calls.append((role, item_type, shard_id, process)) - if shard_id == self.stop_on_shard: - stop.set() - if self.error is not None and shard_id == 0: - raise self.error - return False - - -def config(**overrides): - values = { - "shard_count": 1, - "owner_renew_interval_seconds": 0.01, - "callback_poll_seconds": 0.01, - "shutdown_timeout_seconds": 0.1, - "processing_timeout_seconds": 2, - "redis": ExpiryRedisConfig.model_construct(request_budget_seconds=0.05), - } - values.update(overrides) - return ExpiryConfig.model_construct(**values) - - -def consumer(store, handler, *, max_attempts=3, metadata_enabled=True, metrics=None): - timer_config = config() - metrics = metrics or ExpiryMetrics() - coordinator = ShardLeaseCoordinator(store, timer_config, metrics) - shard_runner = ShardRunner(coordinator, timer_config, metrics) - binding = HandlerBinding( - handler, - HandlerOptions( - concurrency=1, - max_attempts=max_attempts, - metadata_enabled=metadata_enabled, - ), - ) - return ( - ExpiryConsumer( - store, - shard_runner, - "session", - binding, - timer_config, - metrics, - start_shard=0, - ), - shard_runner, - ) - - -def worker_consumer(store, shard_runner, timer_config): - return ExpiryConsumer( - store, - shard_runner, - "session", - HandlerBinding(Handler(), HandlerOptions()), - timer_config, - ExpiryMetrics(), - start_shard=0, - ) - - -@pytest.mark.asyncio -async def test_empty_registry_finishes_and_releases_owner_without_callback(): - store = Store(empty=True) - handler = Handler() - resource_consumer, shard_runner = consumer(store, handler) - assert ( - await asyncio.wait_for( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ), - timeout=2.0, - ) - is True - ) - assert handler.requests == [] - assert store.claim_requests == [] - assert len(store.released) == 1 - assert shard_runner.has_pending is False - - -@pytest.mark.asyncio -async def test_completed_callback_with_missing_metadata_acks_claim_and_releases_owner(): - store = Store() - handler = Handler() - handler.release.set() - resource_consumer, shard_runner = consumer(store, handler) - assert ( - await asyncio.wait_for( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ), - timeout=2.0, - ) - is True - ) - assert [request.identity for request in handler.requests] == [store.identity] - assert store.metadata_reads == [store.item_id] - assert store.acked == [store.claims[0]] - assert store.claim_requests[0][2]["attempt"] == 0 - assert len(store.released) == 1 - assert shard_runner.has_pending is False - - -@pytest.mark.asyncio -async def test_completed_callback_skips_metadata_lookup_when_disabled(): - store = Store() - handler = Handler() - handler.release.set() - resource_consumer, shard_runner = consumer(store, handler, metadata_enabled=False) - - assert ( - await asyncio.wait_for( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ), - timeout=2.0, - ) - is True - ) - assert [request.identity for request in handler.requests] == [store.identity] - assert store.metadata_reads == [] - assert store.acked == [store.claims[0]] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failure", ["retry", "exception"]) -@pytest.mark.parametrize("last_attempt", [False, True]) -async def test_failed_callback_advances_immediately_and_acks_after_last_attempt( - failure, last_attempt -): - store = Store(attempt=1 if last_attempt else 0) - handler = Handler( - status=CallbackStatus.RETRY, - error=RuntimeError("callback failed") if failure == "exception" else None, - ) - resource_consumer, shard_runner = consumer(store, handler, max_attempts=2) - stop = asyncio.Event() - run_task = asyncio.create_task( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - stop, - item_type="session", - ) - ) - try: - await asyncio.wait_for(handler.started.wait(), timeout=2.0) - callback_tasks = set(resource_consumer._callback_tasks) - assert callback_tasks - handler.release.set() - await asyncio.wait_for(handler.finished.wait(), timeout=2.0) - finished, pending = await asyncio.wait(callback_tasks, timeout=2.0) - assert finished == callback_tasks - assert not pending - assert await asyncio.wait_for(run_task, timeout=2.0) is True - assert len(store.acked) == 1 - assert len(store.advanced) == (0 if last_attempt else 1) - assert len(handler.requests) == (1 if last_attempt else 2) - assert store.deferred == [] - assert len(store.released) == 1 - assert shard_runner.has_pending is False - finally: - stop.set() - handler.release.set() - await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) - assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True - - -@pytest.mark.asyncio -@pytest.mark.parametrize("result", ["completed", "retry", "exception"]) -@pytest.mark.parametrize("last_attempt", [False, True]) -async def test_cancelled_callback_never_acks_after_drain_when_handler_suppresses_cancel( - result, last_attempt -): - store = Store(attempt=1 if last_attempt else 0) - handler = Handler( - status=CallbackStatus.COMPLETED if result == "completed" else CallbackStatus.RETRY, - error=RuntimeError("callback failed") if result == "exception" else None, - resist_cancellation=True, - ) - resource_consumer, shard_runner = consumer(store, handler, max_attempts=2) - stop = asyncio.Event() - run_task = asyncio.create_task( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - stop, - item_type="session", - ) - ) - try: - await asyncio.wait_for(handler.started.wait(), timeout=2.0) - with pytest.raises(StoreOperationError) as error: - await resource_consumer.drain(0) - assert error.value.code == "shard_not_stopped" - await asyncio.wait_for(handler.cancelled.wait(), timeout=2.0) - # Keep the shard running: cancellation of the child alone must block ACK, - # without relying on owner loss or the runner's shutdown revocation. - assert not run_task.done() - assert store.acked == [] - assert store.released == [] - handler.release.set() - await asyncio.wait_for(handler.finished.wait(), timeout=2.0) - await resource_consumer.drain(1.0) - assert store.acked == [] - stop.set() - assert await asyncio.wait_for(run_task, timeout=2.0) is True - assert len(handler.requests) == 1 - assert store.released == [] - assert shard_runner.has_pending is False - finally: - stop.set() - handler.release.set() - await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) - assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True - - -@pytest.mark.asyncio -async def test_timed_out_last_attempt_is_retried_without_becoming_exhausted(): - store = Store(attempt=2) - handler = Handler() - handler.release.set() - resource_consumer, shard_runner = consumer(store, handler) - await asyncio.wait_for( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ), - timeout=2.0, - ) - assert len(store.acked) == 1 - assert len(store.claims) == 1 - assert decode_claim_score(store.claims[0].claim_score)[0] == 2 - assert len(handler.requests) == 1 - assert len(store.released) == 1 - assert shard_runner.has_pending is False - - -@pytest.mark.asyncio -async def test_waiting_bucket_defers_registration_to_exact_timeout_deadline(): - store = Store(now_ms=124_000) - handler = Handler() - resource_consumer, shard_runner = consumer(store, handler) - - await asyncio.wait_for( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ), - timeout=2.0, - ) - - assert handler.requests == [] - assert store.claim_requests == [] - assert len(store.deferred) == 1 - assert store.deferred[0][2] == 125_000 - - -@pytest.mark.asyncio -async def test_waiting_attempt_does_not_block_ready_item_in_next_attempt(): - store = Store(now_ms=124_000) - retry_identity = ItemIdentity("session", "account", "user", "retry") - retry_item_id = encode_item_id(retry_identity) - retry_score = encode_ready_claim_score(1) - - async def read_bucket_page(bucket, *, attempt, limit): - if attempt == 0: - return ( - BucketPage(()) - if store.score is None - else BucketPage((BucketItem(store.item_id, store.score),)) - ) - if attempt == 1 and retry_score is not None: - return BucketPage((BucketItem(retry_item_id, retry_score),)) - return BucketPage(()) - - async def claim(bucket, item, *, attempt, started_at_seconds): - nonlocal retry_score - store.claim_requests.append( - (bucket, item, {"attempt": attempt, "started_at_seconds": started_at_seconds}) - ) - if item.item_id != retry_item_id or item.score != retry_score: - return None - retry_score = encode_claim_score(attempt, started_at_seconds) - result = Claim(bucket, item.item_id, retry_score) - store.claims.append(result) - return result - - async def ack(claim): - nonlocal retry_score - if claim.item_id != retry_item_id or claim.claim_score != retry_score: - return False - retry_score = None - store.acked.append(claim) - return True - - store.read_bucket_page = read_bucket_page - store.claim = claim - store.ack = ack - handler = Handler() - handler.release.set() - resource_consumer, shard_runner = consumer(store, handler) - - await asyncio.wait_for( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ), - timeout=2.0, - ) - - assert [request.identity for request in handler.requests] == [retry_identity] - assert len(store.acked) == 1 - assert store.deferred[0][2] == 125_000 - - -@pytest.mark.asyncio -async def test_claim_conflict_rescans_without_duplicate_callback(): - store = Store() - original_claim = store.claim - conflicts = 1 - - async def claim_with_conflict(*args, **kwargs): - nonlocal conflicts - if conflicts: - conflicts -= 1 - store.claim_requests.append((args[0], args[1], kwargs)) - return None - return await original_claim(*args, **kwargs) - - store.claim = claim_with_conflict - handler = Handler() - handler.release.set() - resource_consumer, shard_runner = consumer(store, handler) - - await asyncio.wait_for( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ), - timeout=2.0, - ) - - assert len(store.claim_requests) == 2 - assert len(handler.requests) == 1 - assert len(store.acked) == 1 - - -@pytest.mark.asyncio -async def test_last_explicit_failure_emits_exhausted_metric_and_removes_item(): - events = [] - metrics = ExpiryMetrics(lambda name, value, labels: events.append((name, value, labels))) - store = Store(attempt=1) - handler = Handler(status=CallbackStatus.RETRY) - handler.release.set() - resource_consumer, shard_runner = consumer( - store, - handler, - max_attempts=2, - metrics=metrics, - ) - - await asyncio.wait_for( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ), - timeout=2.0, - ) - - assert store.score is None - assert len(store.acked) == 1 - assert ( - "timer_callback_exhausted", - 1.0, - {"item_type": "session", "outcome": "callback_retry"}, - ) in events - - -@pytest.mark.asyncio -async def test_shard_visit_filters_type_and_processes_only_one_minute(): - store = Store() - second = BucketRef("session", 0, 120_000) - watch = BucketRef("watch", 0, 60_000) - - async def read_registry_page(shard_id, *, after, limit): - return RegistryPage((watch, store.bucket, second), None) - - store.read_registry_page = read_registry_page - handler = Handler() - handler.release.set() - resource_consumer, shard_runner = consumer(store, handler) - - await shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ) - - assert len(handler.requests) == 1 - assert store.claim_requests[0][0] == store.bucket - assert all(bucket != second for bucket, _, _ in store.claim_requests) - - -@pytest.mark.asyncio -async def test_owner_loss_keeps_resistant_callback_pending_and_prevents_late_ack(): - store = Store(lose_on_renew=True) - handler = Handler(resist_cancellation=True) - resource_consumer, shard_runner = consumer(store, handler) - stop = asyncio.Event() - run_task = asyncio.create_task( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - stop, - item_type="session", - ) - ) - next_run = None - try: - await asyncio.wait_for(handler.started.wait(), timeout=2.0) - await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) - store.allow_renew.set() - await asyncio.wait_for(handler.cancelled.wait(), timeout=2.0) - finished, _ = await asyncio.wait({run_task}, timeout=0.3) - assert run_task in finished, ( - "Consumer cleanup must not join a resistant callback indefinitely" - ) - assert run_task.result() is True - assert shard_runner.has_pending is True - assert await asyncio.wait_for(shard_runner.wait_pending(0.01), timeout=2.0) is False - assert store.acked == [] - assert store.released == [] - assert len(resource_consumer._callback_tasks) == 1 - - store.lose_on_renew = False - store.now_ms += 3_000 - next_run = asyncio.create_task( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - resource_consumer.process_shard, - asyncio.Event(), - item_type="session", - ) - ) - - async def wait_for_second_callback(): - while len(handler.requests) < 2: - await asyncio.sleep(0) - - await asyncio.wait_for(wait_for_second_callback(), timeout=2.0) - assert len(resource_consumer._callback_tasks) == 2 - - handler.release.set() - assert await asyncio.wait_for(next_run, timeout=2.0) is True - assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True - assert handler.finished.is_set() - assert store.acked == [store.claims[1]] - assert len(store.released) == 1 - assert not resource_consumer._callback_tasks - assert shard_runner.has_pending is False - finally: - stop.set() - store.allow_renew.set() - handler.release.set() - tasks = [run_task] if next_run is None else [run_task, next_run] - await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=2.0) - assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True - - -@pytest.mark.asyncio -async def test_consumer_worker_delegates_callback_role_and_stops_mid_shard_round(): - store = Store(empty=True) - shard_runner = Runner(stop_on_shard=1) - resource_consumer = worker_consumer(store, shard_runner, config(shard_count=3)) - await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) - assert [(role, item_type, shard_id) for role, item_type, shard_id, _ in shard_runner.calls] == [ - (OwnerRole.CALLBACK, "session", 0), - (OwnerRole.CALLBACK, "session", 1), - ] - assert all( - process == resource_consumer.process_shard for _, _, _, process in shard_runner.calls - ) - - -@pytest.mark.asyncio -async def test_consumer_worker_does_not_enter_runner_when_already_stopped(): - shard_runner = Runner() - resource_consumer = worker_consumer(Store(empty=True), shard_runner, config()) - stop = asyncio.Event() - stop.set() - await resource_consumer.run(stop) - assert shard_runner.calls == [] - - -@pytest.mark.asyncio -async def test_consumer_worker_warns_and_continues_after_unfinished_shard(monkeypatch): - shard_runner = Runner( - stop_on_shard=1, - error=StoreOperationError("shard_not_stopped", "callback still running"), - ) - resource_consumer = worker_consumer(Store(empty=True), shard_runner, config(shard_count=3)) - events = [] - resource_consumer._metrics = ExpiryMetrics( - lambda name, value, labels: events.append((name, value, labels)) - ) - warnings = [] - monkeypatch.setattr( - consumer_module.logger, - "warning", - lambda *args, **kwargs: warnings.append((args, kwargs)), - ) - - await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) - - assert [shard_id for _, _, shard_id, _ in shard_runner.calls] == [0, 1] - assert any( - name == "timer_shard_round" and labels["outcome"] == "orphaned" - for name, _, labels in events - ) - assert warnings - - -@pytest.mark.asyncio -async def test_consumer_worker_continues_shard_round_after_processor_error(): - shard_runner = Runner(stop_on_shard=1, error=RuntimeError("callback failed")) - resource_consumer = worker_consumer(Store(empty=True), shard_runner, config(shard_count=3)) - await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) - assert [shard_id for _, _, shard_id, _ in shard_runner.calls] == [0, 1] - - -@pytest.mark.asyncio -async def test_consumer_worker_propagates_cancellation_instead_of_retrying(): - shard_runner = Runner(error=asyncio.CancelledError()) - resource_consumer = worker_consumer(Store(empty=True), shard_runner, config()) - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(resource_consumer.run(asyncio.Event()), timeout=2.0) - assert len(shard_runner.calls) == 1 diff --git a/tests/scheduling/expiry/test_expiry_bootstrap.py b/tests/scheduling/expiry/test_expiry_bootstrap.py deleted file mode 100644 index f33dfddb4a..0000000000 --- a/tests/scheduling/expiry/test_expiry_bootstrap.py +++ /dev/null @@ -1,156 +0,0 @@ -import asyncio -import json -import logging -import subprocess -import sys - -import pytest - -from openviking.scheduling.expiry import runtime as runtime_module -from openviking_cli import expiry_bootstrap - - -def write_config(tmp_path, *, enabled=True, idle_backend="redis", separate=True): - path = tmp_path / "ov.conf" - path.write_text( - json.dumps( - { - "memory": { - "session_auto_commit": { - "enabled": enabled, - "idle_backend": idle_backend, - } - }, - "expiry": { - "service_name": "documents", - "publisher_separate_process": separate, - "redis": { - "mode": "cluster", - "endpoints": ["redis://proxy.example.com:6379"], - }, - }, - } - ), - encoding="utf-8", - ) - return path - - -def test_internal_publisher_loads_validated_expiry_config(tmp_path): - config = expiry_bootstrap.load_expiry_config(write_config(tmp_path)) - - assert config.service_name == "documents" - assert config.redis.mode == "cluster" - - -@pytest.mark.parametrize( - "options", - [ - {"enabled": False}, - {"idle_backend": "local"}, - {"separate": False}, - ], -) -def test_internal_publisher_rejects_configuration_that_does_not_select_it(tmp_path, options): - with pytest.raises(ValueError): - expiry_bootstrap.load_expiry_config(write_config(tmp_path, **options)) - - -def test_startup_status_is_replaced_atomically(tmp_path): - status_path = tmp_path / "status.json" - - expiry_bootstrap.write_startup_status(status_path, "failed", "redis unavailable") - - assert json.loads(status_path.read_text(encoding="utf-8"))["error"] == "redis unavailable" - assert list(tmp_path.glob("*.tmp")) == [] - - -@pytest.mark.asyncio -async def test_publisher_closes_runtime_before_joining_wait_task(monkeypatch, tmp_path): - instances = [] - - class Runtime: - def __init__(self, config, *, roles): - self.stopped = asyncio.Event() - self.close_calls = 0 - instances.append(self) - - async def start_storage(self): - pass - - async def start_roles(self): - pass - - async def wait(self): - await self.stopped.wait() - - async def close(self): - self.close_calls += 1 - self.stopped.set() - - monkeypatch.setattr(runtime_module, "ExpiryRuntime", Runtime) - stop = asyncio.Event() - stop.set() - - await asyncio.wait_for( - expiry_bootstrap.run_publisher(object(), stop, tmp_path / "status.json"), - timeout=1.0, - ) - - assert instances[0].close_calls == 1 - - -@pytest.mark.parametrize("password", ["short-secret", "private-password-" * 40]) -@pytest.mark.parametrize("failure", ["endpoint", "password_sources", "unrelated"]) -def test_publisher_config_failure_logs_and_status_do_not_expose_password( - tmp_path, caplog, password, failure -): - config_path = write_config(tmp_path) - data = json.loads(config_path.read_text()) - data["expiry"]["redis"]["password"] = password - if failure == "endpoint": - data["expiry"]["redis"]["endpoints"] = ["invalid-endpoint"] - elif failure == "password_sources": - data["expiry"]["redis"]["password_env"] = "REDIS_PASSWORD" - else: - data["default_account"] = [] - config_path.write_text(json.dumps(data)) - status = tmp_path / "status.json" - with caplog.at_level(logging.ERROR): - result = expiry_bootstrap.main( - ["--config", str(config_path), "--startup-status", str(status)] - ) - assert result == 1 - assert password not in caplog.text - assert password not in status.read_text() - assert json.loads(status.read_text())["status"] == "failed" - assert "configuration failed" in caplog.text - assert all(record.exc_info is None for record in caplog.records) - - -def test_publisher_runtime_failure_retains_exception_logging(monkeypatch, tmp_path, caplog): - config_path = write_config(tmp_path) - - async def fail(*args): - raise RuntimeError("runtime failure") - - monkeypatch.setattr(expiry_bootstrap, "run_publisher", fail) - with caplog.at_level(logging.ERROR): - result = expiry_bootstrap.main( - ["--config", str(config_path), "--startup-status", str(tmp_path / "status")] - ) - assert result == 1 - assert "runtime failure" in caplog.text - assert any(record.exc_info is not None for record in caplog.records) - - -def test_module_entrypoint_invokes_main(): - result = subprocess.run( - [sys.executable, "-m", "openviking_cli.expiry_bootstrap", "--help"], - capture_output=True, - text=True, - check=False, - ) - - assert result.returncode == 0 - assert "--startup-status" in result.stdout diff --git a/tests/scheduling/expiry/test_expiry_config.py b/tests/scheduling/expiry/test_expiry_config.py deleted file mode 100644 index ef224b1145..0000000000 --- a/tests/scheduling/expiry/test_expiry_config.py +++ /dev/null @@ -1,223 +0,0 @@ -import traceback - -import pytest -from pydantic import ValidationError - -from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig -from openviking_cli.utils.config.memory_config import SessionAutoCommitConfig -from openviking_cli.utils.config.open_viking_config import OpenVikingConfig - - -@pytest.mark.parametrize("value", ["local", "redis"]) -def test_idle_backend_accepts_supported_values(value): - assert SessionAutoCommitConfig.model_validate({"idle_backend": value}).idle_backend == value - - -def test_idle_backend_rejects_legacy_value(): - with pytest.raises(ValidationError, match="idle_backend"): - SessionAutoCommitConfig.model_validate({"idle_backend": "legacy"}) - - -def test_publisher_uses_separate_process_by_default(): - assert ExpiryConfig().publisher_separate_process is True - - -def test_incomplete_expiry_config_is_allowed_when_redis_backend_is_disabled(): - config = OpenVikingConfig.from_dict({"expiry": {"service_name": ""}}) - - assert config.expiry.redis.endpoints == [] - - -def test_expiry_capacity_defaults(): - config = ExpiryConfig() - - assert config.shard_count == 128 - assert config.session_consumer_concurrency == 8 - - -@pytest.mark.parametrize("value", [1, 64, 128, 512]) -def test_shard_count_accepts_configured_value(value): - assert ExpiryConfig(shard_count=value).shard_count == value - - -@pytest.mark.parametrize("value", [0, 513]) -def test_shard_count_is_bounded(value): - with pytest.raises(ValidationError): - ExpiryConfig(shard_count=value) - - -@pytest.mark.parametrize("value", [1, 8, 1024]) -def test_session_consumer_concurrency_accepts_positive_value(value): - assert ExpiryConfig(session_consumer_concurrency=value).session_consumer_concurrency == value - - -def test_session_consumer_concurrency_must_be_positive(): - with pytest.raises(ValidationError): - ExpiryConfig(session_consumer_concurrency=0) - - -@pytest.mark.parametrize("value", [0, 4097]) -def test_cleanup_batch_size_is_bounded(value): - with pytest.raises(ValidationError): - ExpiryConfig(cleanup_batch_size=value) - - -@pytest.mark.parametrize( - "field", - ["reporting_enabled", "scheduler_enabled", "callback_enabled"], -) -def test_legacy_role_switches_are_rejected(field): - with pytest.raises(ValidationError, match=field): - ExpiryConfig.model_validate({field: True}) - - -def test_cluster_rejects_mixed_transport_schemes(): - with pytest.raises(ValidationError, match="one transport scheme"): - ExpiryConfig( - service_name="documents", - redis={ - "mode": "cluster", - "endpoints": ["redis://one:6379", "rediss://two:6379"], - }, - ) - - -def test_redis_password_can_be_configured_directly(): - config = ExpiryConfig( - service_name="documents", - redis={ - "endpoints": ["localhost:6379"], - "password": "secret", - }, - ) - - assert config.redis.password == "secret" - - -def test_redis_password_sources_are_mutually_exclusive(): - with pytest.raises(ValidationError, match="mutually exclusive"): - ExpiryConfig( - service_name="documents", - redis={ - "endpoints": ["localhost:6379"], - "password": "secret", - "password_env": "REDIS_PASSWORD", - }, - ) - - -@pytest.mark.parametrize("password", ["x", "private-redis-password-" * 20]) -@pytest.mark.parametrize("level", ["redis", "expiry", "aggregate"]) -@pytest.mark.parametrize("failure", ["endpoint", "password_sources", "field_type"]) -def test_redis_config_validation_traceback_does_not_expose_password(password, level, failure): - redis = {"endpoints": ["localhost:6379"], "password": password} - if failure == "endpoint": - redis["endpoints"] = ["invalid-endpoint"] - elif failure == "password_sources": - redis["password_env"] = "REDIS_PASSWORD" - else: - redis["max_connections"] = "invalid" - with pytest.raises((ValueError, ValidationError)) as error: - if level == "redis": - ExpiryRedisConfig.model_validate(redis) - elif level == "expiry": - ExpiryConfig.model_validate({"redis": redis}) - else: - OpenVikingConfig.from_dict({"expiry": {"redis": redis}}) - formatted = "".join(traceback.format_exception(error.value)) - # Short passwords are tested as raw input values rather than arbitrary - # characters which can also occur in ordinary error text. - assert f"'password': '{password}'" not in formatted - if len(password) > 1: - assert password not in formatted - assert "input_value=" not in formatted - - -@pytest.mark.parametrize( - "endpoint", - [ - "redis://user:embedded-secret@localhost", - "redis://user:embedded-secret@localhost:bad-port", - "redis://user:embedded-secret@[bad-address]:6379", - "redis://user:embedded-secret@localhost:6379", - ], -) -def test_redis_endpoint_validation_never_echoes_credentials(endpoint): - with pytest.raises(ValidationError) as error: - ExpiryRedisConfig(endpoints=[endpoint]) - assert "embedded-secret" not in "".join(traceback.format_exception(error.value)) - - -def test_aggregate_validation_hides_redis_password_for_unrelated_error(): - with pytest.raises(ValueError) as error: - OpenVikingConfig.from_dict( - {"default_account": [], "expiry": {"redis": {"password": "private-password"}}} - ) - assert "private-password" not in "".join(traceback.format_exception(error.value)) - - -def test_password_env_must_exist_when_resolved(monkeypatch): - monkeypatch.delenv("MISSING_REDIS_PASSWORD", raising=False) - config = ExpiryRedisConfig(password_env="MISSING_REDIS_PASSWORD") - - with pytest.raises(ValueError, match="MISSING_REDIS_PASSWORD"): - config.resolve_password() - - -def test_password_env_is_resolved_without_exposing_it_in_endpoint(monkeypatch): - monkeypatch.setenv("EXPIRY_REDIS_PASSWORD", "secret") - config = ExpiryRedisConfig(password_env="EXPIRY_REDIS_PASSWORD") - - assert config.resolve_password() == "secret" - - -def test_redis_endpoint_rejects_embedded_credentials(): - with pytest.raises(ValidationError, match="dedicated Redis fields"): - ExpiryRedisConfig(endpoints=["redis://user:secret@localhost:6379"]) - - -@pytest.mark.parametrize( - ("expiry", "message"), - [ - ({}, "service_name"), - ({"service_name": "documents"}, "redis.endpoints"), - ], -) -def test_redis_idle_backend_requires_complete_expiry_config(expiry, message): - with pytest.raises(ValueError, match=message): - OpenVikingConfig.from_dict( - { - "memory": { - "session_auto_commit": { - "enabled": True, - "idle_backend": "redis", - } - }, - "expiry": expiry, - } - ) - - -def test_expiry_top_level_config_is_parsed(): - config = OpenVikingConfig.from_dict( - { - "expiry": { - "service_name": "documents", - "publisher_separate_process": False, - "shard_count": 64, - "session_consumer_concurrency": 16, - "redis": {"endpoints": ["localhost:6379"]}, - }, - "memory": { - "session_auto_commit": { - "enabled": True, - "idle_backend": "redis", - } - }, - } - ) - - assert config.expiry.service_name == "documents" - assert config.expiry.publisher_separate_process is False - assert config.expiry.shard_count == 64 - assert config.expiry.session_consumer_concurrency == 16 diff --git a/tests/scheduling/expiry/test_facade_and_registry.py b/tests/scheduling/expiry/test_facade_and_registry.py deleted file mode 100644 index 3783f28342..0000000000 --- a/tests/scheduling/expiry/test_facade_and_registry.py +++ /dev/null @@ -1,177 +0,0 @@ -import pytest - -from openviking.scheduling.expiry import ( - CallbackResult, - CallbackStatus, - ExpiryError, - ExpiryRegistration, - ExpiryRegistrationResult, - HandlerOptions, - ItemIdentity, -) -from openviking.scheduling.expiry.metrics import ExpiryMetrics -from openviking.scheduling.expiry.models import StoreOperationError -from openviking.scheduling.expiry.registry import HandlerRegistry -from openviking.scheduling.expiry.service import ExpiryRuntimeState, _ExpirySchedulerFacade - - -class Handler: - async def handle(self, request): - return CallbackResult(CallbackStatus.COMPLETED, "ok") - - -class Store: - def __init__(self): - self.calls = [] - self.fail_metadata = False - - async def upsert_resource(self, shard_id, item_id, task): - self.calls.append("resource") - return ExpiryRegistrationResult(task.due_at_ms) - - async def ensure_metadata(self, item_id, task): - self.calls.append("metadata") - if self.fail_metadata: - raise StoreOperationError("redis_error", "metadata failed", retryable=True) - - async def unregister_resource(self, shard_id, item_id, *, delete_metadata=True): - self.calls.append(("unregister", delete_metadata)) - - -def test_registry_validation_and_freeze(): - registry = HandlerRegistry() - registry.register("session", Handler(), HandlerOptions(concurrency=2, max_attempts=3)) - bindings = registry.freeze() - assert bindings["session"].options.max_attempts == 3 - with pytest.raises(ExpiryError) as exc: - registry.register("watch", Handler(), HandlerOptions()) - assert exc.value.code == "registration_closed" - - -def test_registry_rejects_unsupported_type(): - registry = HandlerRegistry() - with pytest.raises(ExpiryError) as exc: - registry.register("watch", Handler(), HandlerOptions()) - assert exc.value.code == "unregistered_type" - - -@pytest.mark.asyncio -async def test_report_writes_resource_before_metadata(): - store = Store() - state = ExpiryRuntimeState(storage_ready=True, accepting_calls=True) - facade = _ExpirySchedulerFacade( - store, - HandlerRegistry(), - state, - shard_count=8, - reporting_enabled=True, - metrics=ExpiryMetrics(), - ) - result = await facade.report( - ExpiryRegistration(ItemIdentity("session", "a", "u", "s"), 1234, 60) - ) - assert result.accepted_due_at_ms == 1234 - assert store.calls == ["resource", "metadata"] - - -@pytest.mark.asyncio -async def test_report_skips_metadata_when_handler_disables_it(): - store = Store() - registry = HandlerRegistry() - registry.register("session", Handler(), HandlerOptions(metadata_enabled=False)) - facade = _ExpirySchedulerFacade( - store, - registry, - ExpiryRuntimeState(storage_ready=True, accepting_calls=True), - shard_count=8, - reporting_enabled=True, - metrics=ExpiryMetrics(), - ) - - result = await facade.report( - ExpiryRegistration(ItemIdentity("session", "a", "u", "s"), 1234, 60) - ) - - assert result.accepted_due_at_ms == 1234 - assert store.calls == ["resource"] - - -@pytest.mark.asyncio -async def test_metadata_failure_does_not_roll_back_resource(): - store = Store() - store.fail_metadata = True - facade = _ExpirySchedulerFacade( - store, - HandlerRegistry(), - ExpiryRuntimeState(storage_ready=True, accepting_calls=True), - shard_count=8, - reporting_enabled=True, - metrics=ExpiryMetrics(), - ) - with pytest.raises(ExpiryError) as exc: - await facade.report(ExpiryRegistration(ItemIdentity("session", "a", "u", "s"), 1234, 60)) - assert exc.value.code == "timer_unavailable" - assert store.calls == ["resource", "metadata"] - - -@pytest.mark.asyncio -async def test_report_rejects_unsupported_type_before_store_write(): - store = Store() - state = ExpiryRuntimeState(storage_ready=True, accepting_calls=True) - facade = _ExpirySchedulerFacade( - store, - HandlerRegistry(), - state, - shard_count=8, - reporting_enabled=True, - metrics=ExpiryMetrics(), - ) - report = ExpiryRegistration(ItemIdentity("watch", "a", "u", "s"), 1234, 60) - with pytest.raises(ExpiryError) as exc: - await facade.report(report) - assert exc.value.code == "unregistered_type" - assert store.calls == [] - - -@pytest.mark.asyncio -async def test_unregister_rejects_unsupported_type_before_store_write(): - store = Store() - facade = _ExpirySchedulerFacade( - store, - HandlerRegistry(), - ExpiryRuntimeState(storage_ready=True, accepting_calls=True), - shard_count=8, - reporting_enabled=True, - metrics=ExpiryMetrics(), - ) - with pytest.raises(ExpiryError) as exc: - await facade.unregister(ItemIdentity("watch", "a", "u", "s")) - assert exc.value.code == "unregistered_type" - assert store.calls == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("metadata_enabled", "delete_metadata"), - [(True, True), (False, False)], -) -async def test_unregister_follows_handler_metadata_option(metadata_enabled, delete_metadata): - store = Store() - registry = HandlerRegistry() - registry.register( - "session", - Handler(), - HandlerOptions(metadata_enabled=metadata_enabled), - ) - facade = _ExpirySchedulerFacade( - store, - registry, - ExpiryRuntimeState(storage_ready=True, accepting_calls=True), - shard_count=8, - reporting_enabled=True, - metrics=ExpiryMetrics(), - ) - - await facade.unregister(ItemIdentity("session", "a", "u", "s")) - - assert store.calls == [("unregister", delete_metadata)] diff --git a/tests/scheduling/expiry/test_leases.py b/tests/scheduling/expiry/test_leases.py deleted file mode 100644 index 5e00142e9d..0000000000 --- a/tests/scheduling/expiry/test_leases.py +++ /dev/null @@ -1,542 +0,0 @@ -import asyncio -import gc -import time -import weakref -from types import SimpleNamespace - -import pytest - -from openviking.scheduling.expiry import leases as leases_module -from openviking.scheduling.expiry import models as models_module -from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig -from openviking.scheduling.expiry.leases import ShardLeaseCoordinator -from openviking.scheduling.expiry.metrics import ExpiryMetrics -from openviking.scheduling.expiry.models import ( - OwnerRole, - OwnerToken, - ShardLease, - StoreOperationError, -) -from openviking.scheduling.expiry.shard_runner import ShardExecution, ShardRunner - - -class Store: - def __init__(self, *, busy=False, renew_outcome="success", block_renew=False): - self.busy = busy - self.renew_outcome = renew_outcome - self.block_renew = block_renew - self.renew_started = asyncio.Event() - self.allow_renew = asyncio.Event() - self.acquired = [] - self.renewed = [] - self.released = [] - - async def try_acquire_owner(self, role, shard_id, token, ttl_seconds, *, item_type=None): - self.acquired.append((role, item_type, shard_id, token, ttl_seconds)) - if self.busy: - return None - return OwnerToken(role, shard_id, token, 9999999999999, item_type) - - async def renew_owner(self, owner, ttl_seconds): - self.renewed.append((owner, ttl_seconds)) - self.renew_started.set() - if self.block_renew: - await self.allow_renew.wait() - if self.renew_outcome == "error": - raise RuntimeError("renew failed") - if self.renew_outcome == "lost": - return None - return OwnerToken( - owner.role, - owner.shard_id, - owner.token, - owner.expires_at_ms + 1, - owner.item_type, - ) - - async def release_owner(self, owner): - self.released.append(owner) - return True - - -def lease(): - return ShardLease( - OwnerToken(OwnerRole.SCHEDULER, 0, "token", 9999999999999), - asyncio.Event(), - float("inf"), - ) - - -def config(**overrides): - values = { - "shard_count": 1, - "owner_renew_interval_seconds": 0.01, - "shutdown_timeout_seconds": 1.0, - "redis": ExpiryRedisConfig.model_construct(request_budget_seconds=0.05), - } - values.update(overrides) - return ExpiryConfig.model_construct(**values) - - -def runner(store, timer_config=None, metrics=None): - timer_config = timer_config or config() - metrics = metrics or ExpiryMetrics() - coordinator = ShardLeaseCoordinator(store, timer_config, metrics) - return ShardRunner(coordinator, timer_config, metrics) - - -@pytest.mark.asyncio -async def test_coordinator_acquires_renews_and_releases_matching_owner(): - store = Store() - coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) - acquired = await coordinator.try_acquire(OwnerRole.CALLBACK, 3, item_type="session") - assert acquired is not None - assert acquired.owner.role is OwnerRole.CALLBACK - assert acquired.owner.item_type == "session" - assert acquired.owner.shard_id == 3 - assert len(acquired.owner.token) == 32 - int(acquired.owner.token, 16) - assert acquired.valid_until_monotonic > time.monotonic() - original = acquired.owner - assert await coordinator.renew(acquired) is True - assert acquired.owner != original - assert acquired.owner.token == original.token - await coordinator.release(acquired) - assert store.released == [acquired.owner] - - -@pytest.mark.asyncio -async def test_coordinator_returns_none_when_shard_is_busy(): - store = Store(busy=True) - coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) - assert await coordinator.try_acquire(OwnerRole.SCHEDULER, 0) is None - assert store.renewed == [] - assert store.released == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("outcome", ["lost", "error"]) -async def test_failed_renewal_marks_lease_lost_and_cannot_revive(outcome): - store = Store(renew_outcome=outcome) - coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) - acquired = await coordinator.try_acquire(OwnerRole.SCHEDULER, 0) - original = acquired.owner - assert await coordinator.renew(acquired) is False - assert acquired.lost.is_set() - store.renew_outcome = "success" - assert await coordinator.renew(acquired) is False - await coordinator.release(acquired) - assert acquired.owner == original - assert len(store.renewed) == 1 - assert store.released == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("invalidation", ["deadline", "lost", "revoked"]) -async def test_successful_renew_response_cannot_revive_invalidated_lease(monkeypatch, invalidation): - clock = SimpleNamespace(now=10.0) - local_time = SimpleNamespace(monotonic=lambda: clock.now) - monkeypatch.setattr(leases_module, "time", local_time) - monkeypatch.setattr(models_module, "time", local_time) - store = Store(block_renew=True) - coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) - acquired = await coordinator.try_acquire(OwnerRole.SCHEDULER, 0) - original = acquired.owner - original_deadline = acquired.valid_until_monotonic - renew_task = asyncio.create_task(coordinator.renew(acquired)) - try: - await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) - if invalidation == "deadline": - clock.now = original_deadline - else: - getattr(acquired, invalidation).set() - store.allow_renew.set() - assert await asyncio.wait_for(renew_task, timeout=2.0) is False - assert acquired.owner == original - assert acquired.valid_until_monotonic == original_deadline - with pytest.raises(StoreOperationError, match="no longer valid"): - acquired.ensure_active() - finally: - store.allow_renew.set() - await asyncio.wait_for(asyncio.gather(renew_task, return_exceptions=True), timeout=2.0) - - -@pytest.mark.asyncio -async def test_release_does_not_delete_expired_owner(): - store = Store() - coordinator = ShardLeaseCoordinator(store, config(), ExpiryMetrics()) - acquired = lease() - acquired.valid_until_monotonic = 0.0 - await coordinator.release(acquired) - assert acquired.lost.is_set() - assert store.released == [] - - -@pytest.mark.asyncio -async def test_runner_releases_normally_completed_processor_and_children(): - store = Store() - shard_runner = runner(store) - observed = [] - - async def process(execution, stop): - execution.ensure_active() - observed.append((execution.owner, execution.shard_id)) - - async def child(): - execution.ensure_active() - observed.append("child") - - await execution.spawn(child) - - assert ( - await asyncio.wait_for( - shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, asyncio.Event()), - timeout=2.0, - ) - is True - ) - assert observed[0][1] == 0 - assert observed[1] == "child" - assert store.released == [observed[0][0]] - assert shard_runner.has_pending is False - - -@pytest.mark.asyncio -async def test_runner_busy_shard_does_not_start_processor(): - store = Store(busy=True) - shard_runner = runner(store) - called = [] - - async def process(execution, stop): - called.append(execution) - - assert ( - await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, asyncio.Event()) - is False - ) - assert called == [] - assert store.released == [] - assert shard_runner.has_pending is False - - -@pytest.mark.asyncio -async def test_callback_runner_scopes_active_work_by_item_type(): - store = Store() - shard_runner = runner(store) - session_started = asyncio.Event() - release_session = asyncio.Event() - - async def session_process(execution, stop): - session_started.set() - await release_session.wait() - execution.ensure_active() - - async def watch_process(execution, stop): - execution.ensure_active() - - session_task = asyncio.create_task( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - session_process, - asyncio.Event(), - item_type="session", - ) - ) - try: - await asyncio.wait_for(session_started.wait(), timeout=2.0) - assert ( - await shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - watch_process, - asyncio.Event(), - item_type="watch", - ) - is True - ) - finally: - release_session.set() - await asyncio.wait_for(session_task, timeout=2.0) - - -@pytest.mark.asyncio -async def test_runner_does_not_acquire_after_stop(): - store = Store() - shard_runner = runner(store) - stop = asyncio.Event() - stop.set() - - async def process(execution, stop): - pytest.fail("A stopped runner must not invoke the processor") - - assert await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, stop) is False - assert store.acquired == [] - - -@pytest.mark.asyncio -async def test_stop_grace_allows_processor_to_finish_and_release(): - store = Store() - shard_runner = runner(store) - stop = asyncio.Event() - started = asyncio.Event() - observed = [] - - async def process(execution, current_stop): - started.set() - await current_stop.wait() - execution.ensure_active() - observed.append(execution.owner) - - run_task = asyncio.create_task( - shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, stop) - ) - try: - await asyncio.wait_for(started.wait(), timeout=2.0) - stop.set() - assert await asyncio.wait_for(run_task, timeout=2.0) is True - assert store.released == observed - assert shard_runner.has_pending is False - finally: - stop.set() - await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) - - -@pytest.mark.asyncio -async def test_renewal_loss_cancels_processor_during_stop_grace(): - store = Store(renew_outcome="lost", block_renew=True) - shard_runner = runner(store) - stop = asyncio.Event() - process_started = asyncio.Event() - stop_observed = asyncio.Event() - process_cancelled = asyncio.Event() - release_process = asyncio.Event() - - async def process(execution, current_stop): - process_started.set() - try: - await current_stop.wait() - stop_observed.set() - await release_process.wait() - except asyncio.CancelledError: - process_cancelled.set() - raise - - run_task = asyncio.create_task( - shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, stop) - ) - cancelled_task = asyncio.create_task(process_cancelled.wait()) - try: - await asyncio.wait_for(process_started.wait(), timeout=2.0) - await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) - stop.set() - await asyncio.wait_for(stop_observed.wait(), timeout=2.0) - await asyncio.sleep(0) - await asyncio.sleep(0) - store.allow_renew.set() - finished, _ = await asyncio.wait({cancelled_task}, timeout=0.3) - assert cancelled_task in finished, "Lease loss must not wait for the stop grace to expire" - await asyncio.wait_for(run_task, timeout=2.0) - assert store.released == [] - assert shard_runner.has_pending is False - finally: - stop.set() - store.allow_renew.set() - release_process.set() - cancelled_task.cancel() - await asyncio.gather(cancelled_task, return_exceptions=True) - await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) - - -@pytest.mark.asyncio -async def test_processor_exception_releases_active_lease(): - store = Store() - shard_runner = runner(store) - owners = [] - - async def process(execution, stop): - owners.append(execution.owner) - raise RuntimeError("processor failed") - - with pytest.raises(RuntimeError, match="processor failed"): - await asyncio.wait_for( - shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, asyncio.Event()), - timeout=2.0, - ) - assert store.released == owners - assert shard_runner.has_pending is False - - -@pytest.mark.asyncio -async def test_cancellation_resistant_processor_is_retained_without_blocking_new_executions(): - store = Store() - events = [] - metrics = ExpiryMetrics(lambda name, value, labels: events.append((name, value, labels))) - shard_runner = runner(store, config(shutdown_timeout_seconds=0.02), metrics) - stop = asyncio.Event() - started = asyncio.Event() - cancelled = asyncio.Event() - release_process = asyncio.Event() - process_refs = [] - - async def process(execution, current_stop): - process_refs.append(weakref.ref(asyncio.current_task())) - started.set() - while not release_process.is_set(): - try: - await release_process.wait() - except asyncio.CancelledError: - cancelled.set() - - async def completed(execution, current_stop): - execution.ensure_active() - - run_task = asyncio.create_task( - shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, process, stop) - ) - try: - await asyncio.wait_for(started.wait(), timeout=2.0) - stop.set() - await asyncio.wait_for(cancelled.wait(), timeout=2.0) - finished, _ = await asyncio.wait({run_task}, timeout=0.3) - assert run_task in finished, ( - "Cleanup must not wait indefinitely for cancellation resistance" - ) - assert run_task.result() is True - assert shard_runner.has_pending is True - gc.collect() - assert process_refs[0]() is not None - assert not process_refs[0]().done() - assert await asyncio.wait_for(shard_runner.wait_pending(0.01), timeout=2.0) is False - assert store.released == [] - assert ( - await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 1, completed, asyncio.Event()) - is True - ) - assert ( - await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, completed, asyncio.Event()) - is True - ) - assert ( - await shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - completed, - asyncio.Event(), - item_type="session", - ) - is True - ) - assert any( - name == "timer_shard_orphaned" and value == 1 and labels["outcome"] == "scheduler" - for name, value, labels in events - ) - release_process.set() - assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True - assert shard_runner.has_pending is False - assert ( - await shard_runner.run_if_acquired(OwnerRole.SCHEDULER, 0, completed, asyncio.Event()) - is True - ) - assert len(store.released) == 4 - finally: - stop.set() - release_process.set() - await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) - assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True - - -@pytest.mark.asyncio -async def test_lease_loss_cancels_spawned_child_and_prevents_late_ack(): - store = Store(renew_outcome="lost", block_renew=True) - shard_runner = runner(store) - child_started = asyncio.Event() - child_cancelled = asyncio.Event() - child_finished = asyncio.Event() - release_child = asyncio.Event() - acked = [] - executions = [] - - async def process(execution, stop): - executions.append(execution) - - async def child(): - child_started.set() - try: - while not release_child.is_set(): - try: - await release_child.wait() - except asyncio.CancelledError: - child_cancelled.set() - execution.ensure_active() - acked.append("ack") - finally: - child_finished.set() - - execution.spawn(child) - await execution.wait(60.0, stop) - - run_task = asyncio.create_task( - shard_runner.run_if_acquired( - OwnerRole.CALLBACK, - 0, - process, - asyncio.Event(), - item_type="session", - ) - ) - try: - await asyncio.wait_for(child_started.wait(), timeout=2.0) - await asyncio.wait_for(store.renew_started.wait(), timeout=2.0) - store.allow_renew.set() - await asyncio.wait_for(child_cancelled.wait(), timeout=2.0) - finished, _ = await asyncio.wait({run_task}, timeout=0.3) - assert run_task in finished - assert run_task.result() is True - assert shard_runner.has_pending is True - assert store.released == [] - called = [] - - async def forbidden_child(): - called.append(True) - - with pytest.raises(StoreOperationError): - executions[0].spawn(forbidden_child) - assert called == [] - release_child.set() - await asyncio.wait_for(child_finished.wait(), timeout=2.0) - assert await shard_runner.wait_pending(1.0) is True - assert acked == [] - assert store.released == [] - assert shard_runner.has_pending is False - finally: - store.allow_renew.set() - release_child.set() - await asyncio.wait_for(asyncio.gather(run_task, return_exceptions=True), timeout=2.0) - assert await asyncio.wait_for(shard_runner.wait_pending(1.0), timeout=2.0) is True - - -@pytest.mark.asyncio -@pytest.mark.parametrize("invalidation", ["lost", "revoked"]) -async def test_execution_wait_wakes_and_rechecks_invalidation(invalidation): - acquired = lease() - execution = ShardExecution(acquired) - wait_task = asyncio.create_task(execution.wait(60.0, asyncio.Event())) - try: - await asyncio.sleep(0) - getattr(acquired, invalidation).set() - with pytest.raises(StoreOperationError): - await asyncio.wait_for(wait_task, timeout=2.0) - finally: - getattr(acquired, invalidation).set() - await asyncio.wait_for(asyncio.gather(wait_task, return_exceptions=True), timeout=2.0) - - -@pytest.mark.asyncio -async def test_execution_wait_rechecks_local_deadline_without_renewal(): - acquired = lease() - acquired.valid_until_monotonic = time.monotonic() + 0.03 - execution = ShardExecution(acquired) - with pytest.raises(StoreOperationError): - await asyncio.wait_for(execution.wait(60.0, asyncio.Event()), timeout=2.0) - assert acquired.lost.is_set() diff --git a/tests/scheduling/expiry/test_models_and_keys.py b/tests/scheduling/expiry/test_models_and_keys.py deleted file mode 100644 index 5f0564bc67..0000000000 --- a/tests/scheduling/expiry/test_models_and_keys.py +++ /dev/null @@ -1,106 +0,0 @@ -import pytest - -from openviking.scheduling.expiry import ( - ExpiryError, - ExpiryMode, - ExpiryRegistration, - ItemIdentity, -) -from openviking.scheduling.expiry.keys import ( - ExpiryKeys, - decode_component, - decode_item_id, - encode_component, - encode_item_id, - shard_for, -) -from openviking.scheduling.expiry.models import ( - SCORE_TIME_BASE, - BucketRef, - OwnerRole, - bucket_minute_ms, - claim_layer_bounds, - decode_claim_score, - encode_claim_score, - encode_ready_claim_score, - metadata_ttl_ms, - validate_report, -) - - -def test_identity_round_trip_is_canonical_and_isolated(): - identity = ItemIdentity("session", "account:a", "用户/{x}", "session:1") - item_id = encode_item_id(identity) - assert decode_item_id(item_id) == identity - assert "用户" not in item_id - assert "{" not in item_id - assert shard_for(item_id, 512) == shard_for(item_id, 512) - - -@pytest.mark.parametrize("encoded", ["=", "YQ==", "***", "_"]) -def test_decode_component_rejects_non_canonical_values(encoded): - with pytest.raises(ExpiryError, match="encoded identity"): - decode_component(encoded) - - -def test_component_round_trip_preserves_unicode(): - value = "账户/with spaces:✓" - assert decode_component(encode_component(value)) == value - - -def test_expiry_keys_only_use_a_control_hash_tag_where_required(): - keys = ExpiryKeys(encode_component("cluster/resource")) - assert keys.service_key == encode_component("cluster/resource") - ref = BucketRef("session", 7, 120_000) - tag = keys.control_tag(7) - assert keys.namespace() == f"expiry:namespace:{keys.service_key}" - assert keys.metadata("item") == f"expiry:metadata:{keys.service_key}:item" - assert keys.source(7) == f"expiry:publisher:source:{keys.service_key}:7" - assert keys.progress(7) == f"expiry:publisher:progress:{tag}" - assert keys.registry(7) == f"expiry:consumer:registry:{tag}" - assert keys.bucket(ref) == f"expiry:consumer:bucket:{keys.service_key}:session:7:120000" - assert "{" not in keys.source(7) - assert "{" not in keys.bucket(ref) - scheduler_owner = keys.owner(OwnerRole.SCHEDULER, 7) - callback_owner = keys.owner(OwnerRole.CALLBACK, 7, item_type="session") - assert scheduler_owner == f"expiry:publisher:owner:{tag}" - assert callback_owner == f"expiry:consumer:owner:session:{tag}" - - -def test_callback_owner_requires_item_type_scope(): - keys = ExpiryKeys("resource") - with pytest.raises(ValueError, match="requires an item type"): - keys.owner(OwnerRole.CALLBACK, 7) - with pytest.raises(ValueError, match="must not include"): - keys.owner(OwnerRole.SCHEDULER, 7, item_type="session") - - -@pytest.mark.parametrize( - ("timeout", "expected"), - [(3600, 86_400_000), (43_200, 86_400_000), (46_800, 93_600_000)], -) -def test_metadata_ttl(timeout, expected): - assert metadata_ttl_ms(timeout) == expected - - -def test_minute_and_claim_boundaries(): - assert bucket_minute_ms(60_000) == 60_000 - assert bucket_minute_ms(60_001) == 120_000 - score = encode_claim_score(2, 123) - assert decode_claim_score(score) == (2, 123) - assert claim_layer_bounds(0) == (0, SCORE_TIME_BASE - 1) - assert claim_layer_bounds(2) == ( - 2 * SCORE_TIME_BASE + 1, - 3 * SCORE_TIME_BASE - 1, - ) - assert encode_ready_claim_score(2) == 2 * SCORE_TIME_BASE + 1 - - -def test_report_rejects_fixed_mode_and_boolean_numbers(): - identity = ItemIdentity("session", "a", "u", "s") - with pytest.raises(ExpiryError) as exc: - validate_report(ExpiryRegistration(identity, 100, 1, ExpiryMode.FIXED)) - assert exc.value.code == "unsupported_mode" - with pytest.raises(ExpiryError) as exc: - validate_report(ExpiryRegistration(identity, True, 1)) - assert exc.value.code == "invalid_report" diff --git a/tests/scheduling/expiry/test_publisher.py b/tests/scheduling/expiry/test_publisher.py deleted file mode 100644 index 7f172deaf9..0000000000 --- a/tests/scheduling/expiry/test_publisher.py +++ /dev/null @@ -1,306 +0,0 @@ -import asyncio - -import pytest - -from openviking.scheduling.expiry import ItemIdentity -from openviking.scheduling.expiry.config import ExpiryConfig -from openviking.scheduling.expiry.keys import encode_item_id -from openviking.scheduling.expiry.metrics import ExpiryMetrics -from openviking.scheduling.expiry.models import ( - OwnerRole, - OwnerToken, - PublishResult, - ShardLease, - ShardWindow, - SourceCursor, - SourceItem, - SourcePage, - StoreOperationError, -) -from openviking.scheduling.expiry.publisher import ExpiryPublisher -from openviking.scheduling.expiry.shard_runner import ShardExecution - - -class Store: - def __init__(self, *, item=None, items=None, cleanup_results=None, upper_ms=60_000): - self.items = list(items if items is not None else (() if item is None else (item,))) - self.progress = 0 - self.upper_ms = upper_ms - self.staged = [] - self.published = [] - self.cleaned = [] - self.cleanup_results = list(cleanup_results or ()) - self.resets = [] - - async def read_scan_window(self, shard_id): - return ShardWindow(self.progress, self.upper_ms, self.upper_ms + 123) - - async def cleanup_source_page(self, shard_id, through_ms, *, limit): - self.cleaned.append((through_ms, limit)) - return self.cleanup_results.pop(0) if self.cleanup_results else 0 - - async def read_source_page( - self, shard_id, *, lower_exclusive_ms, upper_inclusive_ms, after, limit - ): - items = sorted(self.items, key=lambda item: (item.due_at_ms, item.item_id)) - items = [ - item - for item in items - if lower_exclusive_ms < item.due_at_ms <= upper_inclusive_ms - and (after is None or (item.due_at_ms, item.item_id) > (after.due_at_ms, after.item_id)) - ] - page_items = items[:limit] - next_cursor = None - if len(items) > limit: - last = page_items[-1] - next_cursor = SourceCursor(last.due_at_ms, last.item_id) - return SourcePage(tuple(page_items), next_cursor) - - async def reset_unpublished_minute(self, owner, **kwargs): - self.resets.append(kwargs) - return None - - async def stage_items(self, owner, bucket, item_ids): - self.staged.append((bucket, tuple(item_ids))) - - async def publish_minute(self, owner, **kwargs): - self.published.append(kwargs) - self.progress = kwargs["minute_ms"] - return PublishResult("published", kwargs["minute_ms"]) - - -def lease(): - return ShardLease( - OwnerToken(OwnerRole.SCHEDULER, 0, "token", 9999999999999), - asyncio.Event(), - float("inf"), - ) - - -def config(): - return ExpiryConfig.model_construct( - shard_count=1, - scan_batch_size=8, - cleanup_batch_size=8, - registry_page_size=8, - ) - - -@pytest.mark.asyncio -async def test_empty_scan_does_not_publish_or_advance_progress(): - store = Store() - scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) - await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) - assert store.published == [] - assert store.cleaned == [(0, 8)] - - -@pytest.mark.asyncio -async def test_non_empty_minute_is_fully_staged_before_publish(): - item_id = encode_item_id(ItemIdentity("session", "a", "u", "s")) - store = Store(item=SourceItem(item_id, 60_000)) - scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) - await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) - assert store.staged[0][1] == (item_id,) - assert store.published[0]["minute_ms"] == 60_000 - assert store.cleaned == [(0, 8), (60_000, 8), (60_000, 8)] - assert store.resets[0]["item_types"] == ("session",) - - -@pytest.mark.asyncio -async def test_source_cleanup_drains_bounded_pages(): - store = Store(cleanup_results=[8, 8, 3]) - scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) - - await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) - - assert store.cleaned == [(0, 8), (0, 8), (0, 8)] - - -@pytest.mark.asyncio -async def test_source_cleanup_stops_between_pages(): - stop = asyncio.Event() - - class StoppingStore(Store): - async def cleanup_source_page(self, shard_id, through_ms, *, limit): - removed = await super().cleanup_source_page(shard_id, through_ms, limit=limit) - stop.set() - return removed - - store = StoppingStore(cleanup_results=[8, 8]) - scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) - - await scheduler.process_shard(ShardExecution(lease()), stop) - - assert store.cleaned == [(0, 8)] - - -@pytest.mark.asyncio -async def test_source_cleanup_rechecks_lease_between_pages(): - shard_lease = lease() - - class LosingStore(Store): - async def cleanup_source_page(self, shard_id, through_ms, *, limit): - removed = await super().cleanup_source_page(shard_id, through_ms, limit=limit) - shard_lease.lost.set() - return removed - - store = LosingStore(cleanup_results=[8, 8]) - scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) - - with pytest.raises(StoreOperationError, match="Shard lease is no longer valid"): - await scheduler.process_shard(ShardExecution(shard_lease), asyncio.Event()) - - assert store.cleaned == [(0, 8)] - - -@pytest.mark.asyncio -async def test_unsupported_source_item_is_cleaned_without_advancing_progress(timer_logs): - item_id = encode_item_id(ItemIdentity("watch", "a", "u", "s")) - store = Store(item=SourceItem(item_id, 60_000)) - events = [] - metrics = ExpiryMetrics(lambda name, value, labels: events.append((name, value, labels))) - scheduler = ExpiryPublisher(store, None, config(), metrics) - - await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) - - assert store.staged == [] - assert store.published == [] - assert store.progress == 0 - assert store.cleaned == [(0, 8), (60_000, 8)] - assert events == [ - ( - "timer_source_invalid", - 1.0, - {"item_type": "", "outcome": "unsupported_type"}, - ) - ] - assert "reason=unsupported_type count=1" in timer_logs.text - assert item_id not in timer_logs.text - - -@pytest.mark.asyncio -async def test_all_invalid_minute_does_not_block_later_valid_minute(): - invalid_id = encode_item_id(ItemIdentity("watch", "a", "u", "invalid")) - valid_id = encode_item_id(ItemIdentity("session", "a", "u", "valid")) - store = Store( - items=( - SourceItem(invalid_id, 60_000), - SourceItem(valid_id, 120_000), - ), - upper_ms=120_000, - ) - scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) - - await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) - - assert store.staged[0][1] == (valid_id,) - assert store.published[0]["expected_progress_ms"] == 0 - assert store.published[0]["minute_ms"] == 120_000 - assert store.progress == 120_000 - assert store.cleaned == [(0, 8), (60_000, 8), (120_000, 8), (120_000, 8)] - - -@pytest.mark.asyncio -async def test_malformed_source_item_is_skipped_while_valid_item_is_published( - timer_logs, -): - valid_id = encode_item_id(ItemIdentity("session", "a", "u", "valid")) - malformed_id = "not-an-item-id" - store = Store( - items=( - SourceItem(valid_id, 60_000), - SourceItem(malformed_id, 60_000), - ) - ) - scheduler = ExpiryPublisher(store, None, config(), ExpiryMetrics()) - - await scheduler.process_shard(ShardExecution(lease()), asyncio.Event()) - - assert store.staged[0][1] == (valid_id,) - assert len(store.published[0]["buckets"]) == 1 - assert store.published[0]["minute_ms"] == 60_000 - assert store.cleaned == [(0, 8), (60_000, 8), (60_000, 8)] - assert "reason=invalid_identity count=1" in timer_logs.text - assert malformed_id not in timer_logs.text - - -@pytest.fixture -def timer_logs(caplog): - from openviking.scheduling.expiry.metrics import logger - - logger.addHandler(caplog.handler) - try: - yield caplog - finally: - logger.removeHandler(caplog.handler) - - -class Runner: - def __init__(self, *, stop_on_shard=0, error=None): - self.stop_on_shard = stop_on_shard - self.error = error - self.calls = [] - - async def run_if_acquired(self, role, shard_id, process, stop): - self.calls.append((role, shard_id, process)) - if shard_id == self.stop_on_shard: - stop.set() - if self.error is not None and shard_id == 0: - raise self.error - return False - - -@pytest.mark.asyncio -async def test_scheduler_worker_delegates_scheduler_role_and_stops_mid_shard_round(): - shard_runner = Runner(stop_on_shard=1) - timer_config = ExpiryConfig.model_construct(shard_count=3) - scheduler = ExpiryPublisher(Store(), shard_runner, timer_config, ExpiryMetrics()) - await asyncio.wait_for(scheduler.run(asyncio.Event()), timeout=2.0) - assert [(role, shard_id) for role, shard_id, _ in shard_runner.calls] == [ - (OwnerRole.SCHEDULER, 0), - (OwnerRole.SCHEDULER, 1), - ] - assert all(process == scheduler.process_shard for _, _, process in shard_runner.calls) - - -@pytest.mark.asyncio -async def test_scheduler_worker_does_not_enter_runner_when_already_stopped(): - shard_runner = Runner() - scheduler = ExpiryPublisher(Store(), shard_runner, config(), ExpiryMetrics()) - stop = asyncio.Event() - stop.set() - await scheduler.run(stop) - assert shard_runner.calls == [] - - -@pytest.mark.asyncio -async def test_scheduler_worker_propagates_cancellation_instead_of_retrying(): - shard_runner = Runner(error=asyncio.CancelledError()) - scheduler = ExpiryPublisher(Store(), shard_runner, config(), ExpiryMetrics()) - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(scheduler.run(asyncio.Event()), timeout=2.0) - assert len(shard_runner.calls) == 1 - - -@pytest.mark.asyncio -async def test_scheduler_worker_propagates_pending_work_instead_of_acquiring_next_shard(): - shard_runner = Runner( - stop_on_shard=1, - error=StoreOperationError("shard_not_stopped", "scheduler still running"), - ) - timer_config = ExpiryConfig.model_construct(shard_count=3) - scheduler = ExpiryPublisher(Store(), shard_runner, timer_config, ExpiryMetrics()) - with pytest.raises(StoreOperationError) as error: - await asyncio.wait_for(scheduler.run(asyncio.Event()), timeout=2.0) - assert error.value.code == "shard_not_stopped" - assert len(shard_runner.calls) == 1 - - -@pytest.mark.asyncio -async def test_scheduler_worker_continues_shard_round_after_processor_error(): - shard_runner = Runner(stop_on_shard=1, error=RuntimeError("scan failed")) - timer_config = ExpiryConfig.model_construct(shard_count=3) - scheduler = ExpiryPublisher(Store(), shard_runner, timer_config, ExpiryMetrics()) - await asyncio.wait_for(scheduler.run(asyncio.Event()), timeout=2.0) - assert [shard_id for _, shard_id, _ in shard_runner.calls] == [0, 1] diff --git a/tests/scheduling/expiry/test_redis_loop.py b/tests/scheduling/expiry/test_redis_loop.py deleted file mode 100644 index ded5bedf84..0000000000 --- a/tests/scheduling/expiry/test_redis_loop.py +++ /dev/null @@ -1,382 +0,0 @@ -import asyncio -import threading - -import pytest - -from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig -from openviking.scheduling.expiry.models import StoreOperationError -from openviking.scheduling.expiry.runtime import ExpiryRuntime, RedisLoopExecutor -from openviking.scheduling.expiry.service import ExpiryRuntimeState - - -class Runner: - has_pending = False - - async def wait_pending(self, timeout): - return True - - -class Store: - def __init__(self, finished): - self.finished = finished - self.close_calls = 0 - self.closed_while_running = False - - async def close(self): - self.close_calls += 1 - self.closed_while_running |= not self.finished.is_set() - - -def runtime(io, store): - result = object.__new__(ExpiryRuntime) - result._config = ExpiryConfig.model_construct( - shutdown_timeout_seconds=0.03, - redis=ExpiryRedisConfig.model_construct(request_budget_seconds=0.03), - ) - result._state = ExpiryRuntimeState(storage_ready=True, accepting_calls=True) - result._stop = asyncio.Event() - result._tasks = {} - result._storage_started = True - result._storage_closed = False - result._storage_owned = True - result._store_open_attempted = True - result._storage_start_task = None - result._store = store - result._io = io - result._shard_runner = Runner() - return result - - -@pytest.mark.asyncio -async def test_runtime_preserves_dependencies_until_cancelled_remote_operation_really_exits(): - io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=10.0) - await io.start() - thread = io._thread - loop = io._loop - started = threading.Event() - cancelled = threading.Event() - release = threading.Event() - finished = threading.Event() - - async def operation(): - started.set() - try: - while not release.is_set(): - try: - await asyncio.sleep(0.005) - except asyncio.CancelledError: - cancelled.set() - finally: - finished.set() - - store = Store(finished) - timer_runtime = runtime(io, store) - request = asyncio.create_task(io.run(operation)) - close_task = None - try: - assert await asyncio.to_thread(started.wait, 2.0) - request.cancel() - assert await asyncio.to_thread(cancelled.wait, 2.0) - close_task = asyncio.create_task(timer_runtime.close()) - done, _ = await asyncio.wait({close_task}, timeout=0.5) - assert close_task in done, "Shutdown must remain bounded even across the Redis loop" - with pytest.raises(StoreOperationError) as error: - close_task.result() - assert error.value.code == "shard_not_stopped" - assert store.close_calls == 0 - assert not store.closed_while_running - assert timer_runtime._storage_started - assert timer_runtime._state.storage_ready - assert io._thread is thread - assert io._loop is loop - assert thread.is_alive() - assert not finished.is_set() - release.set() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(asyncio.shield(request), timeout=2.0) - assert await asyncio.to_thread(finished.wait, 2.0) - await asyncio.wait_for(timer_runtime.close(), timeout=2.0) - assert store.close_calls == 1 - assert not store.closed_while_running - assert not thread.is_alive() - assert io._thread is None - assert not timer_runtime._storage_started - finally: - release.set() - await asyncio.wait_for(asyncio.gather(request, return_exceptions=True), timeout=2.0) - if close_task is not None: - await asyncio.wait_for(asyncio.gather(close_task, return_exceptions=True), timeout=2.0) - # Also join the captured thread: the old implementation incorrectly - # discarded the only executor reference after its timed join expired. - if thread.is_alive(): - assert await asyncio.to_thread(finished.wait, 2.0) - await io.close(2.0) - await asyncio.to_thread(thread.join, 2.0) - if io._thread is not None: - await io.close(2.0) - - -@pytest.mark.asyncio -async def test_executor_cancelled_wrapper_stays_pending_until_remote_task_exits(): - io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=10.0) - await io.start() - started = threading.Event() - cancelled = threading.Event() - release = threading.Event() - finished = threading.Event() - - async def operation(): - started.set() - try: - while not release.is_set(): - try: - await asyncio.sleep(0.005) - except asyncio.CancelledError: - cancelled.set() - finally: - finished.set() - - request = asyncio.create_task(io.run(operation)) - try: - assert await asyncio.to_thread(started.wait, 2.0) - request.cancel() - assert await asyncio.to_thread(cancelled.wait, 2.0) - done, _ = await asyncio.wait({request}, timeout=0.03) - assert request not in done, "A cancelled bridge is not proof of remote task termination" - # Repeated cancellation during cleanup must not orphan remote work. - request.cancel() - done, _ = await asyncio.wait({request}, timeout=0.03) - assert request not in done - release.set() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(asyncio.shield(request), timeout=2.0) - assert finished.is_set() - finally: - release.set() - await asyncio.wait_for(asyncio.gather(request, return_exceptions=True), timeout=2.0) - await io.close(2.0) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("raises", [False, True]) -async def test_executor_propagates_remote_result_or_exception_and_closes_cleanly(raises): - io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) - await io.start() - thread = io._thread - - async def operation(): - if raises: - raise ValueError("remote error") - return 42 - - try: - if raises: - with pytest.raises(ValueError, match="remote error"): - await io.run(operation) - else: - assert await io.run(operation) == 42 - await io.quiesce(0.1) - with pytest.raises(StoreOperationError) as error: - await io.run(operation) - assert error.value.code == "not_started" - finally: - await io.close(2.0) - assert not thread.is_alive() - - -@pytest.mark.asyncio -async def test_quiesce_atomically_rejects_request_before_registration(): - io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) - await io.start() - submit_entered = threading.Event() - allow_submit = threading.Event() - operation_started = threading.Event() - original_submit = io._submit - - def delayed_submit(operation, *, cleanup=False): - submit_entered.set() - allow_submit.wait() - return original_submit(operation, cleanup=cleanup) - - io._submit = delayed_submit - - async def operation(): - operation_started.set() - - def run_from_thread(): - try: - asyncio.run(io.run(operation)) - except BaseException as exc: - return exc - return None - - request = asyncio.create_task(asyncio.to_thread(run_from_thread)) - try: - assert await asyncio.to_thread(submit_entered.wait, 2.0) - await io.quiesce(0.1) - allow_submit.set() - result = await asyncio.wait_for(request, timeout=2.0) - assert isinstance(result, StoreOperationError) - assert result.code == "not_started" - assert not operation_started.is_set() - finally: - allow_submit.set() - await asyncio.gather(request, return_exceptions=True) - await io.close(2.0) - - -@pytest.mark.asyncio -async def test_cleanup_runs_nested_io_without_reopening_admission_or_deadlocking_gate(): - io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) - await io.start() - observed = [] - - async def operation(): - observed.append(asyncio.get_running_loop()) - return 42 - - async def cleanup(): - return await io.run(operation) - - try: - await io.quiesce(0.1) - assert await io.run_cleanup(cleanup, 1.0) == 42 - assert observed == [io._loop] - with pytest.raises(StoreOperationError) as error: - await io.run(operation) - assert error.value.code == "not_started" - finally: - await io.close(2.0) - - -@pytest.mark.asyncio -async def test_resistant_storage_cleanup_is_bounded_and_retains_thread_for_retry(): - io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) - await io.start() - thread = io._thread - started = threading.Event() - cancelled = threading.Event() - release = threading.Event() - finished = threading.Event() - - class ResistantStore(Store): - async def close(self): - self.close_calls += 1 - if finished.is_set(): - return - started.set() - try: - while not release.is_set(): - try: - await asyncio.sleep(0.005) - except asyncio.CancelledError: - cancelled.set() - finally: - finished.set() - - store = ResistantStore(finished) - timer_runtime = runtime(io, store) - close_task = asyncio.create_task(timer_runtime.close()) - try: - assert await asyncio.to_thread(started.wait, 2.0) - assert await asyncio.to_thread(cancelled.wait, 2.0) - done, _ = await asyncio.wait({close_task}, timeout=0.5) - assert close_task in done - with pytest.raises(StoreOperationError) as error: - close_task.result() - assert error.value.code == "shard_not_stopped" - assert timer_runtime._storage_started - assert timer_runtime._state.storage_ready - assert io._thread is thread - assert thread.is_alive() - assert not finished.is_set() - release.set() - assert await asyncio.to_thread(finished.wait, 2.0) - await asyncio.wait_for(timer_runtime.close(), timeout=2.0) - assert not thread.is_alive() - assert not timer_runtime._storage_started - finally: - release.set() - await asyncio.wait_for(asyncio.gather(close_task, return_exceptions=True), timeout=2.0) - assert await asyncio.to_thread(finished.wait, 2.0) - await io.close(2.0) - - -@pytest.mark.asyncio -async def test_request_timeout_never_reports_success_when_operation_suppresses_cancel(): - io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=0.03) - await io.start() - cancelled = threading.Event() - release = threading.Event() - finished = threading.Event() - - async def operation(): - try: - while not release.is_set(): - try: - await asyncio.sleep(0.005) - except asyncio.CancelledError: - cancelled.set() - return "late success" - finally: - finished.set() - - request = asyncio.create_task(io.run(operation)) - try: - assert await asyncio.to_thread(cancelled.wait, 2.0) - done, _ = await asyncio.wait({request}, timeout=0.03) - assert request not in done - release.set() - with pytest.raises(StoreOperationError) as error: - await asyncio.wait_for(request, timeout=2.0) - assert error.value.code == "redis_timeout" - assert error.value.outcome_unknown - assert finished.is_set() - finally: - release.set() - await asyncio.wait_for(asyncio.gather(request, return_exceptions=True), timeout=2.0) - await io.close(2.0) - - -@pytest.mark.asyncio -async def test_executor_retains_loop_and_thread_if_timed_join_does_not_finish(): - class Loop: - def __init__(self): - self.stops = 0 - - def is_running(self): - return True - - def stop(self): - self.stops += 1 - - def call_soon_threadsafe(self, callback): - callback() - - class Thread: - def __init__(self): - self.alive = True - self.budgets = [] - - def join(self, timeout): - self.budgets.append(timeout) - - def is_alive(self): - return self.alive - - io = RedisLoopExecutor(max_inflight=1, request_budget_seconds=1.0) - loop, thread = Loop(), Thread() - io._loop, io._thread, io._accepting = loop, thread, True - with pytest.raises(StoreOperationError) as error: - await io.close(0.01) - assert error.value.code == "shard_not_stopped" - assert io._loop is loop - assert io._thread is thread - assert io._accepting is False - thread.alive = False - await io.close(0.01) - assert io._loop is None - assert io._thread is None - assert len(thread.budgets) == 2 - assert all(0.0 <= budget <= 0.01 for budget in thread.budgets) - assert loop.stops == 1 diff --git a/tests/scheduling/expiry/test_redis_store.py b/tests/scheduling/expiry/test_redis_store.py deleted file mode 100644 index 947428b23c..0000000000 --- a/tests/scheduling/expiry/test_redis_store.py +++ /dev/null @@ -1,680 +0,0 @@ -import asyncio -from unittest.mock import AsyncMock, Mock - -import pytest - -from openviking.scheduling.expiry.config import ExpiryRedisConfig -from openviking.scheduling.expiry.keys import ExpiryKeys -from openviking.scheduling.expiry.metrics import ExpiryMetrics -from openviking.scheduling.expiry.models import ( - BucketItem, - BucketRef, - Claim, - OwnerRole, - OwnerToken, - RegistryCursor, - SourceCursor, - SourceItem, - StoreOperationError, - encode_claim_score, - encode_ready_claim_score, -) -from openviking.scheduling.expiry.redis_store import RedisExpiryStore - - -class InlineExecutor: - async def run(self, operation): - return await operation() - - -class FlakyClient: - def __init__(self, failures): - self.failures = list(failures) - self.calls = 0 - - async def get(self, _key): - self.calls += 1 - if self.failures: - raise self.failures.pop(0) - return "value" - - -class ScriptClient: - def __init__(self): - self.calls = [] - - async def evalsha(self, sha, key_count, *args): - self.calls.append(("evalsha", sha, key_count, args)) - error_type = type("NoScriptError", (RuntimeError,), {}) - raise error_type("No matching script. Please use EVAL.") - - async def eval(self, source, key_count, *args): - self.calls.append(("eval", source, key_count, args)) - return "loaded" - - -class ClusterTimeClient: - def __init__(self): - self.node = object() - self.key = None - self.target = None - - def get_node_from_key(self, key): - self.key = key - return self.node - - async def time(self, *, target_nodes): - self.target = target_nodes - return (123, 456_789) - - -def store(client): - result = RedisExpiryStore( - ExpiryRedisConfig(), ExpiryKeys("resource"), InlineExecutor(), ExpiryMetrics() - ) - result._client = client - return result - - -@pytest.fixture -async def cluster_store(monkeypatch): - cluster = pytest.importorskip("redis.asyncio.cluster") - timer_store = store(None) - timer_store._config = ExpiryRedisConfig(mode="cluster", endpoints=["127.0.0.1:6379"]) - ping = AsyncMock(return_value=True) - monkeypatch.setattr(cluster.RedisCluster, "ping", ping) - await timer_store.open() - ping.assert_awaited_once() - try: - yield timer_store - finally: - await timer_store.close() - - -def configure_cluster_node(monkeypatch, client, key, execute): - cluster = pytest.importorskip("redis.asyncio.cluster") - node = cluster.ClusterNode("127.0.0.1", 6379, server_type="primary") - manager = client.nodes_manager - manager.default_node = node - manager.nodes_cache = {node.name: node} - manager.slots_cache = {client.keyslot(key): [node]} - - async def initialize(_client): - # Replace topology discovery only; execute_command and its retry/routing - # loops remain the real redis-py implementation, including after close. - _client._initialize = False - return _client - - monkeypatch.setattr(cluster.RedisCluster, "initialize", initialize) - parser = Mock(spec=type(client.commands_parser)) - parser.get_keys = AsyncMock(return_value=[key]) - client.commands_parser = parser - monkeypatch.setattr(cluster.ClusterNode, "execute_command", execute) - return node - - -@pytest.mark.asyncio -async def test_cluster_retry_settings_are_local_to_expiry_client(cluster_store): - from redis.asyncio import Redis - from redis.asyncio.cluster import RedisCluster - - client = cluster_store._client - other_cluster = RedisCluster(host="127.0.0.1", port=6379) - other_standalone = Redis(host="127.0.0.1", port=6379) - try: - assert client.cluster_error_retry_attempts == 0 - assert client.connection_error_retry_attempts == 0 - assert client.get_retry() is None - assert all( - node.connection_kwargs["retry"] is None - for node in client.nodes_manager.startup_nodes.values() - ) - assert other_cluster.cluster_error_retry_attempts == 3 - assert other_cluster.connection_error_retry_attempts == 3 - assert other_cluster.get_retry() is not None - assert other_standalone.connection_pool.connection_kwargs["retry"] is None - finally: - await other_cluster.aclose() - await other_standalone.aclose() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("command", ["DEL", "UNLINK"]) -async def test_cluster_single_key_delete_bypasses_pipeline( - monkeypatch, cluster_store, command -): - key = f"single-key-{command.lower()}" - calls = [] - - async def execute(node, *args, **kwargs): - calls.append((node.name, args, kwargs)) - return 1 - - node = configure_cluster_node(monkeypatch, cluster_store._client, key, execute) - - result = await cluster_store._single_key_command(command, key, retry="idempotent") - - assert result == 1 - assert calls == [(node.name, (command, key), {})] - - -@pytest.mark.asyncio -async def test_standalone_client_constructor_is_unchanged(monkeypatch): - redis_async = pytest.importorskip("redis.asyncio") - client = Mock(spec=redis_async.Redis) - client.ping = AsyncMock(return_value=True) - client.aclose = AsyncMock() - factory = Mock(return_value=client) - monkeypatch.setattr(redis_async.Redis, "from_url", factory) - timer_store = store(None) - timer_store._config = ExpiryRedisConfig(endpoints=["127.0.0.1:6379"]) - try: - await timer_store.open() - factory.assert_called_once_with( - "redis://127.0.0.1:6379", - username=None, - password=None, - decode_responses=True, - socket_connect_timeout=1.0, - socket_timeout=2.0, - max_connections=64, - ) - client.ping.assert_awaited_once() - finally: - await timer_store.close() - client.aclose.assert_awaited_once() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("operation", ["claim", "owner"]) -@pytest.mark.parametrize("error_name", ["ConnectionError", "TimeoutError"]) -async def test_cluster_uncertain_write_is_not_replayed( - monkeypatch, cluster_store, operation, error_name -): - from redis import exceptions - - bucket = BucketRef("session", 3, 60_000) - key = ( - cluster_store._keys.bucket(bucket) - if operation == "claim" - else cluster_store._keys.owner(OwnerRole.CALLBACK, 3, item_type="session") - ) - expected_score = encode_claim_score(1, 123) - state = {"score": expected_score, "owner": None, "calls": 0} - - async def execute(_node, *args, **kwargs): - state["calls"] += 1 - if operation == "claim": - assert args[0] == "EVALSHA" - if state["score"] != int(args[-2]): - return 0 - state["score"] = int(args[-1]) - else: - assert args[0] == "SET" - if state["owner"] is not None: - return False - state["owner"] = args[2] - # The write takes effect before the response is lost. A second call - # would return 0/False and hide the successful first execution. - raise getattr(exceptions, error_name)("response lost after write") - - configure_cluster_node(monkeypatch, cluster_store._client, key, execute) - with pytest.raises(StoreOperationError, match="response lost") as raised: - if operation == "claim": - await cluster_store.claim( - bucket, - BucketItem("item", expected_score), - attempt=1, - started_at_seconds=200, - ) - else: - await cluster_store.try_acquire_owner( - OwnerRole.CALLBACK, 3, "token", 120, item_type="session" - ) - assert raised.value.outcome_unknown is True - assert state["calls"] == 1 - if operation == "claim": - assert state["score"] == encode_claim_score(1, 200) - else: - assert state["owner"] == "token" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("operation", ["read", "idempotent"]) -async def test_cluster_safe_operations_still_retry_in_store(monkeypatch, cluster_store, operation): - from redis.exceptions import ConnectionError - - calls = [] - bucket = BucketRef("session", 3, 60_000) - key = cluster_store._keys.bucket(bucket) - - async def execute(_node, *args, **kwargs): - calls.append(args) - if len(calls) == 1: - raise ConnectionError("temporary") - return "value" if operation == "read" else 1 - - configure_cluster_node(monkeypatch, cluster_store._client, key, execute) - if operation == "read": - assert await cluster_store._command("get", key, retry="read") == "value" - else: - claim = Claim(bucket, "item", encode_claim_score(1, 123)) - assert await cluster_store.ack(claim) is True - assert len(calls) == 2 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("redirect", ["MOVED", "ASK"]) -async def test_cluster_redirections_still_work_without_network_replay( - monkeypatch, cluster_store, redirect -): - from redis.asyncio.cluster import ClusterNode - from redis.exceptions import AskError, MovedError - - client = cluster_store._client - key = cluster_store._keys.bucket(BucketRef("session", 3, 60_000)) - destination = ClusterNode("127.0.0.1", 6380, server_type="primary") - calls = [] - - async def execute(node, *args, **kwargs): - calls.append((node.name, args)) - if node.port == 6379: - error = MovedError if redirect == "MOVED" else AskError - raise error(f"{client.keyslot(key)} 127.0.0.1:6380") - return "OK" if args[0] == "ASKING" else 1 - - source = configure_cluster_node(monkeypatch, client, key, execute) - client.nodes_manager.nodes_cache[destination.name] = destination - claim = await cluster_store.claim( - BucketRef("session", 3, 60_000), - BucketItem("item", encode_claim_score(1, 123)), - attempt=1, - started_at_seconds=200, - ) - assert claim is not None - assert claim.claim_score == encode_claim_score(1, 200) - assert [name for name, _ in calls] == ( - [source.name, destination.name] - if redirect == "MOVED" - else [source.name, destination.name, destination.name] - ) - assert [args[0] for _, args in calls] == ( - ["EVALSHA", "EVALSHA"] if redirect == "MOVED" else ["EVALSHA", "ASKING", "EVALSHA"] - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("error", [RuntimeError("close failed"), asyncio.CancelledError()]) -async def test_client_reference_is_retained_until_close_succeeds(error): - class Client: - def __init__(self): - self.aclose = AsyncMock(side_effect=[error, None]) - - client = Client() - timer_store = store(client) - with pytest.raises(type(error)): - await timer_store.close() - assert timer_store._client is client - await timer_store.close() - assert timer_store._client is None - assert client.aclose.await_count == 2 - - -@pytest.mark.asyncio -async def test_safe_command_retries_transient_redis_failure(): - client = FlakyClient([ConnectionError("temporary")]) - - assert await store(client)._command("get", "key", retry="read") == "value" - assert client.calls == 2 - - -@pytest.mark.asyncio -async def test_uncertain_write_is_left_for_caller_to_resolve(): - client = FlakyClient([ConnectionError("temporary")]) - - with pytest.raises(StoreOperationError) as raised: - await store(client)._command("get", "key", retry="resolve_by_caller") - - assert client.calls == 1 - assert raised.value.outcome_unknown is True - - -@pytest.mark.asyncio -async def test_protocol_error_is_not_retried(): - client = FlakyClient([RuntimeError("wrong_key_type")]) - - with pytest.raises(StoreOperationError) as raised: - await store(client)._command("get", "key", retry="idempotent") - - assert client.calls == 1 - assert raised.value.code == "wrong_key_type" - - -@pytest.mark.asyncio -async def test_evalsha_falls_back_to_eval_on_cache_miss(): - client = ScriptClient() - timer_store = store(client) - - result = await timer_store._eval("upsert_resource", ["source"], ["item", 1], retry="read") - - assert result == "loaded" - assert [call[0] for call in client.calls] == ["evalsha", "eval"] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("delete_metadata", [True, False]) -async def test_unregister_optionally_deletes_metadata(monkeypatch, delete_metadata): - timer_store = store(FlakyClient([])) - command = AsyncMock() - single_key_command = AsyncMock() - monkeypatch.setattr(timer_store, "_command", command) - monkeypatch.setattr(timer_store, "_single_key_command", single_key_command) - - await timer_store.unregister_resource(3, "item", delete_metadata=delete_metadata) - - assert [call.args for call in command.await_args_list] == [ - ("zrem", timer_store._keys.source(3), "item") - ] - if delete_metadata: - single_key_command.assert_awaited_once_with( - "DEL", - timer_store._keys.metadata("item"), - retry="idempotent", - ) - else: - single_key_command.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_bucket_reset_uses_single_key_cluster_commands(monkeypatch): - timer_store = store(FlakyClient([])) - monkeypatch.setattr(timer_store, "assert_owner", AsyncMock()) - monkeypatch.setattr(timer_store, "read_progress", AsyncMock(return_value=0)) - single_key_command = AsyncMock() - monkeypatch.setattr(timer_store, "_single_key_command", single_key_command) - - await timer_store.reset_unpublished_minute( - OwnerToken(OwnerRole.SCHEDULER, 3, "owner", 999), - minute_ms=60_000, - expected_progress_ms=0, - item_types=("session", "watch"), - ) - - assert [call.args[:2] for call in single_key_command.await_args_list] == [ - ("UNLINK", timer_store._keys.bucket(BucketRef("session", 3, 60_000))), - ("UNLINK", timer_store._keys.bucket(BucketRef("watch", 3, 60_000))), - ] - - -@pytest.mark.asyncio -async def test_source_cleanup_page_uses_single_key_atomic_script(monkeypatch): - timer_store = store(FlakyClient([])) - evaluate = AsyncMock(return_value=7) - monkeypatch.setattr(timer_store, "_eval", evaluate) - - removed = await timer_store.cleanup_source_page(3, 60_000, limit=128) - - assert removed == 7 - evaluate.assert_awaited_once_with( - "cleanup_source_page", - [timer_store._keys.source(3)], - [60_000, 128], - retry="idempotent", - ) - - -@pytest.mark.asyncio -async def test_registry_page_uses_score_cursor(monkeypatch): - timer_store = store(FlakyClient([])) - evaluate = AsyncMock(return_value='{"items":[["session:3:60000","70000"]],"next":null}') - monkeypatch.setattr(timer_store, "_eval", evaluate) - - page = await timer_store.read_registry_page( - 3, - after=RegistryCursor(65_000, "session:3:0"), - limit=128, - ) - - assert page.buckets == (BucketRef("session", 3, 60_000),) - evaluate.assert_awaited_once_with( - "read_zset_page", - [timer_store._keys.registry(3)], - [-1, "", 65_000, "session:3:0", 128], - retry="read", - ) - - -@pytest.mark.asyncio -async def test_waiting_bucket_can_defer_shared_registration(monkeypatch): - timer_store = store(FlakyClient([])) - evaluate = AsyncMock(return_value=1) - monkeypatch.setattr(timer_store, "_eval", evaluate) - bucket = BucketRef("session", 3, 60_000) - owner = OwnerToken( - OwnerRole.CALLBACK, - 3, - "owner", - 999, - "session", - ) - - assert ( - await timer_store.defer_registration( - owner, - bucket, - next_poll_at_ms=180_000, - ) - is True - ) - - evaluate.assert_awaited_once_with( - "defer_registration", - [ - timer_store._keys.registry(3), - timer_store._keys.owner( - OwnerRole.CALLBACK, - 3, - item_type="session", - ), - ], - ["owner", timer_store._keys.registry_member(bucket), 180_000], - retry="idempotent", - ) - - -@pytest.mark.asyncio -async def test_redis_time_is_returned_in_milliseconds(monkeypatch): - timer_store = store(FlakyClient([])) - command = AsyncMock(return_value=(123, 456_789)) - monkeypatch.setattr(timer_store, "_command", command) - - assert await timer_store.read_time_ms() == 123_456 - command.assert_awaited_once_with("time") - - -@pytest.mark.asyncio -async def test_cluster_time_is_routed_to_key_primary(): - client = ClusterTimeClient() - timer_store = store(client) - timer_store._config = ExpiryRedisConfig.model_construct(mode="cluster") - - assert await timer_store._read_time_ms_for_key("source-key") == 123_456 - assert client.key == "source-key" - assert client.target is client.node - - -@pytest.mark.asyncio -async def test_source_first_page_uses_native_score_range(monkeypatch): - timer_store = store(FlakyClient([])) - command = AsyncMock(return_value=[("a", 100.0), ("b", 100.0), ("c", 200.0)]) - monkeypatch.setattr(timer_store, "_command", command) - - page = await timer_store.read_source_page( - 3, - lower_exclusive_ms=0, - upper_inclusive_ms=1_000, - after=None, - limit=2, - ) - - assert page.items == (SourceItem("a", 100), SourceItem("b", 100)) - assert page.next_cursor == SourceCursor(100, "b") - command.assert_awaited_once_with( - "zrangebyscore", - timer_store._keys.source(3), - "(0", - 1_000, - start=0, - num=3, - withscores=True, - ) - - -@pytest.mark.asyncio -async def test_source_continuation_uses_stable_cursor_script(monkeypatch): - timer_store = store(FlakyClient([])) - evaluate = AsyncMock(return_value='{"items":[["c","200"]],"next":["200","c"]}') - monkeypatch.setattr(timer_store, "_eval", evaluate) - - page = await timer_store.read_source_page( - 3, - lower_exclusive_ms=0, - upper_inclusive_ms=1_000, - after=SourceCursor(100, "b"), - limit=2, - ) - - assert page.items == (SourceItem("c", 200),) - assert page.next_cursor == SourceCursor(200, "c") - evaluate.assert_awaited_once_with( - "read_zset_page", - [timer_store._keys.source(3)], - [0, 1_000, 100, "b", 2], - retry="read", - ) - - -@pytest.mark.asyncio -async def test_bucket_page_reads_one_attempt_layer_from_offset_zero(monkeypatch): - timer_store = store(FlakyClient([])) - command = AsyncMock(return_value=[("item", float(encode_claim_score(2, 123)))]) - monkeypatch.setattr(timer_store, "_command", command) - bucket = BucketRef("session", 3, 60_000) - - page = await timer_store.read_bucket_page( - bucket, - attempt=2, - limit=128, - ) - - assert page.items == (BucketItem("item", encode_claim_score(2, 123)),) - lower = encode_claim_score(2, 1) - upper = encode_claim_score(3, 1) - 2 - command.assert_awaited_once_with( - "zrangebyscore", - timer_store._keys.bucket(bucket), - lower, - upper, - start=0, - num=128, - withscores=True, - ) - - -@pytest.mark.asyncio -async def test_claim_and_retry_use_score_compare_and_set(monkeypatch): - timer_store = store(FlakyClient([])) - evaluate = AsyncMock(side_effect=[1, 1]) - monkeypatch.setattr(timer_store, "_eval", evaluate) - bucket = BucketRef("session", 3, 60_000) - old_score = encode_claim_score(1, 123) - - claim = await timer_store.claim( - bucket, - BucketItem("item", old_score), - attempt=1, - started_at_seconds=200, - ) - - assert claim == Claim(bucket, "item", encode_claim_score(1, 200)) - evaluate.assert_awaited_once_with( - "cas_zset_member", - [timer_store._keys.bucket(bucket)], - ["set", "item", old_score, encode_claim_score(1, 200)], - retry="resolve_by_caller", - ) - - evaluate.reset_mock() - assert await timer_store.advance_retry(claim) is True - evaluate.assert_awaited_once_with( - "cas_zset_member", - [timer_store._keys.bucket(bucket)], - ["set", "item", claim.claim_score, encode_ready_claim_score(2)], - retry="resolve_by_caller", - ) - - -@pytest.mark.asyncio -async def test_ack_uses_score_compare_and_delete(monkeypatch): - timer_store = store(FlakyClient([])) - evaluate = AsyncMock(return_value=1) - monkeypatch.setattr(timer_store, "_eval", evaluate) - claim = Claim(BucketRef("session", 3, 60_000), "item", encode_claim_score(1, 123)) - - assert await timer_store.ack(claim) is True - evaluate.assert_awaited_once_with( - "cas_zset_member", - [timer_store._keys.bucket(claim.bucket)], - ["delete", "item", claim.claim_score, ""], - retry="idempotent", - ) - - -@pytest.mark.asyncio -async def test_owner_acquire_uses_native_set_nx_px(monkeypatch): - timer_store = store(FlakyClient([])) - command = AsyncMock(return_value=True) - monkeypatch.setattr(timer_store, "_command", command) - - owner = await timer_store.try_acquire_owner( - OwnerRole.CALLBACK, - 3, - "token", - 120, - item_type="session", - ) - - assert owner is not None - assert owner.token == "token" - command.assert_awaited_once_with( - "set", - timer_store._keys.owner(OwnerRole.CALLBACK, 3, item_type="session"), - "token", - nx=True, - px=120_000, - retry="resolve_by_caller", - ) - - -@pytest.mark.asyncio -async def test_owner_renew_and_release_share_compare_script(monkeypatch): - timer_store = store(FlakyClient([])) - evaluate = AsyncMock(side_effect=[1, 1]) - monkeypatch.setattr(timer_store, "_eval", evaluate) - owner = OwnerToken(OwnerRole.SCHEDULER, 3, "token", 1) - key = timer_store._keys.owner(OwnerRole.SCHEDULER, 3) - - assert await timer_store.renew_owner(owner, 120) is not None - assert await timer_store.release_owner(owner) is True - assert evaluate.await_args_list[0].args == ( - "compare_owner", - [key], - ["renew", "token", 120_000], - ) - assert evaluate.await_args_list[0].kwargs == {"retry": "resolve_by_caller"} - assert evaluate.await_args_list[1].args == ( - "compare_owner", - [key], - ["release", "token", ""], - ) - assert evaluate.await_args_list[1].kwargs == {"retry": "idempotent"} diff --git a/tests/scheduling/expiry/test_runtime.py b/tests/scheduling/expiry/test_runtime.py deleted file mode 100644 index 27b746a4a0..0000000000 --- a/tests/scheduling/expiry/test_runtime.py +++ /dev/null @@ -1,608 +0,0 @@ -import asyncio -import threading - -import pytest - -from openviking.scheduling.expiry import runtime as runtime_module -from openviking.scheduling.expiry.config import ExpiryConfig, ExpiryRedisConfig -from openviking.scheduling.expiry.models import StoreOperationError -from openviking.scheduling.expiry.runtime import ExpiryRuntime, ExpiryRuntimeRoles -from openviking.scheduling.expiry.service import ExpiryRuntimeState -from openviking.service.core import OpenVikingService - - -class Store: - def __init__(self): - self.close_calls = 0 - - async def close(self): - self.close_calls += 1 - - -class IO: - def __init__(self): - self.close_budgets = [] - - async def quiesce(self, timeout_seconds): - pass - - async def run_cleanup(self, operation, timeout_seconds): - return await operation() - - async def close(self, timeout_seconds): - self.close_budgets.append(timeout_seconds) - - -class Runner: - def __init__(self, *, pending=False): - self.settled = asyncio.Event() - if not pending: - self.settled.set() - self.wait_budgets = [] - - @property - def has_pending(self): - return not self.settled.is_set() - - async def wait_pending(self, timeout): - self.wait_budgets.append(timeout) - if not self.has_pending: - return True - waiter = asyncio.create_task(self.settled.wait()) - try: - finished, _ = await asyncio.wait({waiter}, timeout=timeout) - return waiter in finished - finally: - waiter.cancel() - await asyncio.gather(waiter, return_exceptions=True) - - -def runtime(*, pending=False): - # Exercise the actual shutdown helpers without constructing Redis clients, - # starting an I/O thread, or replacing the helpers under test. - result = object.__new__(ExpiryRuntime) - result._config = ExpiryConfig.model_construct( - shutdown_timeout_seconds=0.02, - redis=ExpiryRedisConfig.model_construct(request_budget_seconds=0.03), - ) - result._state = ExpiryRuntimeState() - result._state.accepting_calls = True - result._state.storage_ready = True - result._stop = asyncio.Event() - result._tasks = {} - result._storage_started = True - result._storage_closed = False - result._storage_owned = True - result._store_open_attempted = True - result._storage_start_task = None - result._store = Store() - result._io = IO() - result._shard_runner = Runner(pending=pending) - return result - - -@pytest.mark.asyncio -@pytest.mark.parametrize("stage", ["io", "open", "namespace"]) -async def test_cancelled_storage_start_closes_partial_resources(stage): - timer_runtime = ExpiryRuntime( - ExpiryConfig(service_name="startup-test"), roles=ExpiryRuntimeRoles() - ) - entered = asyncio.Event() - original_start = timer_runtime._io.start - - class StartupStore(Store): - async def open(self): - if stage == "open": - entered.set() - await asyncio.Event().wait() - - async def ensure_namespace(self, spec): - if stage == "namespace": - entered.set() - await asyncio.Event().wait() - - async def io_start(): - await original_start() - if stage == "io": - entered.set() - await asyncio.Event().wait() - - timer_runtime._store = StartupStore() - timer_runtime._io.start = io_start - start_task = asyncio.create_task(timer_runtime.start_storage()) - thread = None - try: - await asyncio.wait_for(entered.wait(), timeout=2.0) - thread = timer_runtime._io._thread - start_task.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(start_task, timeout=2.0) - assert not timer_runtime._storage_started - assert not timer_runtime._storage_owned - assert timer_runtime._storage_start_task is None - assert not timer_runtime._state.storage_ready - assert thread is not None and not thread.is_alive() - assert timer_runtime._store.close_calls == (0 if stage == "io" else 1) - await timer_runtime.close() - assert timer_runtime._store.close_calls == (0 if stage == "io" else 1) - finally: - await asyncio.gather(start_task, return_exceptions=True) - await timer_runtime.close() - - -@pytest.mark.asyncio -async def test_cancelled_start_before_io_loop_is_ready_can_retry_close(): - timer_runtime = ExpiryRuntime( - ExpiryConfig(service_name="startup-test", shutdown_timeout_seconds=0.03), - roles=ExpiryRuntimeRoles(), - ) - started = threading.Event() - release = threading.Event() - original_thread_main = timer_runtime._io._thread_main - - def delayed_thread_main(): - started.set() - release.wait() - original_thread_main() - - timer_runtime._io._thread_main = delayed_thread_main - start_task = asyncio.create_task(timer_runtime.start_storage()) - try: - assert await asyncio.to_thread(started.wait, 2.0) - thread = timer_runtime._io._thread - start_task.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(start_task, timeout=2.0) - assert timer_runtime._storage_owned - assert not timer_runtime._storage_started - assert thread is not None and thread.is_alive() - release.set() - await asyncio.wait_for(timer_runtime.close(), timeout=2.0) - assert not timer_runtime._storage_owned - assert not thread.is_alive() - finally: - release.set() - await asyncio.gather(start_task, return_exceptions=True) - await timer_runtime.close() - - -@pytest.mark.asyncio -async def test_resistant_storage_start_is_retained_for_close_retry(): - timer_runtime = ExpiryRuntime( - ExpiryConfig(service_name="startup-test", shutdown_timeout_seconds=0.02), - roles=ExpiryRuntimeRoles(), - ) - entered = asyncio.Event() - release = asyncio.Event() - - class StartupStore(Store): - async def open(self): - entered.set() - while not release.is_set(): - try: - await release.wait() - except asyncio.CancelledError: - pass - - async def ensure_namespace(self, spec): - pass - - timer_runtime._store = StartupStore() - start_task = asyncio.create_task(timer_runtime.start_storage()) - try: - await asyncio.wait_for(entered.wait(), timeout=2.0) - thread = timer_runtime._io._thread - start_task.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(start_task, timeout=2.0) - assert timer_runtime._storage_owned - assert timer_runtime._storage_start_task is not None - assert not timer_runtime._storage_start_task.done() - assert timer_runtime._store.close_calls == 0 - with pytest.raises(StoreOperationError, match="Previous expiry storage startup"): - await timer_runtime.start_storage() - release.set() - await asyncio.wait_for(timer_runtime.close(), timeout=2.0) - assert timer_runtime._store.close_calls == 1 - assert not timer_runtime._storage_owned - assert thread is not None and not thread.is_alive() - finally: - release.set() - await asyncio.gather(start_task, return_exceptions=True) - await timer_runtime.close() - - -@pytest.mark.asyncio -async def test_failed_start_preserves_failure_and_cleans_storage(): - timer_runtime = ExpiryRuntime( - ExpiryConfig(service_name="startup-test"), roles=ExpiryRuntimeRoles() - ) - failure = RuntimeError("namespace failed") - - class StartupStore(Store): - async def open(self): - pass - - async def ensure_namespace(self, spec): - raise failure - - timer_runtime._store = StartupStore() - with pytest.raises(RuntimeError) as error: - await timer_runtime.start_storage() - assert error.value is failure - assert timer_runtime._store.close_calls == 1 - assert not timer_runtime._storage_owned - assert timer_runtime._io._thread is None - - -@pytest.mark.asyncio -async def test_storage_can_start_again_after_partial_start_cleanup(): - timer_runtime = ExpiryRuntime( - ExpiryConfig(service_name="startup-test"), roles=ExpiryRuntimeRoles() - ) - - class StartupStore(Store): - fail_open = True - - async def open(self): - if self.fail_open: - raise RuntimeError("open failed") - - async def ensure_namespace(self, spec): - pass - - timer_runtime._store = StartupStore() - with pytest.raises(RuntimeError, match="open failed"): - await timer_runtime.start_storage() - timer_runtime._store.fail_open = False - try: - await timer_runtime.start_storage() - assert timer_runtime._storage_started - assert timer_runtime._storage_owned - assert timer_runtime._state.storage_ready - await timer_runtime.start_storage() - finally: - await timer_runtime.close() - assert timer_runtime._store.close_calls == 2 - assert not timer_runtime._storage_owned - - -@pytest.mark.asyncio -async def test_cancelled_start_with_failed_cleanup_retains_resources_for_retry(): - timer_runtime = ExpiryRuntime( - ExpiryConfig(service_name="startup-test"), roles=ExpiryRuntimeRoles() - ) - entered = asyncio.Event() - - class StartupStore(Store): - async def open(self): - entered.set() - await asyncio.Event().wait() - - async def close(self): - self.close_calls += 1 - if self.close_calls == 1: - raise RuntimeError("cleanup failed") - - timer_runtime._store = StartupStore() - start_task = asyncio.create_task(timer_runtime.start_storage()) - try: - await asyncio.wait_for(entered.wait(), timeout=2.0) - thread = timer_runtime._io._thread - start_task.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(start_task, timeout=2.0) - assert timer_runtime._storage_owned - assert not timer_runtime._storage_closed - assert thread is not None and thread.is_alive() - await timer_runtime.close() - assert timer_runtime._store.close_calls == 2 - assert not timer_runtime._storage_owned - assert not thread.is_alive() - finally: - await asyncio.gather(start_task, return_exceptions=True) - await timer_runtime.close() - - -def test_consumer_role_requires_reporting(): - with pytest.raises(ValueError, match="requires reporting"): - ExpiryRuntimeRoles(consume=True) - - -@pytest.mark.asyncio -async def test_start_roles_creates_one_callback_task_per_item_type(monkeypatch): - created = [] - - class Consumer: - def __init__(self, store, runner, item_type, binding, config, metrics): - self.item_type = item_type - created.append(item_type) - - async def run(self, stop): - await stop.wait() - - class Registry: - def freeze(self): - return {"session": object(), "watch": object()} - - timer_runtime = object.__new__(ExpiryRuntime) - timer_runtime._storage_started = True - timer_runtime._state = ExpiryRuntimeState() - timer_runtime._config = ExpiryConfig.model_construct() - timer_runtime._roles = ExpiryRuntimeRoles(report=True, consume=True) - timer_runtime._registry = Registry() - timer_runtime._stop = asyncio.Event() - timer_runtime._store = object() - timer_runtime._shard_runner = object() - timer_runtime._metrics = object() - timer_runtime._publisher = None - timer_runtime._consumers = {} - spawned = {} - - def spawn_role(name, operation): - spawned[name] = operation - - timer_runtime._spawn_role = spawn_role - monkeypatch.setattr(runtime_module, "ExpiryConsumer", Consumer) - - await timer_runtime.start_roles() - - assert created == ["session", "watch"] - assert set(timer_runtime._consumers) == {"session", "watch"} - assert set(spawned) == {"callback:session", "callback:watch"} - for operation in spawned.values(): - operation.close() - - -@pytest.mark.asyncio -async def test_start_roles_creates_publisher_only_when_selected(monkeypatch): - class Publisher: - def __init__(self, store, runner, config, metrics): - pass - - async def run(self, stop): - await stop.wait() - - class Registry: - def freeze(self): - return {} - - timer_runtime = object.__new__(ExpiryRuntime) - timer_runtime._storage_started = True - timer_runtime._state = ExpiryRuntimeState() - timer_runtime._config = ExpiryConfig.model_construct() - timer_runtime._roles = ExpiryRuntimeRoles(publish=True) - timer_runtime._registry = Registry() - timer_runtime._stop = asyncio.Event() - timer_runtime._store = object() - timer_runtime._shard_runner = object() - timer_runtime._metrics = object() - timer_runtime._publisher = None - timer_runtime._consumers = {} - spawned = {} - - def spawn_role(name, operation): - spawned[name] = operation - - timer_runtime._spawn_role = spawn_role - monkeypatch.setattr(runtime_module, "ExpiryPublisher", Publisher) - - await timer_runtime.start_roles() - - assert timer_runtime._publisher is not None - assert set(spawned) == {"publisher"} - spawned["publisher"].close() - - -@pytest.mark.asyncio -async def test_stop_roles_clears_cooperative_tasks_before_closing_dependencies(): - timer_runtime = runtime() - started = asyncio.Event() - finished = asyncio.Event() - - async def role(): - started.set() - await timer_runtime._stop.wait() - finished.set() - - role_task = asyncio.create_task(role()) - timer_runtime._tasks["callback"] = role_task - try: - await asyncio.wait_for(started.wait(), timeout=2.0) - await asyncio.wait_for(timer_runtime.stop_roles(), timeout=2.0) - assert finished.is_set() - assert role_task.done() - assert timer_runtime._tasks == {} - assert timer_runtime._state.accepting_calls is False - assert timer_runtime._store.close_calls == 0 - assert timer_runtime._io.close_budgets == [] - await asyncio.wait_for(timer_runtime.close(), timeout=2.0) - assert timer_runtime._store.close_calls == 1 - assert timer_runtime._io.close_budgets == [0.02] - assert timer_runtime._storage_started is False - assert timer_runtime._state.storage_ready is False - finally: - timer_runtime._stop.set() - await asyncio.wait_for(asyncio.gather(role_task, return_exceptions=True), timeout=2.0) - - -@pytest.mark.asyncio -async def test_resistant_role_has_bounded_stop_and_retains_tasks_and_open_dependencies(): - timer_runtime = runtime() - started = asyncio.Event() - cancelled = asyncio.Event() - release_role = asyncio.Event() - - async def role(): - started.set() - while not release_role.is_set(): - try: - await release_role.wait() - except asyncio.CancelledError: - cancelled.set() - - role_task = asyncio.create_task(role()) - timer_runtime._tasks["callback"] = role_task - stop_task = None - close_task = None - try: - await asyncio.wait_for(started.wait(), timeout=2.0) - stop_task = asyncio.create_task(timer_runtime.stop_roles()) - await asyncio.wait_for(cancelled.wait(), timeout=2.0) - finished, _ = await asyncio.wait({stop_task}, timeout=0.3) - assert stop_task in finished, ( - "Role shutdown must not join cancellation resistance indefinitely" - ) - with pytest.raises(StoreOperationError) as error: - stop_task.result() - assert error.value.code == "shard_not_stopped" - assert timer_runtime._tasks == {"callback": role_task} - assert not role_task.done() - assert timer_runtime._state.accepting_calls is False - close_task = asyncio.create_task(timer_runtime.close()) - finished, _ = await asyncio.wait({close_task}, timeout=0.3) - assert close_task in finished - with pytest.raises(StoreOperationError) as error: - close_task.result() - assert error.value.code == "shard_not_stopped" - assert timer_runtime._tasks == {"callback": role_task} - assert timer_runtime._store.close_calls == 0 - assert timer_runtime._io.close_budgets == [] - assert timer_runtime._storage_started is True - assert timer_runtime._state.storage_ready is True - finally: - # Release resistant work before joining shutdown tasks; a cancellation - # watchdog must never become an unbounded join of the resistant role. - release_role.set() - tasks = [task for task in (role_task, stop_task, close_task) if task is not None] - await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=2.0) - await asyncio.wait_for(timer_runtime.close(), timeout=2.0) - assert timer_runtime._tasks == {} - assert timer_runtime._store.close_calls == 1 - assert timer_runtime._io.close_budgets == [0.02] - - -@pytest.mark.asyncio -async def test_runner_pending_blocks_close_after_role_task_has_exited(): - timer_runtime = runtime(pending=True) - - async def role(): - return None - - role_task = asyncio.create_task(role()) - await role_task - timer_runtime._tasks["callback"] = role_task - close_task = asyncio.create_task(timer_runtime.close()) - try: - finished, _ = await asyncio.wait({close_task}, timeout=0.3) - assert close_task in finished - with pytest.raises(StoreOperationError) as error: - close_task.result() - assert error.value.code == "shard_not_stopped" - assert timer_runtime._tasks == {} - assert timer_runtime._shard_runner.has_pending is True - assert timer_runtime._store.close_calls == 0 - assert timer_runtime._io.close_budgets == [] - assert timer_runtime._storage_started is True - assert timer_runtime._state.storage_ready is True - finally: - timer_runtime._shard_runner.settled.set() - await asyncio.wait_for(asyncio.gather(close_task, return_exceptions=True), timeout=2.0) - await asyncio.wait_for(timer_runtime.close(), timeout=2.0) - assert timer_runtime._store.close_calls == 1 - assert timer_runtime._io.close_budgets == [0.02] - assert timer_runtime._storage_started is False - - -@pytest.mark.asyncio -async def test_runtime_retries_thread_join_without_resubmitting_closed_storage_cleanup(): - timer_runtime = runtime() - - class SlowThreadIO(IO): - def __init__(self): - super().__init__() - self.stopping = False - - async def run_cleanup(self, operation, timeout_seconds): - if self.stopping: - raise StoreOperationError("not_started", "Redis loop already stopping") - return await operation() - - async def close(self, timeout_seconds): - self.close_budgets.append(timeout_seconds) - if not self.stopping: - self.stopping = True - raise StoreOperationError("shard_not_stopped", "I/O thread still exiting") - - timer_runtime._io = SlowThreadIO() - with pytest.raises(StoreOperationError) as error: - await timer_runtime.close() - assert error.value.code == "shard_not_stopped" - assert timer_runtime._storage_started - assert timer_runtime._state.storage_ready - assert timer_runtime._store.close_calls == 1 - await timer_runtime.close() - assert timer_runtime._store.close_calls == 1 - assert timer_runtime._io.close_budgets == [0.02, 0.02] - assert not timer_runtime._storage_started - assert not timer_runtime._state.storage_ready - - -class SessionService: - def __init__(self): - self.adapter = object() - self.adapter_updates = [] - - def set_session_expiry_adapter(self, adapter): - self.adapter_updates.append(adapter) - self.adapter = adapter - - -class Runtime: - def __init__(self, *, error=None): - self.error = error - self.close_calls = 0 - - async def close(self): - self.close_calls += 1 - if self.error is not None: - raise self.error - - -def service(expiry_runtime): - result = object.__new__(OpenVikingService) - result._expiry_runtime = expiry_runtime - result._session_service = SessionService() - return result - - -@pytest.mark.asyncio -async def test_core_retains_runtime_and_adapter_when_timer_work_has_not_stopped(): - shutdown_error = StoreOperationError("shard_not_stopped", "callback still running") - expiry_runtime = Runtime(error=shutdown_error) - core = service(expiry_runtime) - original_adapter = core._session_service.adapter - with pytest.raises(StoreOperationError) as error: - await core._stop_expiry() - assert error.value is shutdown_error - assert expiry_runtime.close_calls == 1 - assert core._expiry_runtime is expiry_runtime - assert core._session_service.adapter is original_adapter - assert core._session_service.adapter_updates == [] - expiry_runtime.error = None - await core._stop_expiry() - assert expiry_runtime.close_calls == 2 - assert core._expiry_runtime is None - assert core._session_service.adapter is None - assert core._session_service.adapter_updates == [None] - - -@pytest.mark.asyncio -async def test_core_clears_runtime_and_detaches_adapter_after_successful_close(): - expiry_runtime = Runtime() - core = service(expiry_runtime) - await core._stop_expiry() - assert expiry_runtime.close_calls == 1 - assert core._expiry_runtime is None - assert core._session_service.adapter is None - assert core._session_service.adapter_updates == [None] diff --git a/tests/scheduling/expiry/test_session_expiry.py b/tests/scheduling/expiry/test_session_expiry.py deleted file mode 100644 index 5239802237..0000000000 --- a/tests/scheduling/expiry/test_session_expiry.py +++ /dev/null @@ -1,252 +0,0 @@ -import asyncio -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest - -from openviking.scheduling.expiry import ( - CallbackRequest, - CallbackStatus, - ExpiryMode, - ItemIdentity, -) -from openviking.service.session_expiry import ( - SessionExpiryAdapter, - SessionTimeoutHandler, - effective_idle_timeout, - install_session_expiry, -) -from openviking.session.session import SessionExpirySnapshot - - -class Scheduler: - def __init__(self): - self.reports = [] - self.unregistered = [] - self.handlers = [] - - async def report(self, report): - self.reports.append(report) - return SimpleNamespace(accepted_due_at_ms=report.due_at_ms) - - async def unregister(self, identity): - self.unregistered.append(identity) - - def register_handler(self, item_type, handler, *, options=None): - self.handlers.append((item_type, handler, options)) - - -def test_install_session_expiry_uses_session_defaults(): - scheduler = Scheduler() - sessions = SimpleNamespace(set_session_expiry_adapter=lambda adapter: None) - - install_session_expiry(scheduler, sessions) - - assert len(scheduler.handlers) == 1 - item_type, _, options = scheduler.handlers[0] - assert item_type == "session" - assert options.concurrency == 8 - assert options.metadata_enabled is False - - -def test_install_session_expiry_uses_configured_concurrency(): - scheduler = Scheduler() - sessions = SimpleNamespace(set_session_expiry_adapter=lambda adapter: None) - - install_session_expiry(scheduler, sessions, concurrency=32) - - assert scheduler.handlers[0][2].concurrency == 32 - - -def test_adapter_builds_due_from_last_message_and_current_policy(): - scheduler = Scheduler() - adapter = SessionExpiryAdapter(scheduler) - snapshot = SessionExpirySnapshot( - "a", - "u", - "s", - "2026-01-01T00:00:00+00:00", - {"idle_timeout_seconds": 60}, - ) - report = adapter.build_report(snapshot) - assert report is not None - assert report.due_at_ms == 1767225660000 - assert report.timeout_seconds == 60 - - -def test_disabled_policy_produces_no_report(): - adapter = SessionExpiryAdapter(Scheduler()) - snapshot = SessionExpirySnapshot("a", "u", "s", "2026-01-01T00:00:00Z", None) - assert adapter.build_report(snapshot) is None - assert effective_idle_timeout({"idle_timeout_seconds": 0}) is None - - -@pytest.mark.asyncio -@pytest.mark.parametrize("reason", ["idle_cooldown", "idle_activity"]) -async def test_handler_reports_deferred_before_completing_old_event(reason): - scheduler = Scheduler() - - class Sessions: - async def commit_on_timeout(self, session_id, ctx): - return { - "status": "deferred", - "reason": reason, - "due_at_ms": 999, - "timeout_seconds": 30, - } - - identity = ItemIdentity("session", "a", "u", "s") - result = await SessionTimeoutHandler(Sessions(), scheduler).handle(CallbackRequest(identity)) - assert result.status is CallbackStatus.COMPLETED - assert scheduler.reports[0].due_at_ms == 999 - - -@pytest.mark.asyncio -async def test_handler_maps_not_found_to_completed(): - class Sessions: - async def commit_on_timeout(self, session_id, ctx): - return {"status": "skipped", "reason": "not_found"} - - result = await SessionTimeoutHandler(Sessions(), Scheduler()).handle( - CallbackRequest(ItemIdentity("session", "a", "u", "s")) - ) - assert result.status is CallbackStatus.COMPLETED - - -@pytest.mark.asyncio -@pytest.mark.parametrize("reason", ["idle_cooldown", "idle_activity"]) -async def test_handler_deferred_report_failure_propagates(reason): - class Sessions: - async def commit_on_timeout(self, session_id, ctx): - return { - "status": "deferred", - "reason": reason, - "due_at_ms": 999, - "timeout_seconds": 30, - } - - scheduler = Scheduler() - error = RuntimeError("reschedule unavailable") - scheduler.report = AsyncMock(spec=scheduler.report, side_effect=error) - identity = ItemIdentity("session", "a", "u", "s") - - with pytest.raises(RuntimeError, match="reschedule unavailable") as caught: - await SessionTimeoutHandler(Sessions(), scheduler).handle(CallbackRequest(identity)) - - assert caught.value is error - scheduler.report.assert_awaited_once() - report = scheduler.report.await_args.args[0] - assert report.identity == identity - assert report.due_at_ms == 999 - assert report.timeout_seconds == 30 - assert report.mode is ExpiryMode.IDLE_ONESHOT - - -@pytest.mark.asyncio -async def test_handler_reads_source_session_from_identity_without_metadata(): - calls = [] - - class Sessions: - async def commit_on_timeout(self, session_id, ctx): - calls.append((session_id, ctx)) - return {"status": "accepted", "task_id": "task"} - - scheduler = Scheduler() - identity = ItemIdentity("session", "account", "user", "s") - - result = await SessionTimeoutHandler(Sessions(), scheduler).handle(CallbackRequest(identity)) - - assert result.status is CallbackStatus.COMPLETED - assert len(calls) == 1 - session_id, ctx = calls[0] - assert session_id == "s" - assert ctx.account_id == "account" - assert ctx.user.user_id == "user" - assert scheduler.reports == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("reason", ["idle_disabled", "no_messages"]) -async def test_handler_terminal_skips_do_not_reschedule(reason): - class Sessions: - async def commit_on_timeout(self, session_id, ctx): - return {"status": "skipped", "reason": reason} - - scheduler = Scheduler() - result = await SessionTimeoutHandler(Sessions(), scheduler).handle( - CallbackRequest(ItemIdentity("session", "a", "u", "s")) - ) - - assert result.status is CallbackStatus.COMPLETED - assert scheduler.reports == [] - - -@pytest.mark.asyncio -async def test_handler_commit_failure_propagates(): - class Sessions: - async def commit_on_timeout(self, session_id, ctx): - raise RuntimeError("source commit failed") - - scheduler = Scheduler() - with pytest.raises(RuntimeError, match="source commit failed"): - await SessionTimeoutHandler(Sessions(), scheduler).handle( - CallbackRequest(ItemIdentity("session", "a", "u", "s")) - ) - assert scheduler.reports == [] - - -@pytest.mark.asyncio -async def test_adapter_append_reports_current_snapshot_identity_and_policy(): - scheduler = Scheduler() - snapshot = SessionExpirySnapshot( - "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 120} - ) - - await SessionExpiryAdapter(scheduler).report_after_append(snapshot) - - assert len(scheduler.reports) == 1 - report = scheduler.reports[0] - assert report.identity == ItemIdentity("session", "a", "u", "s") - assert report.due_at_ms == 1767225720000 - assert report.timeout_seconds == 120 - assert report.mode is ExpiryMode.IDLE_ONESHOT - - -@pytest.fixture -def adapter_logs(caplog): - from openviking.service.session_expiry import logger - - logger.addHandler(caplog.handler) - try: - yield caplog - finally: - logger.removeHandler(caplog.handler) - - -@pytest.mark.asyncio -async def test_adapter_append_report_failure_is_best_effort(adapter_logs): - scheduler = Scheduler() - scheduler.report = AsyncMock( - spec=scheduler.report, side_effect=RuntimeError("redis unavailable") - ) - snapshot = SessionExpirySnapshot( - "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 60} - ) - - await SessionExpiryAdapter(scheduler).report_after_append(snapshot) - - scheduler.report.assert_awaited_once() - assert "report_failed" in adapter_logs.text - - -@pytest.mark.asyncio -async def test_adapter_append_report_cancellation_propagates(): - scheduler = Scheduler() - scheduler.report = AsyncMock(spec=scheduler.report, side_effect=asyncio.CancelledError()) - snapshot = SessionExpirySnapshot( - "a", "u", "s", "2026-01-01T00:00:00Z", {"idle_timeout_seconds": 60} - ) - - with pytest.raises(asyncio.CancelledError): - await SessionExpiryAdapter(scheduler).report_after_append(snapshot) - scheduler.report.assert_awaited_once() diff --git a/tests/scheduling/expiry/test_session_timeout_service.py b/tests/scheduling/expiry/test_session_timeout_service.py deleted file mode 100644 index 49027ea289..0000000000 --- a/tests/scheduling/expiry/test_session_timeout_service.py +++ /dev/null @@ -1,639 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -"""Service-level idle checks and best-effort acceptance metadata recording.""" - -import asyncio -import copy -import json -from datetime import datetime, timedelta, timezone -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest - -from openviking.core.namespace import canonical_session_uri -from openviking.scheduling.expiry.config import ExpiryConfig -from openviking.service.session_service import SessionService -from openviking.session import Session -from openviking.session.session import SessionMeta -from openviking.utils.time_utils import parse_iso_datetime -from openviking_cli.exceptions import NotFoundError -from openviking_cli.utils.config.memory_config import SessionAutoCommitConfig - - -class MetadataFS: - """Minimal storage with observable locking and controllable concurrent writes.""" - - def __init__(self, ctx): - self.session_uri = canonical_session_uri(ctx, "s") - self.meta_uri = f"{self.session_uri}/.meta.json" - self.meta = { - "auto_commit_policy": {"idle_timeout_seconds": 60, "keep_recent_count": 12}, - "last_message_at": "2026-01-01T00:00:00Z", - "last_idle_commit_accepted_at": "", - "message_count": 1, - "commit_count": 3, - "updated_at": "2026-01-01T00:00:00Z", - } - self.exists = True - self.locked = False - self.events = [] - self.writes = [] - self.on_acquire = None - self.read_error = None - self.write_error = None - self.raw_meta = None - self._async_agfs = SimpleNamespace( - pathlock_acquire_exact=self.acquire, - pathlock_release=self.release, - ) - - def _uri_to_path(self, uri, *, ctx): - assert uri == self.session_uri - return "/local/session/s" - - async def acquire(self, path, **kwargs): - assert path == "/local/session/s" - assert not self.locked - self.locked = True - self.events.append("acquire") - if self.on_acquire is not None: - self.on_acquire() - return "lease" - - async def release(self, lease): - assert lease == "lease" - assert self.locked - self.events.append("release") - self.locked = False - - async def stat(self, uri, *, ctx, **kwargs): - assert uri == self.session_uri - self.events.append("stat") - if not self.exists: - raise NotFoundError("s", "session") - return {"isDir": True} - - async def read_file(self, uri, *, ctx, **kwargs): - assert uri == self.meta_uri - assert self.locked, "Acceptance metadata must be re-read under the session lock" - self.events.append("read") - if not self.exists: - raise NotFoundError("s", "session") - if self.read_error is not None: - raise self.read_error - return self.raw_meta if self.raw_meta is not None else json.dumps(self.meta) - - async def write_file(self, uri, content, *, ctx, **kwargs): - assert uri == self.meta_uri - assert self.locked - assert kwargs.get("lease_ref") is None, ( - "Child .meta.json must take its own auto-pathlock under the held " - "session-directory lock, not reuse the directory lease" - ) - assert self.exists, "Recording acceptance must not recreate a deleted session" - self.events.append("write") - if self.write_error is not None: - raise self.write_error - self.meta = json.loads(content) - self.writes.append(copy.deepcopy(self.meta)) - - -@pytest.fixture -def idle_session(): - return SimpleNamespace( - session_id="s", - messages=[object()], - meta=SessionMeta( - session_id="s", - auto_commit_policy={"idle_timeout_seconds": 60, "keep_recent_count": 12}, - last_message_at="2026-01-01T00:00:00Z", - message_count=1, - keep_recent_count=12, - ), - commit_async=AsyncMock(spec=Session.commit_async.__get__(object(), Session)), - ) - - -@pytest.fixture -def timeout_service(idle_session, request_context, monkeypatch): - class FrozenDatetime(datetime): - @classmethod - def now(cls, tz=None): - value = cls(2026, 1, 1, 0, 2, tzinfo=timezone.utc) - return value.astimezone(tz) if tz is not None else value.replace(tzinfo=None) - - monkeypatch.setattr("openviking.service.session_service.datetime", FrozenDatetime) - fs = MetadataFS(request_context) - service = SessionService(viking_fs=fs) - service.get = AsyncMock(spec=service.get, return_value=idle_session) - return service, fs - - -@pytest.fixture -def now(): - return datetime(2026, 1, 1, 0, 2, tzinfo=timezone.utc) - - -@pytest.mark.parametrize("policy", [None, {"idle_timeout_seconds": 0}]) -def test_timeout_check_disabled_policy_is_pure(idle_session, now, policy): - idle_session.meta.auto_commit_policy = policy - idle_session.meta.last_message_at = "" - before = copy.deepcopy(idle_session.meta.to_dict()) - - result = SessionService._check_timeout_commit(idle_session, now) - - assert result == {"status": "skipped", "reason": "idle_disabled"} - assert idle_session.meta.to_dict() == before - idle_session.commit_async.assert_not_called() - - -def test_timeout_check_empty_messages_precedes_cooldown_and_activity(idle_session, now): - idle_session.messages = [] - idle_session.meta.message_count = 99 - idle_session.meta.last_idle_commit_accepted_at = now.isoformat() - idle_session.meta.last_message_at = "" - - assert SessionService._check_timeout_commit(idle_session, now) == { - "status": "skipped", - "reason": "no_messages", - } - - -def test_timeout_check_uses_messages_not_keep_window_or_pending_tokens(idle_session, now): - idle_session.meta.message_count = 0 - idle_session.meta.pending_tokens = 0 - idle_session.meta.keep_recent_count = 100 - - assert SessionService._check_timeout_commit(idle_session, now) is None - - -@pytest.mark.parametrize("offset_ms", [-1, 0, 1]) -@pytest.mark.parametrize( - "last_message_at", - ["2026-01-01T00:01:00Z", "2026-01-01T00:01:00", "2026-01-01T08:01:00+08:00"], -) -def test_timeout_check_activity_boundary(idle_session, now, offset_ms, last_message_at): - idle_session.meta.last_message_at = last_message_at - before = copy.deepcopy(idle_session.meta.to_dict()) - result = SessionService._check_timeout_commit( - idle_session, now + timedelta(milliseconds=offset_ms) - ) - - if offset_ms < 0: - assert result == { - "status": "deferred", - "reason": "idle_activity", - "timeout_seconds": 60, - "due_at_ms": int(now.timestamp() * 1000), - } - else: - assert result is None - assert idle_session.meta.to_dict() == before - - -@pytest.mark.parametrize("offset_ms", [-1, 0, 1]) -def test_timeout_check_cooldown_boundary(idle_session, now, offset_ms): - idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:01:00Z" - checked_at = now + timedelta(milliseconds=offset_ms) - - result = SessionService._check_timeout_commit(idle_session, checked_at) - - if offset_ms < 0: - assert result == { - "status": "deferred", - "reason": "idle_cooldown", - "timeout_seconds": 60, - "due_at_ms": int(checked_at.timestamp() * 1000) + 60_000, - } - else: - assert result is None - - -def test_timeout_check_cooldown_uses_current_timeout_and_precedes_activity(idle_session, now): - idle_session.meta.auto_commit_policy = {"idle_timeout_seconds": 120} - idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:00:30Z" - idle_session.meta.last_message_at = "2026-01-01T00:01:59Z" - idle_session.meta.last_auto_commit_at = "2000-01-01T00:00:00Z" - - assert SessionService._check_timeout_commit(idle_session, now) == { - "status": "deferred", - "reason": "idle_cooldown", - "timeout_seconds": 120, - "due_at_ms": int(now.timestamp() * 1000) + 120_000, - } - - -def test_timeout_check_legacy_auto_commit_time_does_not_start_idle_cooldown(idle_session, now): - idle_session.meta.last_auto_commit_at = now.isoformat() - - assert SessionService._check_timeout_commit(idle_session, now) is None - - -@pytest.mark.asyncio -@pytest.mark.parametrize("working_memory_enabled", [None, False, True]) -async def test_timeout_commit_delegates_original_commit_and_records_after_acceptance( - timeout_service, idle_session, request_context, monkeypatch, working_memory_enabled -): - service, fs = timeout_service - idle_session.meta.memory_policy = ( - None - if working_memory_enabled is None - else {"working_memory": {"enabled": working_memory_enabled}} - ) - result = {"status": "accepted", "archived": True, "task_id": "task", "archive_uri": "archive"} - accepted_at = "2026-01-01T00:03:00.123Z" - - async def commit(**kwargs): - fs.events.append("commit") - return result - - clock_calls = 0 - - def acceptance_clock(): - nonlocal clock_calls - if clock_calls == 0: - assert fs.events == ["commit"], "Acceptance timestamp is sampled after commit returns" - clock_calls += 1 - return accepted_at - - idle_session.commit_async.side_effect = commit - monkeypatch.setattr( - "openviking.service.session_service.get_current_timestamp", acceptance_clock - ) - - actual = await service.commit_on_timeout("s", request_context) - - assert actual is result - service.get.assert_awaited_once_with("s", request_context, auto_create=False) - idle_session.commit_async.assert_awaited_once_with( - keep_recent_count=0, - persist_keep_recent_count=False, - record_auto_commit_success=True, - ) - assert fs.meta["last_idle_commit_accepted_at"] == accepted_at - assert len(fs.writes) == 1 - assert fs.events.index("commit") < fs.events.index("acquire") < fs.events.index("read") - assert fs.events[-1] == "release" - - -@pytest.mark.asyncio -async def test_timeout_commit_does_not_recheck_activity_after_outer_check( - timeout_service, idle_session, request_context -): - service, fs = timeout_service - result = {"status": "accepted", "archived": True} - - async def append_then_original_commit(**kwargs): - idle_session.messages.append(object()) - idle_session.meta.last_message_at = datetime.now(timezone.utc).isoformat() - return result - - idle_session.commit_async.side_effect = append_then_original_commit - - assert await service.commit_on_timeout("s", request_context) is result - assert len(idle_session.messages) == 2 - idle_session.commit_async.assert_awaited_once_with( - keep_recent_count=0, - persist_keep_recent_count=False, - record_auto_commit_success=True, - ) - assert len(fs.writes) == 1 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("reason", ["idle_disabled", "no_messages"]) -async def test_timeout_commit_outer_skip_does_not_commit_or_record( - timeout_service, idle_session, request_context, reason -): - service, fs = timeout_service - if reason == "idle_disabled": - idle_session.meta.auto_commit_policy = None - else: - idle_session.messages = [] - - assert await service.commit_on_timeout("s", request_context) == { - "status": "skipped", - "reason": reason, - } - idle_session.commit_async.assert_not_awaited() - assert fs.events == [] - assert fs.writes == [] - - -@pytest.mark.asyncio -async def test_timeout_commit_skipped_original_commit_does_not_record( - timeout_service, idle_session, request_context -): - service, fs = timeout_service - result = {"status": "skipped", "reason": "no_messages", "archived": False} - idle_session.commit_async.return_value = result - - assert await service.commit_on_timeout("s", request_context) is result - idle_session.commit_async.assert_awaited_once() - assert fs.events == [] - assert fs.writes == [] - - -@pytest.mark.asyncio -async def test_timeout_commit_not_found_loading_is_skipped(timeout_service, request_context): - service, fs = timeout_service - service.get.side_effect = NotFoundError("s", "session") - - assert await service.commit_on_timeout("s", request_context) == { - "status": "skipped", - "reason": "not_found", - } - service.get.assert_awaited_once_with("s", request_context, auto_create=False) - assert fs.events == [] - - -@pytest.mark.asyncio -async def test_timeout_commit_other_load_errors_propagate(timeout_service, request_context): - service, fs = timeout_service - error = RuntimeError("metadata storage unavailable") - service.get.side_effect = error - - with pytest.raises(RuntimeError, match="metadata storage unavailable") as caught: - await service.commit_on_timeout("s", request_context) - - assert caught.value is error - assert fs.events == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("not_found", [False, True]) -async def test_timeout_commit_commit_errors_propagate_without_recording( - timeout_service, idle_session, request_context, not_found -): - service, fs = timeout_service - error = NotFoundError("archive", "archive") if not_found else RuntimeError("commit failed") - idle_session.commit_async.side_effect = error - - with pytest.raises(type(error)) as caught: - await service.commit_on_timeout("s", request_context) - - assert caught.value is error - idle_session.commit_async.assert_awaited_once() - assert fs.events == [] - - -@pytest.fixture -def service_logs(caplog): - from openviking.service.session_service import logger - - logger.addHandler(caplog.handler) - try: - yield caplog - finally: - logger.removeHandler(caplog.handler) - - -@pytest.mark.asyncio -async def test_timeout_commit_metadata_write_failure_still_returns_accepted( - timeout_service, idle_session, request_context, service_logs -): - service, fs = timeout_service - fs.write_error = RuntimeError("acceptance metadata write failed") - result = {"status": "accepted", "archived": True, "task_id": "task"} - idle_session.commit_async.return_value = result - - assert await service.commit_on_timeout("s", request_context) is result - assert "write" in fs.events - assert fs.events[-1] == "release" - assert fs.writes == [] - assert "Failed to record idle commit time" in service_logs.text - - -@pytest.mark.asyncio -async def test_record_idle_time_rereads_under_lock_and_preserves_concurrent_fields( - timeout_service, request_context -): - service, fs = timeout_service - concurrent = { - **fs.meta, - "auto_commit_policy": {"idle_timeout_seconds": 180, "keep_recent_count": 77}, - "last_message_at": "2026-01-01T00:02:59Z", - "message_count": 9, - "commit_count": 4, - "pending_tokens": 4321, - "future_extension": {"opaque": [1, 2, 3]}, - } - fs.on_acquire = lambda: setattr(fs, "meta", copy.deepcopy(concurrent)) - accepted_at = "2026-01-01T00:03:00.123Z" - - await service._record_idle_commit_time("s", request_context, accepted_at) - - assert fs.meta["last_idle_commit_accepted_at"] == accepted_at - assert parse_iso_datetime(fs.meta["updated_at"]) >= parse_iso_datetime(concurrent["updated_at"]) - assert { - key: value - for key, value in fs.meta.items() - if key not in {"last_idle_commit_accepted_at", "updated_at"} - } == { - key: value - for key, value in concurrent.items() - if key not in {"last_idle_commit_accepted_at", "updated_at"} - } - assert len(fs.writes) == 1 - assert fs.events.index("acquire") < fs.events.index("read") < fs.events.index("write") - assert fs.events[-1] == "release" - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "existing_at", - ["2026-01-01T00:04:00Z", "2026-01-01T08:04:00+08:00"], -) -async def test_record_idle_time_never_moves_existing_acceptance_backwards( - timeout_service, request_context, existing_at -): - service, fs = timeout_service - fs.meta["last_idle_commit_accepted_at"] = existing_at - - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - assert parse_iso_datetime(fs.meta["last_idle_commit_accepted_at"]) == parse_iso_datetime( - existing_at - ) - assert fs.events[-1] == "release" - - -@pytest.mark.asyncio -async def test_record_idle_time_deleted_session_is_not_recreated(timeout_service, request_context): - service, fs = timeout_service - fs.on_acquire = lambda: setattr(fs, "exists", False) - - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - assert fs.exists is False - assert fs.writes == [] - assert fs.events[-1] == "release" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failure", ["read_error", "invalid_json", "not_a_dict", "write_error"]) -async def test_record_idle_time_metadata_failures_are_logged_best_effort( - timeout_service, request_context, service_logs, failure -): - service, fs = timeout_service - if failure == "read_error": - fs.read_error = RuntimeError("metadata read failed") - elif failure == "invalid_json": - fs.raw_meta = "{invalid" - elif failure == "not_a_dict": - fs.raw_meta = "[]" - else: - fs.write_error = RuntimeError("metadata write failed") - - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - assert fs.writes == [] - assert fs.events[-1] == "release" - assert "Failed to record idle commit time" in service_logs.text - - -@pytest.mark.asyncio -@pytest.mark.parametrize("reason", ["idle_activity", "idle_cooldown"]) -async def test_timeout_commit_deferred_does_not_commit_or_record( - timeout_service, idle_session, request_context, reason -): - service, fs = timeout_service - if reason == "idle_activity": - idle_session.meta.last_message_at = "2026-01-01T00:01:59Z" - expected_due_ms = 1767225779000 - else: - idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:01:59Z" - expected_due_ms = 1767225780000 - - assert await service.commit_on_timeout("s", request_context) == { - "status": "deferred", - "reason": reason, - "due_at_ms": expected_due_ms, - "timeout_seconds": 60, - } - idle_session.commit_async.assert_not_awaited() - assert fs.events == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("value", ["", "invalid timestamp"]) -async def test_timeout_commit_invalid_activity_time_fails_closed( - timeout_service, idle_session, request_context, value -): - service, fs = timeout_service - idle_session.meta.last_message_at = value - - with pytest.raises((RuntimeError, ValueError)): - await service.commit_on_timeout("s", request_context) - - idle_session.commit_async.assert_not_awaited() - assert fs.events == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("operation", ["read", "write"]) -async def test_record_idle_time_cancellation_propagates_and_releases_lock( - timeout_service, request_context, operation -): - service, fs = timeout_service - cancellation = asyncio.CancelledError() - setattr(fs, f"{operation}_error", cancellation) - - with pytest.raises(asyncio.CancelledError) as caught: - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - assert caught.value is cancellation - assert fs.events[-1] == "release" - assert fs.locked is False - assert fs.writes == [] - - -@pytest.mark.asyncio -async def test_record_idle_time_lock_failure_is_logged_best_effort( - timeout_service, request_context, service_logs -): - service, fs = timeout_service - acquire = AsyncMock(spec=fs.acquire, side_effect=RuntimeError("lock unavailable")) - fs._async_agfs.pathlock_acquire_exact = acquire - - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - acquire.assert_awaited_once() - assert fs.events == [] - assert fs.writes == [] - assert "Failed to record idle commit time" in service_logs.text - - -@pytest.mark.asyncio -async def test_record_idle_time_replaces_invalid_previous_acceptance( - timeout_service, request_context, service_logs -): - service, fs = timeout_service - fs.meta["last_idle_commit_accepted_at"] = "invalid timestamp" - accepted_at = "2026-01-01T00:03:00Z" - - await service._record_idle_commit_time("s", request_context, accepted_at) - - assert fs.meta["last_idle_commit_accepted_at"] == accepted_at - assert len(fs.writes) == 1 - assert fs.events[-1] == "release" - assert "Replacing invalid idle commit time" in service_logs.text - - -@pytest.fixture -def enable_default_local_backend(monkeypatch): - from openviking.service import core - - original_get_config = core.get_openviking_config - - def enabled_config(): - config = original_get_config() - config.memory.session_auto_commit = SessionAutoCommitConfig(enabled=True) - return config - - monkeypatch.setattr(core, "get_openviking_config", enabled_config) - - -@pytest.fixture -def local_service(enable_default_local_backend, service): - return service - - -@pytest.mark.asyncio -async def test_default_local_backend_starts_scheduler_without_expiry(local_service): - assert SessionAutoCommitConfig().idle_backend == "local" - assert local_service._session_auto_commit_scheduler is not None - assert local_service._session_auto_commit_scheduler._running is True - assert local_service._expiry_runtime is None - assert local_service.sessions._session_expiry_adapter is None - - -@pytest.fixture -def configure_expiry_without_enabling_redis_backend(monkeypatch): - from openviking.service import core - - original_get_config = core.get_openviking_config - - def enabled_config(): - config = original_get_config() - config.memory.session_auto_commit = SessionAutoCommitConfig(enabled=False) - config.expiry = ExpiryConfig( - service_name="unused", - redis={"endpoints": ["127.0.0.1:1"]}, - ) - return config - - monkeypatch.setattr(core, "get_openviking_config", enabled_config) - - -@pytest.fixture -def expiry_without_redis_backend_service(configure_expiry_without_enabling_redis_backend, service): - return service - - -@pytest.mark.asyncio -async def test_expiry_config_without_redis_backend_does_not_start_runtime( - expiry_without_redis_backend_service, -): - assert expiry_without_redis_backend_service._expiry_runtime is None diff --git a/tests/server/test_agent_evolution_global_setting.py b/tests/server/test_agent_evolution_global_setting.py index fc32d4d1cb..3a49ac0a12 100644 --- a/tests/server/test_agent_evolution_global_setting.py +++ b/tests/server/test_agent_evolution_global_setting.py @@ -19,6 +19,7 @@ from openviking.service.session_service import SessionService from openviking.session import Session from openviking_cli.session.user_id import UserIdentifier +from openviking_cli.utils.config.open_viking_config import OpenVikingConfig class _FakeAGFS: @@ -217,9 +218,13 @@ def test_server_default_memory_policy_is_configured_on_session_service(fake_viki async def test_auto_commit_defaults_apply_to_new_sessions(fake_viking_fs, monkeypatch): + ov_config = OpenVikingConfig() + monkeypatch.setattr("openviking.server.app.get_openviking_config", lambda: ov_config) + with pytest.raises(ValidationError, match="must be 0 or at least 300"): + ServerConfig(user_config_defaults={"auto_commit_policy": {"idle_timeout_seconds": 299}}) sessions = SessionService(viking_fs=fake_viking_fs) config = ServerConfig( - user_config_defaults={"auto_commit_policy": {"idle_timeout_seconds": 7200}} + user_config_defaults={"auto_commit_policy": {"idle_timeout_seconds": 300}} ) create_app( config=config, @@ -244,22 +249,6 @@ async def test_auto_commit_defaults_apply_to_new_sessions(fake_viking_fs, monkey assert session.meta.auto_commit_policy == expected -def test_deployment_auto_commit_default_rejects_idle_timeout_below_five_minutes(): - with pytest.raises(ValidationError, match="must be 0 or at least 300"): - ServerConfig(user_config_defaults={"auto_commit_policy": {"idle_timeout_seconds": 299}}) - - -@pytest.mark.parametrize("idle_timeout_seconds", [0, 300]) -def test_deployment_auto_commit_default_accepts_valid_idle_timeout(idle_timeout_seconds): - config = ServerConfig( - user_config_defaults={"auto_commit_policy": {"idle_timeout_seconds": idle_timeout_seconds}} - ) - - policy = config.user_config_defaults.auto_commit_policy - assert policy is not None - assert policy["idle_timeout_seconds"] == idle_timeout_seconds - - async def test_existing_session_observes_updated_account_value(fake_viking_fs): runtime_config = _FakeRuntimeConfig() service = SessionService(viking_fs=fake_viking_fs) diff --git a/tests/server/test_api_sessions.py b/tests/server/test_api_sessions.py index 4fab3c7b18..de7ec8afc9 100644 --- a/tests/server/test_api_sessions.py +++ b/tests/server/test_api_sessions.py @@ -707,75 +707,6 @@ async def test_auto_commit_policy_rejects_null_fields(client: httpx.AsyncClient) assert "auto_commit_policy=null" in resp.json()["error"]["message"] -@pytest.mark.parametrize("idle_timeout_seconds", [-1, 1, 299]) -async def test_auto_commit_policy_rejects_new_idle_timeout_below_five_minutes( - client: httpx.AsyncClient, - idle_timeout_seconds: int, -): - resp = await client.post( - "/api/v1/sessions", - json={"auto_commit_policy": {"idle_timeout_seconds": idle_timeout_seconds}}, - ) - - assert resp.status_code == 400 - assert "must be 0 or at least 300" in resp.json()["error"]["message"] - - -@pytest.mark.parametrize("idle_timeout_seconds", [0, 300]) -async def test_auto_commit_policy_accepts_disabled_or_five_minute_idle_timeout( - client: httpx.AsyncClient, - idle_timeout_seconds: int, -): - resp = await client.post( - "/api/v1/sessions", - json={"auto_commit_policy": {"idle_timeout_seconds": idle_timeout_seconds}}, - ) - - assert resp.status_code == 200 - assert ( - resp.json()["result"]["auto_commit_policy"]["idle_timeout_seconds"] == idle_timeout_seconds - ) - - -async def test_update_auto_commit_policy_rejects_idle_timeout_below_five_minutes( - client: httpx.AsyncClient, -): - created = await client.post("/api/v1/sessions", json={}) - session_id = created.json()["result"]["session_id"] - - updated = await client.patch( - f"/api/v1/sessions/{session_id}/config", - json={"auto_commit_policy": {"idle_timeout_seconds": 299}}, - ) - - assert updated.status_code == 400 - assert "must be 0 or at least 300" in updated.json()["error"]["message"] - - -async def test_updating_other_policy_field_preserves_legacy_idle_timeout( - client: httpx.AsyncClient, - service, -): - created = await client.post("/api/v1/sessions", json={}) - session_id = created.json()["result"]["session_id"] - ctx = RequestContext(user=DEFAULT_USER, role=Role.ADMIN) - session = await service.sessions.get(session_id, ctx) - await session.update_config( - auto_commit_policy={"idle_timeout_seconds": 60}, - update_auto_commit_policy=True, - ) - - updated = await client.patch( - f"/api/v1/sessions/{session_id}/config", - json={"auto_commit_policy": {"message_count_threshold": 25}}, - ) - - assert updated.status_code == 200 - policy = updated.json()["result"]["auto_commit_policy"] - assert policy["idle_timeout_seconds"] == 60 - assert policy["message_count_threshold"] == 25 - - async def test_auto_created_session_uses_default_policy_when_server_default_enabled( client: httpx.AsyncClient, service, @@ -836,7 +767,16 @@ async def test_create_session_clamps_config_above_bounds(client: httpx.AsyncClie assert policy["pending_token_threshold"] == 50000 -async def test_get_session_returns_effective_config(client: httpx.AsyncClient): +async def test_session_idle_policy_validates_writes_and_preserves_stored_values( + client: httpx.AsyncClient, service +): + rejected = await client.post( + "/api/v1/sessions", + json={"auto_commit_policy": {"idle_timeout_seconds": 299}}, + ) + assert rejected.status_code == 400 + assert "must be 0 or at least 300" in rejected.json()["error"]["message"] + create_resp = await client.post("/api/v1/sessions", json={}) session_id = create_resp.json()["result"]["session_id"] @@ -844,6 +784,40 @@ async def test_get_session_returns_effective_config(client: httpx.AsyncClient): assert resp.status_code == 200 assert resp.json()["result"]["auto_commit_policy"] is None + # Emulate a stored policy from before the HTTP write-time minimum existed. + ctx = RequestContext(user=DEFAULT_USER, role=Role.ROOT) + session = await service.sessions.get(session_id, ctx) + await session.update_config( + auto_commit_policy={"idle_timeout_seconds": 60}, + update_auto_commit_policy=True, + ) + config_url = f"/api/v1/sessions/{session_id}/config" + updated = await client.patch( + config_url, json={"auto_commit_policy": {"message_count_threshold": 25}} + ) + assert updated.status_code == 200 + policy = updated.json()["result"]["auto_commit_policy"] + assert policy["idle_timeout_seconds"] == 60 + assert policy["message_count_threshold"] == 25 + + rejected = await client.patch( + config_url, json={"auto_commit_policy": {"idle_timeout_seconds": 299}} + ) + assert rejected.status_code == 400 + persisted = await client.get(f"/api/v1/sessions/{session_id}") + assert persisted.json()["result"]["auto_commit_policy"] == policy + + updated = await client.patch( + config_url, json={"auto_commit_policy": {"idle_timeout_seconds": 300}} + ) + assert updated.status_code == 200 + assert updated.json()["result"]["auto_commit_policy"]["idle_timeout_seconds"] == 300 + disabled = await client.patch( + config_url, json={"auto_commit_policy": {"idle_timeout_seconds": 0}} + ) + assert disabled.status_code == 200 + assert disabled.json()["result"]["auto_commit_policy"]["idle_timeout_seconds"] == 0 + async def test_patch_session_config_is_not_supported(client: httpx.AsyncClient): create_resp = await client.post( diff --git a/tests/server/test_bootstrap.py b/tests/server/test_bootstrap.py index 20a7bdbe4f..8ed58bfe80 100644 --- a/tests/server/test_bootstrap.py +++ b/tests/server/test_bootstrap.py @@ -11,8 +11,10 @@ import openviking.server.app as app_module import openviking.server.bootstrap as bootstrap +from openviking.scheduling.expiry.config import ExpiryConfig from openviking.server.config import ServerConfig from openviking.utils.agfs_utils import resolve_queuefs_mount_point +from openviking_cli.utils.config.memory_config import SessionAutoCommitConfig from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton from openviking_cli.utils.config.storage_config import StorageConfig @@ -385,7 +387,9 @@ def run(): @pytest.mark.parametrize("app_startup_fails", [False, True]) -def test_main_restart_cleans_up_bot_and_preserves_launch_command(monkeypatch, app_startup_fails): +def test_main_restart_cleans_up_managed_children_and_preserves_launch_command( + monkeypatch, app_startup_fails +): import sys config = ServerConfig(host="127.0.0.1", port=1933, with_bot=True) @@ -396,39 +400,65 @@ def test_main_restart_cleans_up_bot_and_preserves_launch_command(monkeypatch, ap monkeypatch.setattr(bootstrap, "load_server_config", lambda path: config) monkeypatch.setattr(bootstrap, "resolve_config_path", lambda *a: Path("/tmp/explicit.conf")) monkeypatch.setattr( - OpenVikingConfigSingleton, "initialize", classmethod(lambda cls, **kw: None) + OpenVikingConfigSingleton, + "initialize", + classmethod( + lambda cls, **kw: SimpleNamespace( + memory=SimpleNamespace( + session_auto_commit=SessionAutoCommitConfig(enabled=True, idle_backend="redis") + ), + expiry=ExpiryConfig(), + ) + ), ) monkeypatch.setattr( "openviking_cli.utils.ollama.detect_ollama_in_config", lambda c: (False, "", 0) ) monkeypatch.setattr(bootstrap, "configure_uvicorn_logging", lambda: None) - monkeypatch.setattr(bootstrap, "create_app", lambda *a, **kw: "app") + + def create_app(*args, **kwargs): + assert steps == [("start", Path("/tmp/explicit.conf"))] + if app_startup_fails: + raise RuntimeError("application initialization failed") + return "app" + + monkeypatch.setattr(bootstrap, "create_app", create_app) monkeypatch.setattr(bootstrap, "_abort_if_port_in_use", lambda *a: None) monkeypatch.setattr(bootstrap, "_start_vikingbot_gateway", lambda *a, **kw: "bot") monkeypatch.setattr( bootstrap, "_stop_vikingbot_gateway", lambda bot: steps.append(("stop", bot)) ) + monkeypatch.setattr( + bootstrap, + "_start_expiry_publisher", + lambda path: steps.append(("start", path)) or "publisher", + ) + monkeypatch.setattr( + bootstrap, "_stop_expiry_publisher", lambda child: steps.append(("stop", child)) + ) monkeypatch.setattr(bootstrap, "_run_restartable_server", lambda *a, **kw: True) monkeypatch.setattr( os, "execv", lambda executable, args: steps.append(("exec", executable, args)) ) monkeypatch.setenv("OPENVIKING_CONFIG_FILE", "/tmp/explicit.conf") + lifecycle = [ + ("start", Path("/tmp/explicit.conf")), + ("stop", "publisher"), + ("stop", "bot"), + ] if app_startup_fails: - def fail_app(*args, **kwargs): - raise RuntimeError("application initialization failed") - - monkeypatch.setattr(bootstrap, "create_app", fail_app) with pytest.raises(RuntimeError, match="application initialization failed"): bootstrap.main() - assert steps == [("stop", "bot")] + assert steps == lifecycle return bootstrap.main() - assert steps == [("stop", "bot"), ("exec", sys.executable, original_argv)] + assert steps == [*lifecycle, ("exec", sys.executable, original_argv)] assert os.environ["OPENVIKING_CONFIG_FILE"] == "/tmp/explicit.conf" -def _run_restart_recovery_probe(tmp_path, monkeypatch, stage): - """Run one restart-recovery scenario in an isolated subprocess.""" +@pytest.mark.parametrize("failure", ["bind", "lifespan", "bootstrap", "success", "retry_failure"]) +def test_restart_recovery_through_real_exec(tmp_path, monkeypatch, failure): + """Exercise socket/lifespan failures and a single recovery with real execv.""" import hashlib import json import socket @@ -447,7 +477,7 @@ def _run_restart_recovery_probe(tmp_path, monkeypatch, stage): with socket.socket() as occupied: occupied.bind(("127.0.0.1", 0)) occupied.listen() - path.write_text(json.dumps({"port": occupied.getsockname()[1], "stage": stage})) + path.write_text(json.dumps({"port": occupied.getsockname()[1], "stage": failure})) # A normal save backup may contain an intermediate, never-started draft. path.with_name("ov.conf.studio.bak").write_bytes(b'{"stage":"intermediate"}') RestartRecovery().prepare(hashlib.sha256(path.read_bytes()).hexdigest()) @@ -485,36 +515,20 @@ async def lifespan(app): """) ) result = subprocess.run( - [sys.executable, str(script), str(path), stage], + [sys.executable, str(script), str(path), failure], env=os.environ.copy(), capture_output=True, text=True, timeout=40, ) - return result, path, backup, original - - -@pytest.mark.parametrize("failure", ["bind", "lifespan", "bootstrap", "retry_failure"]) -def test_restart_recovery_rolls_back_through_real_exec(tmp_path, monkeypatch, failure): - """Exercise startup failures and a single rollback with real execv.""" - result, path, backup, original = _run_restart_recovery_probe(tmp_path, monkeypatch, failure) if failure == "retry_failure": assert result.returncode != 0 assert result.stderr.count("retrying once") == 1 assert path.read_bytes() == original else: assert result.returncode == 0, result.stderr - assert "RECOVERED" in result.stdout - assert path.read_bytes() == original - assert not backup.exists() - - -def test_restart_recovery_disarms_after_successful_startup(tmp_path, monkeypatch): - result, path, backup, original = _run_restart_recovery_probe(tmp_path, monkeypatch, "success") - - assert result.returncode == 0, result.stderr - assert "APPLIED" in result.stdout - assert path.read_bytes() != original + assert ("APPLIED" if failure == "success" else "RECOVERED") in result.stdout + assert (path.read_bytes() == original) == (failure != "success") assert not backup.exists() diff --git a/tests/session/test_session_commit.py b/tests/session/test_session_commit.py index 5b046d0aed..fef3c154ec 100644 --- a/tests/session/test_session_commit.py +++ b/tests/session/test_session_commit.py @@ -5,14 +5,22 @@ import asyncio import json +from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock from openviking.message import TextPart +from openviking.scheduling.expiry import ( + CallbackRequest, + CallbackStatus, + ExpiryScheduler, + ItemIdentity, +) from openviking.server.identity import RequestContext from openviking.service.core import OpenVikingService from openviking.service.task_tracker import get_task_tracker from openviking.session import Session +from openviking.utils.time_utils import parse_iso_datetime async def _wait_for_task(task_id: str, timeout: float = 30.0) -> dict: @@ -34,101 +42,9 @@ async def _marker_exists(session, archive_uri: str, name: str) -> bool: return False -class _RecordingExpiryReporter: - def __init__(self) -> None: - self.snapshots = [] - self.failures = [] - - async def report_after_append(self, snapshot) -> None: - self.snapshots.append(snapshot) - - def record_report_failure(self, reason, exc=None) -> None: - self.failures.append((reason, exc)) - - class TestCommit: """Test commit""" - async def test_append_reports_repaired_meta_after_authoritative_reload_failure( - self, session_with_messages: Session, monkeypatch - ): - reporter = _RecordingExpiryReporter() - session_with_messages._expiry_reporter = reporter - session_with_messages.meta.auto_commit_policy = {"idle_timeout_seconds": 60} - await session_with_messages._save_meta() - original_read_file = session_with_messages._viking_fs.read_file - meta_uri = f"{session_with_messages.uri}/.meta.json" - failed = False - - async def fail_meta_reload_once(uri, *args, **kwargs): - nonlocal failed - if uri == meta_uri and not failed: - failed = True - raise RuntimeError("synthetic metadata reload failure") - return await original_read_file(uri, *args, **kwargs) - - monkeypatch.setattr(session_with_messages._viking_fs, "read_file", fail_meta_reload_once) - - await session_with_messages.add_message_async("user", [TextPart("latest activity")]) - - assert failed is True - assert len(reporter.snapshots) == 1 - assert reporter.snapshots[0].last_message_at == session_with_messages.meta.last_message_at - assert reporter.snapshots[0].auto_commit_policy == {"idle_timeout_seconds": 60} - assert reporter.failures == [] - - async def test_service_timeout_archives_append_after_check_without_idle_phase1_extensions( - self, session_with_messages: Session, service, request_context, monkeypatch - ): - session = session_with_messages - session.meta.auto_commit_policy = {"idle_timeout_seconds": 60, "keep_recent_count": 2} - session.meta.keep_recent_count = 2 - session.meta.last_message_at = "2000-01-01T00:00:00Z" - await session._save_meta() - original_commit = Session.commit_async - commits = [] - extract = AsyncMock( - spec=session._session_compressor.extract_long_term_memories, - return_value=[], - ) - monkeypatch.setattr(session._session_compressor, "extract_long_term_memories", extract) - - async def append_before_commit(loaded_session, **kwargs): - commits.append(kwargs) - await loaded_session.add_message_async("user", [TextPart("append after idle check")]) - return await original_commit(loaded_session, **kwargs) - - monkeypatch.setattr(Session, "commit_async", append_before_commit) - - result = await service.sessions.commit_on_timeout(session.session_id, request_context) - - assert result["status"] == "accepted" - assert commits == [ - { - "keep_recent_count": 0, - "persist_keep_recent_count": False, - "record_auto_commit_success": True, - } - ] - archived = await session._viking_fs.read_file( - f"{result['archive_uri']}/messages.jsonl", ctx=request_context - ) - archived_messages = [json.loads(line) for line in archived.split("\n") if line.strip()] - assert len(archived_messages) == 5 - assert archived_messages[-1]["parts"][0]["text"] == "append after idle check" - persisted = await service.sessions.get(session.session_id, request_context) - assert persisted.messages == [] - assert persisted.meta.keep_recent_count == 2 - assert persisted.meta.last_idle_commit_accepted_at - phase1 = await persisted._read_phase1_meta(result["archive_uri"]) - assert phase1["status"] == "ready" - assert "idle_timeout" not in phase1 - assert "last_idle_commit_accepted_at" not in phase1 - assert "last_idle_commit_archive" not in persisted.meta.to_dict() - assert (await _wait_for_task(result["task_id"]))["status"] == "completed" - extract.assert_awaited_once() - assert len(extract.await_args.kwargs["messages"]) == 5 - async def test_commit_preserves_unicode_separators_and_accepts_later_messages( self, wm_session_with_messages: Session ): @@ -159,12 +75,43 @@ async def test_commit_preserves_unicode_separators_and_accepts_later_messages( if part["type"] == "text" } - async def test_commit_extracts_memories( + async def test_idle_commit_reschedules_activity_and_extracts_the_full_backlog( self, wm_session_with_messages: Session, service: OpenVikingService, + monkeypatch, ): - """Test commit kicks off background memory extraction""" + """Expiry rechecks persisted activity, then uses the ordinary archive/extraction path.""" + session = wm_session_with_messages + await session.update_config( + auto_commit_policy={"idle_timeout_seconds": 300, "keep_recent_count": 2}, + update_auto_commit_policy=True, + ) + session.meta.keep_recent_count = 2 + await session._save_meta() + scheduler = MagicMock(spec=ExpiryScheduler) + service.sessions.configure_expiry(scheduler) + handler = scheduler.register_handler.call_args.args[1] + session = await service.sessions.get(session.session_id, session.ctx) + await session.add_message_async("user", [TextPart("latest activity")]) + report = scheduler.report.await_args.args[0] + last_message_at = parse_iso_datetime(session.meta.last_message_at) + assert report.identity == ItemIdentity( + "session", session.ctx.account_id, session.ctx.user.user_id, session.session_id + ) + assert report.due_at_ms == int(last_message_at.timestamp() * 1000) + 300_000 + + clock = MagicMock(wraps=datetime) + clock.now.return_value = last_message_at + timedelta(seconds=299) + monkeypatch.setattr("openviking.service.session_service.datetime", clock) + callback = await handler.handle(CallbackRequest(report.identity)) + assert callback.status is CallbackStatus.COMPLETED + assert callback.reason == "idle_rescheduled" + assert scheduler.report.await_count == 2 + assert scheduler.report.await_args.args[0] == report + persisted = await service.sessions.get(session.session_id, session.ctx) + assert len(persisted.messages) == 5 + assert persisted.meta.commit_count == 0 async def extract_long_term_memories(**kwargs): archive_uri = kwargs["archive_uri"] @@ -183,7 +130,29 @@ async def extract_long_term_memories(**kwargs): return_value={"contexts": [], "session_skills": []} ) - result = await wm_session_with_messages.commit_async() + # An append after the idle check belongs to the commit's snapshot. A + # scheduler outage must not fail that durable append or its commit. + original_commit = Session.commit_async + + async def append_before_commit(loaded_session, **kwargs): + scheduler.report.side_effect = RuntimeError("expiry unavailable") + await loaded_session.add_message_async("user", [TextPart("append after idle check")]) + return await original_commit(loaded_session, **kwargs) + + monkeypatch.setattr(Session, "commit_async", append_before_commit) + clock.now.return_value = last_message_at + timedelta(seconds=300) + result = await service.sessions.commit_on_timeout(session.session_id, session.ctx) + assert result["status"] == "accepted" + archive_content = await session._viking_fs.read_file( + f"{result['archive_uri']}/messages.jsonl", ctx=session.ctx + ) + archived_messages = [json.loads(line) for line in archive_content.split("\n") if line] + assert len(archived_messages) == 6 + assert archived_messages[-1]["parts"][0]["text"] == "append after idle check" + persisted = await service.sessions.get(session.session_id, session.ctx) + assert persisted.messages == [] + assert persisted.meta.keep_recent_count == 2 + assert persisted.meta.last_idle_commit_accepted_at task_id = result["task_id"] # Wait for background memory extraction to complete diff --git a/tests/unit/session/test_auto_commit_policy.py b/tests/unit/session/test_auto_commit_policy.py index e94895030a..c24256c526 100644 --- a/tests/unit/session/test_auto_commit_policy.py +++ b/tests/unit/session/test_auto_commit_policy.py @@ -14,9 +14,7 @@ MAX_IDLE_TIMEOUT_SECONDS, MAX_MESSAGE_COUNT_THRESHOLD, MAX_PENDING_TOKEN_THRESHOLD, - MIN_IDLE_TIMEOUT_SECONDS, AutoCommitPolicy, - validate_idle_timeout_input, ) from openviking_cli.exceptions import InvalidArgumentError @@ -85,24 +83,6 @@ def test_from_dict_clamps_negatives_to_zero(): } -def test_from_dict_preserves_legacy_idle_timeout_below_write_minimum(): - policy = AutoCommitPolicy.from_dict({"idle_timeout_seconds": 60}) - - assert policy.idle_timeout_seconds == 60 - - -@pytest.mark.parametrize("value", [0, 300, 604800]) -def test_validate_idle_timeout_input_accepts_zero_or_at_least_minimum(value): - assert validate_idle_timeout_input(value) == value - assert MIN_IDLE_TIMEOUT_SECONDS == 300 - - -@pytest.mark.parametrize("value", [-1, 1, 299]) -def test_validate_idle_timeout_input_rejects_nonzero_values_below_minimum(value): - with pytest.raises(ValueError, match="must be 0 or at least 300"): - validate_idle_timeout_input(value) - - def test_from_dict_rejects_unknown_keys(): with pytest.raises(InvalidArgumentError): AutoCommitPolicy.from_dict({"enabled": True}) diff --git a/tests/unit/test_server_bootstrap_expiry_publisher.py b/tests/unit/test_server_bootstrap_expiry_publisher.py deleted file mode 100644 index 288a612a67..0000000000 --- a/tests/unit/test_server_bootstrap_expiry_publisher.py +++ /dev/null @@ -1,200 +0,0 @@ -import os -import signal -import subprocess -import sys -import textwrap -import time -from pathlib import Path -from types import SimpleNamespace - -import pytest - -import openviking.server.bootstrap as bootstrap - - -class FakeProcess: - pid = 4321 - - def __init__(self, *, returncode=None): - self.returncode = returncode - self.terminated = False - self.killed = False - - def poll(self): - return self.returncode - - def wait(self, timeout=None): - return 0 if self.returncode is None else self.returncode - - def terminate(self): - self.terminated = True - self.returncode = 0 - - def kill(self): - self.killed = True - self.returncode = -9 - - -class FakeThread: - def __init__(self, *, target, args, name, daemon): - self.target = target - self.args = args - self.name = name - self.daemon = daemon - self.started = False - - def start(self): - self.started = True - - def join(self, timeout=None): - pass - - -def test_start_expiry_publisher_uses_internal_module_and_resolved_config(monkeypatch): - captured = {} - process = FakeProcess() - - def popen(command, **kwargs): - captured["command"] = command - captured["kwargs"] = kwargs - return process - - monkeypatch.setattr(bootstrap.subprocess, "Popen", popen) - monkeypatch.setattr(bootstrap, "_wait_for_expiry_publisher_ready", lambda *_: None) - monkeypatch.setattr(bootstrap.threading, "Thread", FakeThread) - - managed = bootstrap._start_expiry_publisher(Path("/tmp/ov.conf")) - - assert captured["command"][1:3] == ["-m", "openviking_cli.expiry_bootstrap"] - assert captured["command"][captured["command"].index("--config") + 1] == "/tmp/ov.conf" - assert "--startup-status" in captured["command"] - assert captured["kwargs"]["start_new_session"] == (bootstrap.os.name != "nt") - assert managed.process is process - assert managed.monitor_thread is not None - assert managed.monitor_thread.started is True - - -def test_monitor_stops_parent_when_publisher_exits_unexpectedly(monkeypatch): - process = FakeProcess(returncode=7) - managed = bootstrap.ExpiryPublisherProcess(process) - signals = [] - monkeypatch.setattr(bootstrap.os, "kill", lambda pid, signum: signals.append((pid, signum))) - - bootstrap._monitor_expiry_publisher(managed) - - assert signals == [(bootstrap.os.getpid(), bootstrap.signal.SIGTERM)] - - -def test_monitor_ignores_expected_publisher_shutdown(monkeypatch): - process = FakeProcess(returncode=0) - managed = bootstrap.ExpiryPublisherProcess(process) - managed.stopping.set() - signals = [] - monkeypatch.setattr(bootstrap.os, "kill", lambda pid, signum: signals.append((pid, signum))) - - bootstrap._monitor_expiry_publisher(managed) - - assert signals == [] - - -def test_readiness_surfaces_child_error(tmp_path): - status_path = tmp_path / "status.json" - status_path.write_text('{"status":"failed","error":"redis unavailable"}', encoding="utf-8") - process = SimpleNamespace(poll=lambda: None) - - try: - bootstrap._wait_for_expiry_publisher_ready( - process, - status_path, - timeout_seconds=0.1, - ) - except RuntimeError as exc: - assert "redis unavailable" in str(exc) - else: - raise AssertionError("readiness failure was not propagated") - - -@pytest.mark.skipif(os.name == "nt", reason="SIGTERM lifecycle is POSIX-specific") -def test_sigterm_runs_outer_finally_and_stops_detached_process(tmp_path): - ready_path = tmp_path / "ready" - cleanup_path = tmp_path / "cleaned" - child_pid_path = tmp_path / "child.pid" - script = textwrap.dedent( - """ - import subprocess - import sys - from contextlib import asynccontextmanager - from pathlib import Path - - import uvicorn - from fastapi import FastAPI - - from openviking.server.bootstrap import _sigterm_runs_finally - - ready_path = Path(sys.argv[1]) - cleanup_path = Path(sys.argv[2]) - child_pid_path = Path(sys.argv[3]) - child = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(300)"], - start_new_session=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - child_pid_path.write_text(str(child.pid), encoding="utf-8") - - @asynccontextmanager - async def lifespan(_app): - ready_path.write_text("ready", encoding="utf-8") - yield - - app = FastAPI(lifespan=lifespan) - try: - with _sigterm_runs_finally(): - uvicorn.run(app, host="127.0.0.1", port=0, log_level="error") - finally: - cleanup_path.write_text("cleaned", encoding="utf-8") - if child.poll() is None: - child.terminate() - child.wait(timeout=5) - """ - ) - process = subprocess.Popen( - [ - sys.executable, - "-c", - script, - str(ready_path), - str(cleanup_path), - str(child_pid_path), - ], - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - child_pid = None - output = "" - try: - deadline = time.monotonic() + 60 - while not ready_path.exists() and process.poll() is None and time.monotonic() < deadline: - time.sleep(0.05) - assert ready_path.exists(), "Uvicorn child did not enter application lifespan" - child_pid = int(child_pid_path.read_text(encoding="utf-8")) - - process.send_signal(signal.SIGTERM) - output, _ = process.communicate(timeout=20) - - assert process.returncode == 128 + signal.SIGTERM, output - assert cleanup_path.exists(), output - with pytest.raises(ProcessLookupError): - os.kill(child_pid, 0) - finally: - if process.poll() is None: - process.kill() - process.wait(timeout=5) - if child_pid is not None: - try: - os.kill(child_pid, signal.SIGKILL) - except ProcessLookupError: - pass From 2664af02599ecf58d40e94dd45a751a5bd06fbcf Mon Sep 17 00:00:00 2001 From: lujiakun Date: Wed, 14 Oct 2026 20:33:07 +0800 Subject: [PATCH 5/5] fix(session): harden idle timeout commits --- openviking/scheduling/expiry/consumer.py | 4 +- openviking/scheduling/expiry/publisher.py | 2 +- openviking/service/session_expiry.py | 2 +- openviking/service/session_service.py | 80 +---- openviking/session/session.py | 22 +- tests/scheduling/expiry/test_consumer.py | 74 +++- tests/scheduling/expiry/test_publisher.py | 7 +- .../scheduling/expiry/test_session_expiry.py | 10 +- .../expiry/test_session_timeout_service.py | 331 ++---------------- tests/session/test_session_commit.py | 112 +++++- .../test_session_retention_integration.py | 3 +- 11 files changed, 247 insertions(+), 400 deletions(-) diff --git a/openviking/scheduling/expiry/consumer.py b/openviking/scheduling/expiry/consumer.py index 934ed1e3c6..bdb4485ca7 100644 --- a/openviking/scheduling/expiry/consumer.py +++ b/openviking/scheduling/expiry/consumer.py @@ -109,7 +109,7 @@ async def run(self, stop: asyncio.Event) -> None: if not stop.is_set(): try: await asyncio.wait_for(stop.wait(), self._config.callback_poll_seconds) - except TimeoutError: + except asyncio.TimeoutError: pass async def process_shard(self, lease: ShardExecution, stop: asyncio.Event) -> None: @@ -292,7 +292,7 @@ async def _execute( ) except asyncio.CancelledError: raise - except TimeoutError as exc: + except asyncio.TimeoutError as exc: self._record_callback_error(claim, exc) lease.ensure_active() await self._finish_failed_attempt(lease, claim, "callback_timeout") diff --git a/openviking/scheduling/expiry/publisher.py b/openviking/scheduling/expiry/publisher.py index f491e05e5c..aa66fc3e3d 100644 --- a/openviking/scheduling/expiry/publisher.py +++ b/openviking/scheduling/expiry/publisher.py @@ -28,7 +28,7 @@ async def _wait_until_next_round(stop: asyncio.Event, delay: float) -> None: try: await asyncio.wait_for(stop.wait(), delay) - except TimeoutError: + except asyncio.TimeoutError: pass diff --git a/openviking/service/session_expiry.py b/openviking/service/session_expiry.py index 76dde89f09..cd5faa82a9 100644 --- a/openviking/service/session_expiry.py +++ b/openviking/service/session_expiry.py @@ -126,7 +126,7 @@ async def handle(self, request: CallbackRequest) -> CallbackResult: return self._to_callback_result(result) async def _report_deferred(self, identity: ItemIdentity, result: dict[str, Any]) -> None: - if result.get("reason") not in {"idle_cooldown", "idle_activity"}: + if result.get("reason") != "idle_cooldown": raise ValueError("Unexpected deferred session result") await self._scheduler.report( ExpiryRegistration( diff --git a/openviking/service/session_service.py b/openviking/service/session_service.py index f4966d8361..ee32755555 100644 --- a/openviking/service/session_service.py +++ b/openviking/service/session_service.py @@ -7,7 +7,6 @@ """ import asyncio -import json from dataclasses import replace from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Dict, List, Optional @@ -36,7 +35,7 @@ from openviking.storage.viking_fs import VikingFS from openviking.storage.vikingdb_manager import VikingDBManager from openviking.utils.tags import normalize_search_tags -from openviking.utils.time_utils import get_current_timestamp, parse_iso_datetime +from openviking.utils.time_utils import parse_iso_datetime from openviking_cli.exceptions import ( AlreadyExistsError, NotFoundError, @@ -47,7 +46,6 @@ from openviking_cli.utils.config.memory_config import SessionAutoCommitConfig logger = get_logger(__name__) -_SESSION_META_LOCK_TIMEOUT_SECONDS = 30.0 def _parse_utc_timestamp(value: str) -> datetime: @@ -493,11 +491,11 @@ async def commit_async( return result async def commit_on_timeout(self, session_id: str, ctx: RequestContext) -> Dict[str, Any]: - """Validate once, then reuse the ordinary session commit API. + """Treat an idle-timeout callback as a signal to commit current messages. - Appends after validation are allowed: commit_async owns the snapshot - boundary and its existing lock/queue/recovery behavior. This wrapper - neither rechecks idle under that lock nor waits for memory extraction. + The callback does not recheck ``last_message_at``. Messages appended + after the timeout was scheduled are included in the commit snapshot; + later appends schedule their own timeout callback. """ try: session = await self.get(session_id, ctx, auto_create=False) @@ -512,16 +510,15 @@ async def commit_on_timeout(self, session_id: str, ctx: RequestContext) -> Dict[ keep_recent_count=0, persist_keep_recent_count=False, record_auto_commit_success=True, + record_idle_commit_success=True, ) - if result.get("status") == "accepted": - await self._record_idle_commit_time(session_id, ctx, get_current_timestamp()) self._record_lifecycle_metric("commit", "ok" if result.get("status") else "error") self._record_archive_metric("ok" if result.get("archived") else "skip") return result @staticmethod def _check_timeout_commit(session: Session, now: datetime) -> Optional[Dict[str, Any]]: - """Check the freshly loaded state without inspecting archive internals.""" + """Check terminal skips and cooldown without rechecking idle activity.""" policy_data = session.meta.auto_commit_policy if policy_data is None: return {"status": "skipped", "reason": "idle_disabled"} @@ -543,71 +540,8 @@ def _check_timeout_commit(session: Session, now: datetime) -> Optional[Dict[str, "timeout_seconds": timeout_seconds, "due_at_ms": now_ms + timeout_seconds * 1000, } - if not session.meta.last_message_at: - raise RuntimeError("Session last_message_at is missing") - last_message_at = _parse_utc_timestamp(session.meta.last_message_at) - activity_due_ms = int(last_message_at.timestamp() * 1000) + timeout_seconds * 1000 - if now_ms < activity_due_ms: - return { - "status": "deferred", - "reason": "idle_activity", - "timeout_seconds": timeout_seconds, - "due_at_ms": activity_due_ms, - } return None - async def _record_idle_commit_time( - self, session_id: str, ctx: RequestContext, accepted_at: str - ) -> None: - """Best-effort patch of the latest metadata after an accepted commit. - - Acquire the session lock only after commit_async returns. Re-read and - patch the raw dictionary rather than saving a stale Session.meta, so a - concurrent append/config change and unknown fields are preserved. - """ - try: - self._ensure_initialized() - fs = self._viking_fs - session_uri = canonical_session_uri(ctx, session_id) - session_path = fs._uri_to_path(session_uri, ctx=ctx) - lease = await fs._async_agfs.pathlock_acquire_exact( - session_path, timeout_secs=_SESSION_META_LOCK_TIMEOUT_SECONDS - ) - try: - # A missing meta (including a session deleted meanwhile) must - # not be replaced with stale data or recreate the session. - content = await fs.read_file(f"{session_uri}/.meta.json", ctx=ctx) - metadata = json.loads(content) - if not isinstance(metadata, dict): - raise ValueError("Session metadata must be a JSON object") - previous = metadata.get("last_idle_commit_accepted_at") - if previous: - try: - previous_dt = _parse_utc_timestamp(previous) - except (TypeError, ValueError): - logger.warning("Replacing invalid idle commit time for %s", session_id) - else: - if previous_dt >= _parse_utc_timestamp(accepted_at): - return - metadata["last_idle_commit_accepted_at"] = accepted_at - metadata["updated_at"] = get_current_timestamp() - # The held lease is an exact lock on the session directory, used - # to serialize against concurrent append/config meta rewrites. - # Writing the child .meta.json must take its own auto-pathlock - # (the directory lease does not cover a child-path write), just - # like Session._save_meta() does under the same directory lock. - await fs.write_file( - f"{session_uri}/.meta.json", - json.dumps(metadata, ensure_ascii=False), - ctx=ctx, - ) - finally: - await fs._async_agfs.pathlock_release(lease) - except Exception: - # Recording failure must not turn an already accepted commit into - # a retry. Cancellation still propagates (it is a BaseException). - logger.warning("Failed to record idle commit time for %s", session_id, exc_info=True) - async def get_commit_task(self, task_id: str, ctx: RequestContext) -> Optional[Dict[str, Any]]: """Query background commit task status by task_id for the calling owner.""" task = await get_task_tracker().get( diff --git a/openviking/session/session.py b/openviking/session/session.py index 91c8d7c266..c66cce97b3 100644 --- a/openviking/session/session.py +++ b/openviking/session/session.py @@ -321,8 +321,8 @@ class SessionMeta: # Timestamp of the most recent successful auto-commit, surfaced via session # GET and used to throttle auto-commit frequency. last_auto_commit_at: str = "" - # Timeout commits record their accepted time externally; this is separate - # from the legacy auto-commit statistic and is not a Phase-1 ready marker. + # Timeout commits stamp their accepted time after the Phase 1 ready marker; + # this remains separate from the legacy auto-commit statistic. last_idle_commit_accepted_at: str = "" # Default custom scalar tags applied to event memories extracted from this # session. Maps to config.memory_extraction_config.events.tags in the API. @@ -1316,6 +1316,7 @@ async def commit_async( min_raw_tail_steps: Optional[int] = None, persist_keep_recent_count: bool = True, record_auto_commit_success: bool = False, + record_idle_commit_success: bool = False, event_tags: Optional[List[str]] = None, reset_context: bool = False, ) -> Dict[str, Any]: @@ -1344,6 +1345,9 @@ async def commit_async( record_auto_commit_success: When ``True``, clear the auto-commit error fields and stamp ``last_auto_commit_at`` in the same lock-protected meta update as the commit boundary. + record_idle_commit_success: When ``True``, best-effort stamp + ``last_idle_commit_accepted_at`` after the Phase 1 ready marker + is durable and before releasing the Session lock. event_tags: Per-commit override for the custom scalar tags applied to event memories. ``None`` uses the session default (``meta.event_search_tags``); an empty list disables default-tag @@ -1682,6 +1686,20 @@ async def commit_async( self._meta.last_auto_commit_at = get_current_timestamp() await self._save_meta() await self._write_phase1_ready_marker(archive_uri) + if record_idle_commit_success: + previous_accepted_at = self._meta.last_idle_commit_accepted_at + previous_updated_at = self._meta.updated_at + try: + self._meta.last_idle_commit_accepted_at = get_current_timestamp() + await self._save_meta() + except Exception: + self._meta.last_idle_commit_accepted_at = previous_accepted_at + self._meta.updated_at = previous_updated_at + logger.warning( + "Failed to record idle commit time for %s", + self.session_id, + exc_info=True, + ) except Exception as e: logger.error(f"[commit] Failed during {phase1_stage}: {e}") # Whether the queue write failed or a queued Phase 1 stopped diff --git a/tests/scheduling/expiry/test_consumer.py b/tests/scheduling/expiry/test_consumer.py index d0707e999b..cf1bd4dbbf 100644 --- a/tests/scheduling/expiry/test_consumer.py +++ b/tests/scheduling/expiry/test_consumer.py @@ -182,8 +182,16 @@ def config(**overrides): return ExpiryConfig.model_construct(**values) -def consumer(store, handler, *, max_attempts=3, metadata_enabled=True, metrics=None): - timer_config = config() +def consumer( + store, + handler, + *, + max_attempts=3, + metadata_enabled=True, + metrics=None, + timer_config=None, +): + timer_config = timer_config or config() metrics = metrics or ExpiryMetrics() coordinator = ShardLeaseCoordinator(store, timer_config, metrics) shard_runner = ShardRunner(coordinator, timer_config, metrics) @@ -416,6 +424,42 @@ async def test_timed_out_last_attempt_is_retried_without_becoming_exhausted(): assert shard_runner.has_pending is False +@pytest.mark.asyncio +async def test_callback_timeout_is_handled_as_a_failed_attempt(): + store = Store() + handler = Handler() + events = [] + metrics = ExpiryMetrics(lambda name, value, labels: events.append((name, value, labels))) + resource_consumer, shard_runner = consumer( + store, + handler, + max_attempts=1, + metrics=metrics, + timer_config=config(processing_timeout_seconds=0.01), + ) + + assert ( + await asyncio.wait_for( + shard_runner.run_if_acquired( + OwnerRole.CALLBACK, + 0, + resource_consumer.process_shard, + asyncio.Event(), + item_type="session", + ), + timeout=2.0, + ) + is True + ) + + assert handler.cancelled.is_set() + assert store.acked == [store.claims[0]] + assert any( + name == "timer_callback_exhausted" and labels["outcome"] == "callback_timeout" + for name, _, labels in events + ) + + @pytest.mark.asyncio async def test_waiting_bucket_defers_registration_to_exact_timeout_deadline(): store = Store(now_ms=124_000) @@ -689,6 +733,32 @@ async def test_consumer_worker_does_not_enter_runner_when_already_stopped(): assert shard_runner.calls == [] +@pytest.mark.asyncio +async def test_consumer_worker_continues_after_poll_timeout(): + stop = asyncio.Event() + + class PollingRunner: + def __init__(self): + self.calls = 0 + + async def run_if_acquired(self, role, shard_id, process, observed_stop, *, item_type=None): + self.calls += 1 + if self.calls == 2: + observed_stop.set() + return False + + runner = PollingRunner() + resource_consumer = worker_consumer( + Store(empty=True), + runner, + config(callback_poll_seconds=0.001), + ) + + await asyncio.wait_for(resource_consumer.run(stop), timeout=2.0) + + assert runner.calls == 2 + + @pytest.mark.asyncio async def test_consumer_worker_warns_and_continues_after_unfinished_shard(monkeypatch): shard_runner = Runner( diff --git a/tests/scheduling/expiry/test_publisher.py b/tests/scheduling/expiry/test_publisher.py index 7f172deaf9..452762ba71 100644 --- a/tests/scheduling/expiry/test_publisher.py +++ b/tests/scheduling/expiry/test_publisher.py @@ -17,7 +17,7 @@ SourcePage, StoreOperationError, ) -from openviking.scheduling.expiry.publisher import ExpiryPublisher +from openviking.scheduling.expiry.publisher import ExpiryPublisher, _wait_until_next_round from openviking.scheduling.expiry.shard_runner import ShardExecution @@ -86,6 +86,11 @@ def config(): ) +@pytest.mark.asyncio +async def test_round_wait_timeout_is_a_normal_scheduler_tick(): + await _wait_until_next_round(asyncio.Event(), 0.001) + + @pytest.mark.asyncio async def test_empty_scan_does_not_publish_or_advance_progress(): store = Store() diff --git a/tests/scheduling/expiry/test_session_expiry.py b/tests/scheduling/expiry/test_session_expiry.py index 5239802237..bac7a8fbc5 100644 --- a/tests/scheduling/expiry/test_session_expiry.py +++ b/tests/scheduling/expiry/test_session_expiry.py @@ -82,15 +82,14 @@ def test_disabled_policy_produces_no_report(): @pytest.mark.asyncio -@pytest.mark.parametrize("reason", ["idle_cooldown", "idle_activity"]) -async def test_handler_reports_deferred_before_completing_old_event(reason): +async def test_handler_reports_cooldown_before_completing_old_event(): scheduler = Scheduler() class Sessions: async def commit_on_timeout(self, session_id, ctx): return { "status": "deferred", - "reason": reason, + "reason": "idle_cooldown", "due_at_ms": 999, "timeout_seconds": 30, } @@ -114,13 +113,12 @@ async def commit_on_timeout(self, session_id, ctx): @pytest.mark.asyncio -@pytest.mark.parametrize("reason", ["idle_cooldown", "idle_activity"]) -async def test_handler_deferred_report_failure_propagates(reason): +async def test_handler_cooldown_report_failure_propagates(): class Sessions: async def commit_on_timeout(self, session_id, ctx): return { "status": "deferred", - "reason": reason, + "reason": "idle_cooldown", "due_at_ms": 999, "timeout_seconds": 30, } diff --git a/tests/scheduling/expiry/test_session_timeout_service.py b/tests/scheduling/expiry/test_session_timeout_service.py index 49027ea289..41b5ffe9a4 100644 --- a/tests/scheduling/expiry/test_session_timeout_service.py +++ b/tests/scheduling/expiry/test_session_timeout_service.py @@ -1,102 +1,29 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -"""Service-level idle checks and best-effort acceptance metadata recording.""" +"""Service-level idle checks and timeout commit delegation.""" -import asyncio import copy -import json from datetime import datetime, timedelta, timezone from types import SimpleNamespace from unittest.mock import AsyncMock import pytest -from openviking.core.namespace import canonical_session_uri from openviking.scheduling.expiry.config import ExpiryConfig from openviking.service.session_service import SessionService from openviking.session import Session from openviking.session.session import SessionMeta -from openviking.utils.time_utils import parse_iso_datetime from openviking_cli.exceptions import NotFoundError from openviking_cli.utils.config.memory_config import SessionAutoCommitConfig class MetadataFS: - """Minimal storage with observable locking and controllable concurrent writes.""" + """Storage sentinel: timeout delegation must not perform metadata I/O itself.""" def __init__(self, ctx): - self.session_uri = canonical_session_uri(ctx, "s") - self.meta_uri = f"{self.session_uri}/.meta.json" - self.meta = { - "auto_commit_policy": {"idle_timeout_seconds": 60, "keep_recent_count": 12}, - "last_message_at": "2026-01-01T00:00:00Z", - "last_idle_commit_accepted_at": "", - "message_count": 1, - "commit_count": 3, - "updated_at": "2026-01-01T00:00:00Z", - } - self.exists = True - self.locked = False + del ctx self.events = [] self.writes = [] - self.on_acquire = None - self.read_error = None - self.write_error = None - self.raw_meta = None - self._async_agfs = SimpleNamespace( - pathlock_acquire_exact=self.acquire, - pathlock_release=self.release, - ) - - def _uri_to_path(self, uri, *, ctx): - assert uri == self.session_uri - return "/local/session/s" - - async def acquire(self, path, **kwargs): - assert path == "/local/session/s" - assert not self.locked - self.locked = True - self.events.append("acquire") - if self.on_acquire is not None: - self.on_acquire() - return "lease" - - async def release(self, lease): - assert lease == "lease" - assert self.locked - self.events.append("release") - self.locked = False - - async def stat(self, uri, *, ctx, **kwargs): - assert uri == self.session_uri - self.events.append("stat") - if not self.exists: - raise NotFoundError("s", "session") - return {"isDir": True} - - async def read_file(self, uri, *, ctx, **kwargs): - assert uri == self.meta_uri - assert self.locked, "Acceptance metadata must be re-read under the session lock" - self.events.append("read") - if not self.exists: - raise NotFoundError("s", "session") - if self.read_error is not None: - raise self.read_error - return self.raw_meta if self.raw_meta is not None else json.dumps(self.meta) - - async def write_file(self, uri, content, *, ctx, **kwargs): - assert uri == self.meta_uri - assert self.locked - assert kwargs.get("lease_ref") is None, ( - "Child .meta.json must take its own auto-pathlock under the held " - "session-directory lock, not reuse the directory lease" - ) - assert self.exists, "Recording acceptance must not recreate a deleted session" - self.events.append("write") - if self.write_error is not None: - raise self.write_error - self.meta = json.loads(content) - self.writes.append(copy.deepcopy(self.meta)) @pytest.fixture @@ -148,7 +75,7 @@ def test_timeout_check_disabled_policy_is_pure(idle_session, now, policy): idle_session.commit_async.assert_not_called() -def test_timeout_check_empty_messages_precedes_cooldown_and_activity(idle_session, now): +def test_timeout_check_empty_messages_precedes_cooldown(idle_session, now): idle_session.messages = [] idle_session.meta.message_count = 99 idle_session.meta.last_idle_commit_accepted_at = now.isoformat() @@ -168,27 +95,15 @@ def test_timeout_check_uses_messages_not_keep_window_or_pending_tokens(idle_sess assert SessionService._check_timeout_commit(idle_session, now) is None -@pytest.mark.parametrize("offset_ms", [-1, 0, 1]) @pytest.mark.parametrize( "last_message_at", - ["2026-01-01T00:01:00Z", "2026-01-01T00:01:00", "2026-01-01T08:01:00+08:00"], + ["", "invalid timestamp", "2026-01-01T00:01:59Z"], ) -def test_timeout_check_activity_boundary(idle_session, now, offset_ms, last_message_at): +def test_timeout_check_ignores_last_message_at(idle_session, now, last_message_at): idle_session.meta.last_message_at = last_message_at before = copy.deepcopy(idle_session.meta.to_dict()) - result = SessionService._check_timeout_commit( - idle_session, now + timedelta(milliseconds=offset_ms) - ) - if offset_ms < 0: - assert result == { - "status": "deferred", - "reason": "idle_activity", - "timeout_seconds": 60, - "due_at_ms": int(now.timestamp() * 1000), - } - else: - assert result is None + assert SessionService._check_timeout_commit(idle_session, now) is None assert idle_session.meta.to_dict() == before @@ -210,10 +125,9 @@ def test_timeout_check_cooldown_boundary(idle_session, now, offset_ms): assert result is None -def test_timeout_check_cooldown_uses_current_timeout_and_precedes_activity(idle_session, now): +def test_timeout_check_cooldown_uses_current_timeout(idle_session, now): idle_session.meta.auto_commit_policy = {"idle_timeout_seconds": 120} idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:00:30Z" - idle_session.meta.last_message_at = "2026-01-01T00:01:59Z" idle_session.meta.last_auto_commit_at = "2000-01-01T00:00:00Z" assert SessionService._check_timeout_commit(idle_session, now) == { @@ -232,8 +146,8 @@ def test_timeout_check_legacy_auto_commit_time_does_not_start_idle_cooldown(idle @pytest.mark.asyncio @pytest.mark.parametrize("working_memory_enabled", [None, False, True]) -async def test_timeout_commit_delegates_original_commit_and_records_after_acceptance( - timeout_service, idle_session, request_context, monkeypatch, working_memory_enabled +async def test_timeout_commit_delegates_idle_success_recording_to_commit( + timeout_service, idle_session, request_context, working_memory_enabled ): service, fs = timeout_service idle_session.meta.memory_policy = ( @@ -242,25 +156,7 @@ async def test_timeout_commit_delegates_original_commit_and_records_after_accept else {"working_memory": {"enabled": working_memory_enabled}} ) result = {"status": "accepted", "archived": True, "task_id": "task", "archive_uri": "archive"} - accepted_at = "2026-01-01T00:03:00.123Z" - - async def commit(**kwargs): - fs.events.append("commit") - return result - - clock_calls = 0 - - def acceptance_clock(): - nonlocal clock_calls - if clock_calls == 0: - assert fs.events == ["commit"], "Acceptance timestamp is sampled after commit returns" - clock_calls += 1 - return accepted_at - - idle_session.commit_async.side_effect = commit - monkeypatch.setattr( - "openviking.service.session_service.get_current_timestamp", acceptance_clock - ) + idle_session.commit_async.return_value = result actual = await service.commit_on_timeout("s", request_context) @@ -270,15 +166,14 @@ def acceptance_clock(): keep_recent_count=0, persist_keep_recent_count=False, record_auto_commit_success=True, + record_idle_commit_success=True, ) - assert fs.meta["last_idle_commit_accepted_at"] == accepted_at - assert len(fs.writes) == 1 - assert fs.events.index("commit") < fs.events.index("acquire") < fs.events.index("read") - assert fs.events[-1] == "release" + assert fs.events == [] + assert fs.writes == [] @pytest.mark.asyncio -async def test_timeout_commit_does_not_recheck_activity_after_outer_check( +async def test_timeout_commit_includes_messages_appended_before_commit_snapshot( timeout_service, idle_session, request_context ): service, fs = timeout_service @@ -297,8 +192,9 @@ async def append_then_original_commit(**kwargs): keep_recent_count=0, persist_keep_recent_count=False, record_auto_commit_success=True, + record_idle_commit_success=True, ) - assert len(fs.writes) == 1 + assert fs.writes == [] @pytest.mark.asyncio @@ -378,138 +274,17 @@ async def test_timeout_commit_commit_errors_propagate_without_recording( assert fs.events == [] -@pytest.fixture -def service_logs(caplog): - from openviking.service.session_service import logger - - logger.addHandler(caplog.handler) - try: - yield caplog - finally: - logger.removeHandler(caplog.handler) - - -@pytest.mark.asyncio -async def test_timeout_commit_metadata_write_failure_still_returns_accepted( - timeout_service, idle_session, request_context, service_logs -): - service, fs = timeout_service - fs.write_error = RuntimeError("acceptance metadata write failed") - result = {"status": "accepted", "archived": True, "task_id": "task"} - idle_session.commit_async.return_value = result - - assert await service.commit_on_timeout("s", request_context) is result - assert "write" in fs.events - assert fs.events[-1] == "release" - assert fs.writes == [] - assert "Failed to record idle commit time" in service_logs.text - - -@pytest.mark.asyncio -async def test_record_idle_time_rereads_under_lock_and_preserves_concurrent_fields( - timeout_service, request_context -): - service, fs = timeout_service - concurrent = { - **fs.meta, - "auto_commit_policy": {"idle_timeout_seconds": 180, "keep_recent_count": 77}, - "last_message_at": "2026-01-01T00:02:59Z", - "message_count": 9, - "commit_count": 4, - "pending_tokens": 4321, - "future_extension": {"opaque": [1, 2, 3]}, - } - fs.on_acquire = lambda: setattr(fs, "meta", copy.deepcopy(concurrent)) - accepted_at = "2026-01-01T00:03:00.123Z" - - await service._record_idle_commit_time("s", request_context, accepted_at) - - assert fs.meta["last_idle_commit_accepted_at"] == accepted_at - assert parse_iso_datetime(fs.meta["updated_at"]) >= parse_iso_datetime(concurrent["updated_at"]) - assert { - key: value - for key, value in fs.meta.items() - if key not in {"last_idle_commit_accepted_at", "updated_at"} - } == { - key: value - for key, value in concurrent.items() - if key not in {"last_idle_commit_accepted_at", "updated_at"} - } - assert len(fs.writes) == 1 - assert fs.events.index("acquire") < fs.events.index("read") < fs.events.index("write") - assert fs.events[-1] == "release" - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "existing_at", - ["2026-01-01T00:04:00Z", "2026-01-01T08:04:00+08:00"], -) -async def test_record_idle_time_never_moves_existing_acceptance_backwards( - timeout_service, request_context, existing_at -): - service, fs = timeout_service - fs.meta["last_idle_commit_accepted_at"] = existing_at - - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - assert parse_iso_datetime(fs.meta["last_idle_commit_accepted_at"]) == parse_iso_datetime( - existing_at - ) - assert fs.events[-1] == "release" - - -@pytest.mark.asyncio -async def test_record_idle_time_deleted_session_is_not_recreated(timeout_service, request_context): - service, fs = timeout_service - fs.on_acquire = lambda: setattr(fs, "exists", False) - - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - assert fs.exists is False - assert fs.writes == [] - assert fs.events[-1] == "release" - - @pytest.mark.asyncio -@pytest.mark.parametrize("failure", ["read_error", "invalid_json", "not_a_dict", "write_error"]) -async def test_record_idle_time_metadata_failures_are_logged_best_effort( - timeout_service, request_context, service_logs, failure -): - service, fs = timeout_service - if failure == "read_error": - fs.read_error = RuntimeError("metadata read failed") - elif failure == "invalid_json": - fs.raw_meta = "{invalid" - elif failure == "not_a_dict": - fs.raw_meta = "[]" - else: - fs.write_error = RuntimeError("metadata write failed") - - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - assert fs.writes == [] - assert fs.events[-1] == "release" - assert "Failed to record idle commit time" in service_logs.text - - -@pytest.mark.asyncio -@pytest.mark.parametrize("reason", ["idle_activity", "idle_cooldown"]) -async def test_timeout_commit_deferred_does_not_commit_or_record( - timeout_service, idle_session, request_context, reason +async def test_timeout_commit_cooldown_does_not_commit_or_record( + timeout_service, idle_session, request_context ): service, fs = timeout_service - if reason == "idle_activity": - idle_session.meta.last_message_at = "2026-01-01T00:01:59Z" - expected_due_ms = 1767225779000 - else: - idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:01:59Z" - expected_due_ms = 1767225780000 + idle_session.meta.last_idle_commit_accepted_at = "2026-01-01T00:01:59Z" assert await service.commit_on_timeout("s", request_context) == { "status": "deferred", - "reason": reason, - "due_at_ms": expected_due_ms, + "reason": "idle_cooldown", + "due_at_ms": 1767225780000, "timeout_seconds": 60, } idle_session.commit_async.assert_not_awaited() @@ -517,68 +292,18 @@ async def test_timeout_commit_deferred_does_not_commit_or_record( @pytest.mark.asyncio -@pytest.mark.parametrize("value", ["", "invalid timestamp"]) -async def test_timeout_commit_invalid_activity_time_fails_closed( +@pytest.mark.parametrize("value", ["", "invalid timestamp", "2026-01-01T00:01:59Z"]) +async def test_timeout_commit_ignores_last_message_at( timeout_service, idle_session, request_context, value ): service, fs = timeout_service + result = {"status": "accepted", "archived": True} idle_session.meta.last_message_at = value + idle_session.commit_async.return_value = result - with pytest.raises((RuntimeError, ValueError)): - await service.commit_on_timeout("s", request_context) - - idle_session.commit_async.assert_not_awaited() - assert fs.events == [] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("operation", ["read", "write"]) -async def test_record_idle_time_cancellation_propagates_and_releases_lock( - timeout_service, request_context, operation -): - service, fs = timeout_service - cancellation = asyncio.CancelledError() - setattr(fs, f"{operation}_error", cancellation) - - with pytest.raises(asyncio.CancelledError) as caught: - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - assert caught.value is cancellation - assert fs.events[-1] == "release" - assert fs.locked is False - assert fs.writes == [] - - -@pytest.mark.asyncio -async def test_record_idle_time_lock_failure_is_logged_best_effort( - timeout_service, request_context, service_logs -): - service, fs = timeout_service - acquire = AsyncMock(spec=fs.acquire, side_effect=RuntimeError("lock unavailable")) - fs._async_agfs.pathlock_acquire_exact = acquire - - await service._record_idle_commit_time("s", request_context, "2026-01-01T00:03:00Z") - - acquire.assert_awaited_once() + assert await service.commit_on_timeout("s", request_context) is result + idle_session.commit_async.assert_awaited_once() assert fs.events == [] - assert fs.writes == [] - assert "Failed to record idle commit time" in service_logs.text - - -@pytest.mark.asyncio -async def test_record_idle_time_replaces_invalid_previous_acceptance( - timeout_service, request_context, service_logs -): - service, fs = timeout_service - fs.meta["last_idle_commit_accepted_at"] = "invalid timestamp" - accepted_at = "2026-01-01T00:03:00Z" - - await service._record_idle_commit_time("s", request_context, accepted_at) - - assert fs.meta["last_idle_commit_accepted_at"] == accepted_at - assert len(fs.writes) == 1 - assert fs.events[-1] == "release" - assert "Replacing invalid idle commit time" in service_logs.text @pytest.fixture diff --git a/tests/session/test_session_commit.py b/tests/session/test_session_commit.py index 5b046d0aed..1e4040624a 100644 --- a/tests/session/test_session_commit.py +++ b/tests/session/test_session_commit.py @@ -8,6 +8,8 @@ from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock +import pytest + from openviking.message import TextPart from openviking.server.identity import RequestContext from openviking.service.core import OpenVikingService @@ -87,11 +89,7 @@ async def test_service_timeout_archives_append_after_check_without_idle_phase1_e await session._save_meta() original_commit = Session.commit_async commits = [] - extract = AsyncMock( - spec=session._session_compressor.extract_long_term_memories, - return_value=[], - ) - monkeypatch.setattr(session._session_compressor, "extract_long_term_memories", extract) + monkeypatch.setattr(Session, "resume_queued_commit", AsyncMock(return_value=True)) async def append_before_commit(loaded_session, **kwargs): commits.append(kwargs) @@ -108,6 +106,7 @@ async def append_before_commit(loaded_session, **kwargs): "keep_recent_count": 0, "persist_keep_recent_count": False, "record_auto_commit_success": True, + "record_idle_commit_success": True, } ] archived = await session._viking_fs.read_file( @@ -125,9 +124,106 @@ async def append_before_commit(loaded_session, **kwargs): assert "idle_timeout" not in phase1 assert "last_idle_commit_accepted_at" not in phase1 assert "last_idle_commit_archive" not in persisted.meta.to_dict() - assert (await _wait_for_task(result["task_id"]))["status"] == "completed" - extract.assert_awaited_once() - assert len(extract.await_args.kwargs["messages"]) == 5 + + async def test_idle_acceptance_is_persisted_after_phase1_ready( + self, session_with_messages: Session, monkeypatch + ): + session = session_with_messages + events = [] + original_save_meta = session._save_meta + original_write_ready = session._write_phase1_ready_marker + + async def record_save(*args, **kwargs): + events.append(("save", bool(session.meta.last_idle_commit_accepted_at))) + await original_save_meta(*args, **kwargs) + + async def record_ready(*args, **kwargs): + await original_write_ready(*args, **kwargs) + events.append(("ready", bool(session.meta.last_idle_commit_accepted_at))) + + monkeypatch.setattr(session, "_save_meta", record_save) + monkeypatch.setattr(session, "_write_phase1_ready_marker", record_ready) + + result = await session.commit_async(record_idle_commit_success=True) + + assert result["status"] == "accepted" + assert events == [("save", False), ("ready", False), ("save", True)] + persisted = json.loads( + await session._viking_fs.read_file(f"{session.uri}/.meta.json", ctx=session.ctx) + ) + assert persisted["last_idle_commit_accepted_at"] + assert (await session._read_phase1_meta(result["archive_uri"]))["status"] == "ready" + + async def test_idle_acceptance_write_failure_keeps_phase1_accepted( + self, session_with_messages: Session, monkeypatch + ): + session = session_with_messages + session.meta.last_idle_commit_accepted_at = "2026-01-01T00:00:00Z" + await session._save_meta() + events = [] + save_count = 0 + phase1_updated_at = "" + original_save_meta = session._save_meta + original_write_ready = session._write_phase1_ready_marker + + async def fail_idle_save(*args, **kwargs): + nonlocal phase1_updated_at, save_count + save_count += 1 + events.append("save") + if save_count == 2: + session.meta.updated_at = "failed-write-timestamp" + raise RuntimeError("idle acceptance metadata write failed") + await original_save_meta(*args, **kwargs) + phase1_updated_at = session.meta.updated_at + + async def record_ready(*args, **kwargs): + await original_write_ready(*args, **kwargs) + events.append("ready") + + monkeypatch.setattr(session, "_save_meta", fail_idle_save) + monkeypatch.setattr(session, "_write_phase1_ready_marker", record_ready) + + result = await session.commit_async(record_idle_commit_success=True) + + assert result["status"] == "accepted" + assert events == ["save", "ready", "save"] + assert session.meta.last_idle_commit_accepted_at == "2026-01-01T00:00:00Z" + assert session.meta.updated_at == phase1_updated_at + persisted = json.loads( + await session._viking_fs.read_file(f"{session.uri}/.meta.json", ctx=session.ctx) + ) + assert persisted["last_idle_commit_accepted_at"] == "2026-01-01T00:00:00Z" + assert persisted["updated_at"] == phase1_updated_at + assert (await session._read_phase1_meta(result["archive_uri"]))["status"] == "ready" + assert not await _marker_exists(session, result["archive_uri"], ".failed.json") + + async def test_idle_acceptance_is_not_recorded_when_ready_marker_fails( + self, session_with_messages: Session, monkeypatch + ): + session = session_with_messages + save_count = 0 + original_save_meta = session._save_meta + + async def record_save(*args, **kwargs): + nonlocal save_count + save_count += 1 + await original_save_meta(*args, **kwargs) + + async def fail_ready(*args, **kwargs): + raise RuntimeError("ready marker unavailable") + + monkeypatch.setattr(session, "_save_meta", record_save) + monkeypatch.setattr(session, "_write_phase1_ready_marker", fail_ready) + + with pytest.raises(RuntimeError, match="ready marker unavailable"): + await session.commit_async(record_idle_commit_success=True) + + assert save_count == 1 + assert session.meta.last_idle_commit_accepted_at == "" + persisted = json.loads( + await session._viking_fs.read_file(f"{session.uri}/.meta.json", ctx=session.ctx) + ) + assert persisted["last_idle_commit_accepted_at"] == "" async def test_commit_preserves_unicode_separators_and_accepts_later_messages( self, wm_session_with_messages: Session diff --git a/tests/session/test_session_retention_integration.py b/tests/session/test_session_retention_integration.py index cf8c0703c5..ec0af852ba 100644 --- a/tests/session/test_session_retention_integration.py +++ b/tests/session/test_session_retention_integration.py @@ -1743,7 +1743,7 @@ async def enqueue(self, *_args, **_kwargs): ) with pytest.raises(RuntimeError, match="queue unavailable"): - await session.commit_async() + await session.commit_async(record_idle_commit_success=True) states = await session._archives.scan_states() context = await session.get_session_context() @@ -1761,6 +1761,7 @@ async def enqueue(self, *_args, **_kwargs): assert [message.content for message in raw] == ["do not lose me"] assert [message["parts"][0]["text"] for message in context["messages"]] == ["do not lose me"] assert context["stats"]["failedArchives"] == 1 + assert session.meta.last_idle_commit_accepted_at == "" async def test_phase1_root_rewrite_failure_marks_orphan_archive_failed(