diff --git a/.github/actions/python-distributions/action.yml b/.github/actions/python-distributions/action.yml index fa3878cf..9a155ab9 100644 --- a/.github/actions/python-distributions/action.yml +++ b/.github/actions/python-distributions/action.yml @@ -61,9 +61,13 @@ runs: import adx_environment import akernel_sdk from akernel_sdk import ComposeProject, DockerRuntime, ServiceHandle, ServiceProcessHandle + from akernel_sdk import BackendOperationError, InsufficientResourcesError, SchedulingError from importlib.resources import files assert all(callable(value) for value in (ComposeProject, DockerRuntime, ServiceHandle, ServiceProcessHandle)) + assert issubclass(InsufficientResourcesError, SchedulingError) + assert issubclass(SchedulingError, BackendOperationError) + assert SchedulingError("smoke").reasons == ("unknown",) for helper in ("_image_bridge_guest.py", "_docker_network_guest.py", "_service_process_guest.py"): assert files("akernel_sdk").joinpath(helper).read_text() diff --git a/AGENTS.md b/AGENTS.md index 1a0ad9aa..efd195ba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -725,6 +725,12 @@ the pinned native PermissionDenied exception without inventing a missing status. `BackendOperationError` preserves optional structured service fields and the original cause; missing context remains `None`. Keep retry/outcome semantics backend-owned and document public error fields in `sdk/python/README.md`. +Confirmed cancelled queue requests with structured diagnostics use +`SchedulingError` and, when scalar shortages contribute with no eligible worker, +`InsufficientResourcesError`. Preserve raw diagnostics and native causes while +explaining all blockers; eligible workers or unusable diagnostics must not be +presented as confirmed resource shortages. Keep classification independent of +retry policy and do not infer usable capacity from global per-dimension maxima. Keep sandbox cleanup explicit through context managers or `kill()`, with observable, retryable deletion failures and workload exceptions preserved on diff --git a/sdk/python/README.md b/sdk/python/README.md index 6eab24e5..09f6e586 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -1469,6 +1469,104 @@ original identity when querying or retrying. Do not infer execution or cleanup from a timeout message. `status_code` describes the service failure (including an SSE final event's error code); it need not equal the stream's HTTP status. +Confirmed scheduling cancellations with structured diagnostics raise +`SchedulingError`, a subclass of `BackendOperationError`. If no worker is +eligible and CPU, memory or disk shortages contribute, the exception is the +more specific `InsufficientResourcesError`, which also subclasses +`SchedulingError`. Existing `except BackendOperationError` handlers still work. + +The service's `error.schedulingFailure` object is retained as +`error.scheduling_failure`, with its original field names and values: + +| Field | Meaning | +| --- | --- | +| `snapshotRevision` | Revision of the scheduling snapshot at queue expiry; retained for diagnostics. | +| `nodeCount` | Number of workers known to the scheduler. Zero establishes `no_workers`. | +| `eligibleNodes` | Number passing all hard placement checks in that snapshot. A positive value produces `queue_timeout`. | +| `requested` | Requested resources: `cpu_millis` in millicores, `memory_bytes` and `disk_bytes` in bytes. | +| `maxCapacity`, `maxAvailable` | Resource objects with the same keys/units. Maxima span workers; available values already subtract reservations. | +| `rejections` | Map of rejection keys to worker counts. Counts may overlap; zero counts do not establish blockers. | + +Counters and resource amounts use unsigned 64-bit integers; booleans, negative +values and numeric strings are not coerced into valid evidence. Missing or +unusable eligibility data cannot establish a resource shortage; classification +also requires valid positive resource-rejection counts. Additional fields and +future rejection keys remain available in the original object. +Keep schema extensions aligned with ADX's `platform/api/http/error-contract.md`; +adding compatible-worker capacity diagnostics requires a service-side contract +before the SDK can distinguish temporary occupancy from an oversized request. + +The scheduling exceptions expose: + +- `reasons`: a tuple of positive worker rejection keys from the service, sorted + by key. Counts can overlap; their sum is not a worker count. Unknown future + rejection keys are retained. +- `resource_shortages`: contributing resource names (`"cpu"`, `"memory"`, + `"disk"`), in that order. Other constraints can also contribute. +- `suggestions`: a tuple of concrete recovery suggestions. The exception message + includes these suggestions, readable requested resources, and rejection counts. + +```python +from akernel_sdk import InsufficientResourcesError, Sandbox, SchedulingError + +try: + sandbox = Sandbox(cpu=2000, memory=8192, storage_mb=20480) +except InsufficientResourcesError as error: + print("Unavailable resources:", ", ".join(error.resource_shortages)) + print(error) # Summary and actions; this does not retry the request. + print("All placement blockers:", error.reasons) +except SchedulingError as error: + print(error) # Runtime, devices, placement, volumes, workers, or queue budget. +else: + sandbox.kill() +``` + +| Reported blocker | Action to consider | +| --- | --- | +| `insufficient_cpu`, `insufficient_memory`, `insufficient_disk` | Reduce the request, release idle sandboxes, or add suitable worker capacity. | +| `runtime_fit`, `device_fit` | Select a supported runtime or available accelerator model/count, or add compatible workers. | +| `node_available`, `no_workers` | Ask the operator to check worker health/registration and register healthy workers. | +| `node_affinity`, `placement_groups`, `environment_affinity`, `topology_spread` | Adjust placement constraints or add workers that satisfy them. | +| `waiting_for_volume`, `volume_topology` | Wait for the volume to be released or select workers with access to its storage. | + +For example, a mixed rejection produces a message like: + +```text +create sandbox failed: the cluster has insufficient resources for this request; +no worker satisfies all resource and placement requirements. +Requested: 2 CPU, 8 GiB memory, 20 GiB disk. +Worker rejections: insufficient memory (14), unsupported runtime (1). +Suggestions: Reduce the memory request, release idle sandboxes, or add workers +with more available memory. Select a supported runtime or add workers that +advertise the requested runtime. +``` + +When the snapshot still has eligible workers, `reasons` is `("queue_timeout",)` +and `resource_shortages` is empty: the queue budget expired, but resource shortage +is not established. Missing or unusable diagnostic fields produce +`reasons=("unknown",)` and retain the raw diagnostic for investigation. Older +services without a structured diagnosis keep their existing exception behavior. +In particular, `nodeCount > 0`, `eligibleNodes == 0` and an empty `rejections` +map produce `unknown`: workers exist, but the snapshot does not explain the +rejection. This case establishes neither `no_workers` nor a resource shortage. +Plain transport timeouts, initialization failures, backend rate limits and +uncertain outcomes do not become scheduling errors just because their message +mentions resources or a deadline. + +The service's `code`, `retry`, `outcome`, IDs and original `scheduling_failure` +are preserved; the native message remains in `__cause__`. No additional attempt, +backoff or queue budget is introduced. `retry="never"` means this queue wait is +finished; after addressing the blockers, callers may deliberately start a new +create request. Unknown outcomes keep their original same-operation semantics. + +Capacity and available maxima in the raw diagnostic span all workers, including +workers rejected by runtime or placement filters. Each resource maximum can +come from a different worker. They do not establish that a suitable worker can +run the request, or reliably distinguish temporary occupancy from an oversized +request on compatible workers. The default summary therefore presents the +requested resources and blockers without presenting those maxima as usable +capacity. Diagnostics describe the expiry snapshot, not capacity after scaling. + ### Client-managed Docker/Compose runtime diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index 8a934648..ffef49b7 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -30,11 +30,13 @@ BackendNotInstalledError, BackendOperationError, CommandExitException, + InsufficientResourcesError, InvalidBackendError, NotFoundException, PermissionDeniedException, RateLimitException, SandboxNotFoundException, + SchedulingError, ServiceBusyException, TimeoutException, UnsupportedBackendFeatureError, @@ -93,6 +95,8 @@ "BackendNotInstalledError", "UnsupportedBackendFeatureError", "BackendOperationError", + "SchedulingError", + "InsufficientResourcesError", "AuthenticationException", "PermissionDeniedException", "NotFoundException", diff --git a/sdk/python/akernel_sdk/_backends/_scheduling.py b/sdk/python/akernel_sdk/_backends/_scheduling.py new file mode 100644 index 00000000..505b5de2 --- /dev/null +++ b/sdk/python/akernel_sdk/_backends/_scheduling.py @@ -0,0 +1,187 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Explain a scheduling snapshot without inferring retry or execution state.""" + +from collections.abc import Mapping +from dataclasses import dataclass + +_RESOURCE_NAMES = {"cpu": "CPU", "memory": "memory", "disk": "disk"} +_PLACEMENT_HINT = "Relax placement constraints or add workers that satisfy them." +_OPERATOR_HINT = ( + "Inspect scheduling_failure with the cluster operator for remaining placement " + "constraints." +) +_BLOCKERS = { + "node_available": ( + "worker unavailable", + "Ask the cluster operator to check worker health, registration and " + "drain status.", + ), + "runtime_fit": ( + "unsupported runtime", + "Select a supported runtime or add workers that advertise the " + "requested runtime.", + ), + "device_fit": ( + "accelerator unavailable or incompatible", + "Request an available accelerator model/count or add compatible workers.", + ), + "node_affinity": ("node affinity mismatch", _PLACEMENT_HINT), + "placement_groups": ("placement group mismatch", _PLACEMENT_HINT), + "environment_affinity": ("environment affinity mismatch", _PLACEMENT_HINT), + "topology_spread": ("topology constraint", _PLACEMENT_HINT), + "waiting_for_volume": ( + "volume in use", + "Wait for the volume to be released or select a different volume.", + ), + "volume_topology": ( + "storage inaccessible from worker", + "Select workers with access to the required storage or adjust " + "volume placement.", + ), +} + + +@dataclass(frozen=True) +class SchedulingExplanation: + message: str + reasons: tuple[str, ...] + resource_shortages: tuple[str, ...] = () + suggestions: tuple[str, ...] = () + + +def _count(value: object) -> int | None: + # These are unsigned service counters/resources. Reject bool and malformed + # values rather than coercing them into a diagnosis or masking the failure. + if isinstance(value, int) and not isinstance(value, bool) and 0 <= value < 2**64: + return value + return None + + +def _bytes(value: int) -> str: + units = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB") + for power, unit in reversed(tuple(enumerate(units))): + scale = 1024**power + if value >= scale: + return f"{value / scale:g} {unit}" + return "0 B" + + +def _requested(details: Mapping[str, object]) -> str: + requested = details.get("requested") + if not isinstance(requested, Mapping): + return "" + parts = [] + for key, resource in ( + ("cpu_millis", "CPU"), + ("memory_bytes", "memory"), + ("disk_bytes", "disk"), + ): + value = _count(requested.get(key)) + if value is not None: + amount = f"{value / 1000:g}" if resource == "CPU" else _bytes(value) + parts.append(f"{amount} {resource}") + return " Requested: " + ", ".join(parts) + "." if parts else "" + + +def explain_scheduling_failure(details: object) -> SchedulingExplanation: + """Use only explicit current-snapshot evidence; retain raw details elsewhere.""" + unknown = SchedulingExplanation( + "the scheduling queue deadline expired without a usable placement diagnosis.", + ("unknown",), + suggestions=(_OPERATOR_HINT,), + ) + if not isinstance(details, Mapping): + return unknown + eligible = _count(details.get("eligibleNodes")) + nodes = _count(details.get("nodeCount")) + if eligible is None or (nodes is not None and eligible > nodes): + return unknown + if eligible > 0: + noun = "worker" if eligible == 1 else "workers" + return SchedulingExplanation( + f"the scheduling queue deadline expired with {eligible} eligible {noun}." + + _requested(details), + ("queue_timeout",), + suggestions=( + "Consider a larger schedule_timeout for a new create request and ask " + "the cluster operator to check scheduler load.", + ), + ) + rejections = details.get("rejections") + counts = ( + { + key: count + for key, value in rejections.items() + if isinstance(key, str) and key and (count := _count(value)) + } + if isinstance(rejections, Mapping) + else {} + ) + if nodes is not None and ( + any(count > nodes for key, count in counts.items() if key != "no_workers") + or (nodes > 0 and "no_workers" in counts) + ): + return unknown + if nodes == 0 or "no_workers" in counts: + hint = "Ask the cluster operator to register healthy workers." + return SchedulingExplanation( + "the cluster has no workers available for scheduling." + + _requested(details), + ("no_workers",), + suggestions=(hint,), + ) + if not counts: + return unknown + resources = tuple( + name for name in _RESOURCE_NAMES if f"insufficient_{name}" in counts + ) + if resources: + summary = ( + "the cluster has insufficient resources for this request; " + "no worker satisfies all resource and placement requirements." + ) + else: + summary = ( + "no worker satisfies the current availability or placement requirements." + ) + labels = [] + hints = [] + for resource in resources: + key = f"insufficient_{resource}" + name = _RESOURCE_NAMES[resource] + labels.append(f"insufficient {name} ({counts[key]})") + hints.append( + f"Reduce the {name} request, release idle sandboxes, or add workers " + f"with more available {name}." + ) + for reason, count in sorted(counts.items()): + if reason in {f"insufficient_{name}" for name in resources}: + continue + label, hint = _BLOCKERS.get( + reason, ("other placement constraint", _OPERATOR_HINT) + ) + labels.append(f"{label} ({count})") + hints.append(hint) + return SchedulingExplanation( + summary + + _requested(details) + + " Worker rejections: " + + ", ".join(labels) + + ".", + tuple(sorted(counts)), + resources, + tuple(dict.fromkeys(hints)), + ) diff --git a/sdk/python/akernel_sdk/_backends/adx.py b/sdk/python/akernel_sdk/_backends/adx.py index 71fa1934..6dc8a9c2 100644 --- a/sdk/python/akernel_sdk/_backends/adx.py +++ b/sdk/python/akernel_sdk/_backends/adx.py @@ -52,6 +52,7 @@ SnapshotInfo, TrafficPolicy, ) +from ._scheduling import explain_scheduling_failure from .base import ( Backend, BackendConfig, @@ -66,10 +67,12 @@ from .errors import ( AuthenticationException, BackendOperationError, + InsufficientResourcesError, NotFoundException, PermissionDeniedException, RateLimitException, SandboxNotFoundException, + SchedulingError, ServiceBusyException, TimeoutException, UnsupportedBackendFeatureError, @@ -292,6 +295,7 @@ def _convert_error(operation: str, error: Exception) -> BackendOperationError: TimeoutException, RateLimitException, ServiceBusyException, + SchedulingError, ), ): error_type = type(error) @@ -299,8 +303,28 @@ def _convert_error(operation: str, error: Exception) -> BackendOperationError: error_type = NotFoundException elif isinstance(error, (TimeoutError, httpx.TimeoutException)): error_type = TimeoutException + scheduling_failure = getattr(error, "scheduling_failure", None) + if ( + isinstance(scheduling_failure, dict) + and getattr(error, "code", None) == "DEADLINE_EXCEEDED" + and getattr(error, "retry", None) == "never" + and getattr(error, "outcome", None) == "not_started" + and status in (None, 504) + ): + explanation = explain_scheduling_failure(scheduling_failure) + error_type = ( + InsufficientResourcesError + if explanation.resource_shortages + else SchedulingError + ) + message = f"{operation} failed: {error}" + if issubclass(error_type, SchedulingError): + explanation = explain_scheduling_failure(scheduling_failure) + message = f"{operation} failed: {explanation.message}" + if explanation.suggestions: + message += " Suggestions: " + " ".join(explanation.suggestions) return error_type( - f"{operation} failed: {error}", + message, code=getattr(error, "code", None), retry=getattr(error, "retry", None), outcome=getattr(error, "outcome", None), diff --git a/sdk/python/akernel_sdk/_backends/errors.py b/sdk/python/akernel_sdk/_backends/errors.py index 29796779..86214db8 100644 --- a/sdk/python/akernel_sdk/_backends/errors.py +++ b/sdk/python/akernel_sdk/_backends/errors.py @@ -16,6 +16,8 @@ from typing import TYPE_CHECKING, Any +from ._scheduling import explain_scheduling_failure + if TYPE_CHECKING: from ..types import CommandResult @@ -111,6 +113,43 @@ class ServiceBusyException(BackendOperationError): """The service is temporarily unavailable; inspect retry/outcome context.""" +class SchedulingError(BackendOperationError): + """A service-confirmed scheduling cancellation with placement diagnostics. + + ``reasons`` contains positive rejection keys when no worker is eligible, + ``queue_timeout`` when workers are eligible, or ``unknown`` for unusable + diagnostics. ``resource_shortages`` lists contributing CPU/memory/disk + dimensions; other constraints may also contribute. ``suggestions`` offers + recovery advice without authorizing a retry or changing service context. + Original diagnostics remain in ``scheduling_failure`` and the native error + in ``__cause__``. Rejection counts can overlap and are not a node total. + """ + + @property + def reasons(self) -> tuple[str, ...]: + """Reported rejection keys, or the queue_timeout/unknown classification.""" + return explain_scheduling_failure(self.scheduling_failure).reasons + + @property + def resource_shortages(self) -> tuple[str, ...]: + """Contributing resources: cpu, memory and/or disk; empty when unconfirmed.""" + return explain_scheduling_failure(self.scheduling_failure).resource_shortages + + @property + def suggestions(self) -> tuple[str, ...]: + """Actions to consider before deliberately creating a new request.""" + return explain_scheduling_failure(self.scheduling_failure).suggestions + + +class InsufficientResourcesError(SchedulingError): + """No worker is eligible and CPU, memory or disk shortages contribute. + + Runtime, placement or volume constraints may also contribute; inspect + ``reasons`` and ``suggestions`` rather than assuming resources are the + sole blocker. The snapshot does not predict capacity after scaling. + """ + + class CommandExitException(BackendOperationError): """An explicitly checked command exited unsuccessfully.""" diff --git a/sdk/python/examples/sandbox_runtime.py b/sdk/python/examples/sandbox_runtime.py index 27036cc8..1c26725c 100644 --- a/sdk/python/examples/sandbox_runtime.py +++ b/sdk/python/examples/sandbox_runtime.py @@ -22,20 +22,29 @@ """ import os +import sys -from akernel_sdk import Sandbox +from akernel_sdk import Sandbox, SchedulingError def run(runtime: str | None) -> None: - sandbox = ( - Sandbox(cpu=1000, memory=2048) - if runtime is None - else Sandbox( - runtime=runtime, - cpu=1000, - memory=2048, + try: + sandbox = ( + Sandbox(cpu=1000, memory=2048) + if runtime is None + else Sandbox( + runtime=runtime, + cpu=1000, + memory=2048, + ) ) - ) + except SchedulingError as error: + print(error, file=sys.stderr) # Includes blockers and recovery suggestions. + if error.resource_shortages: + print("Unavailable resources:", error.resource_shortages, file=sys.stderr) + print("Request ID:", error.request_id, file=sys.stderr) + # Preserve the failed result; changing runtime or retrying is caller policy. + raise with sandbox: result = sandbox.commands.run("uname -s") assert result.exit_code == 0, result.stderr diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 4c0499c6..f871bfa5 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "akernel-sdk" -version = "0.2.0rc6" +version = "0.2.0rc7" description = "Python SDK and CLI for AKernel remote sandboxes" readme = "README.md" license = "Apache-2.0" diff --git a/sdk/python/tests/unit/test_scheduling_errors.py b/sdk/python/tests/unit/test_scheduling_errors.py new file mode 100644 index 00000000..544fb7d5 --- /dev/null +++ b/sdk/python/tests/unit/test_scheduling_errors.py @@ -0,0 +1,384 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Actionable scheduling failures retain authoritative retry/outcome semantics.""" + +import json +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import httpx +from adx_environment._transport import EnvironmentHTTPError + +from akernel_sdk import ( + BackendOperationError, + InsufficientResourcesError, + SchedulingError, + TimeoutException, +) +from akernel_sdk._addresses import Endpoint +from akernel_sdk._backends import adx +from akernel_sdk._backends.base import BackendConfig +from tests.unit.test_adx_backend import _spec + + +def diagnosis(**overrides): + result = { + "nodeCount": 15, + "eligibleNodes": 0, + "requested": { + "cpu_millis": 2000, + "memory_bytes": 8 * 1024**3, + "disk_bytes": 20 * 1024**3, + }, + "maxCapacity": {"memory_bytes": 32 * 1024**3}, + "maxAvailable": {"memory_bytes": 32 * 1024**3}, + "rejections": {"insufficient_memory": 14, "runtime_fit": 1}, + } + result.update(overrides) + return result + + +def native_error(details, **overrides): + context = { + "code": "DEADLINE_EXCEEDED", + "retry": "never", + "outcome": "not_started", + "request_id": "create-test", + "environment_id": "default-test", + "scheduling_failure": details, + } + context.update(overrides) + return EnvironmentHTTPError(504, {}, "native scheduling diagnostics", **context) + + +class SchedulingErrorTest(unittest.TestCase): + def convert(self, details, **overrides): + return adx._convert_error("create sandbox", native_error(details, **overrides)) + + def test_mixed_memory_runtime_failure_explains_all_blockers(self): + details = diagnosis() + error = self.convert(details) + self.assertIsInstance(error, InsufficientResourcesError) + self.assertIsInstance(error, SchedulingError) + self.assertIsInstance(error, BackendOperationError) + self.assertEqual(error.resource_shortages, ("memory",)) + self.assertEqual(error.reasons, ("insufficient_memory", "runtime_fit")) + text = str(error) + for phrase in ( + "insufficient resources", + "2 CPU", + "8 GiB memory", + "20 GiB disk", + "insufficient memory (14)", + "unsupported runtime (1)", + "release idle sandboxes", + "requested runtime", + ): + self.assertIn(phrase, text) + self.assertNotIn("32 GiB", text) + self.assertNotIn("native scheduling diagnostics", text) + self.assertIs(error.scheduling_failure, details) + + def test_each_scalar_shortage_is_actionable_and_preserves_units(self): + for resource in ("cpu", "memory", "disk"): + with self.subTest(resource=resource): + error = self.convert( + diagnosis( + rejections={f"insufficient_{resource}": 15}, + requested={ + "cpu_millis": 1500, + "memory_bytes": 512 * 1024**2, + "disk_bytes": 1536 * 1024**2, + }, + ) + ) + self.assertIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.resource_shortages, (resource,)) + self.assertIn("1.5 CPU, 512 MiB memory, 1.5 GiB disk", str(error)) + self.assertTrue( + any(resource in hint.lower() for hint in error.suggestions) + ) + + def test_nonresource_blockers_have_specific_guidance(self): + for reason, advice in ( + ("runtime_fit", "runtime"), + ("device_fit", "accelerator"), + ("node_available", "health"), + ("node_affinity", "placement"), + ("placement_groups", "placement"), + ("environment_affinity", "placement"), + ("topology_spread", "placement"), + ("waiting_for_volume", "volume"), + ("volume_topology", "storage"), + ): + with self.subTest(reason=reason): + error = self.convert(diagnosis(rejections={reason: 15})) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, (reason,)) + self.assertEqual(error.resource_shortages, ()) + self.assertTrue(any(advice in hint for hint in error.suggestions)) + self.assertNotIn("insufficient resources", str(error)) + + def test_no_workers_is_distinct_from_insufficient_capacity(self): + for reasons in ({}, {"no_workers": 1}): + with self.subTest(rejections=reasons): + error = self.convert(diagnosis(nodeCount=0, rejections=reasons)) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, ("no_workers",)) + self.assertIn("no workers", str(error)) + self.assertIn("register", " ".join(error.suggestions)) + + def test_eligible_workers_do_not_imply_cluster_shortage(self): + error = self.convert(diagnosis(eligibleNodes=1)) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, ("queue_timeout",)) + self.assertEqual(error.resource_shortages, ()) + self.assertIn("1 eligible worker", str(error)) + self.assertIn("schedule_timeout", " ".join(error.suggestions)) + self.assertNotIn("insufficient memory", str(error)) + + def test_zero_eligible_workers_without_rejections_remains_unexplained(self): + details = diagnosis(nodeCount=15, eligibleNodes=0, rejections={}) + error = self.convert(details) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, ("unknown",)) + self.assertEqual(error.resource_shortages, ()) + self.assertIs(error.scheduling_failure, details) + self.assertIn("without a usable placement diagnosis", str(error)) + self.assertIn("cluster operator", " ".join(error.suggestions)) + self.assertNotIn("no workers", str(error)) + self.assertEqual((error.retry, error.outcome), ("never", "not_started")) + + def test_incomplete_or_malformed_diagnoses_do_not_invent_shortages(self): + for details in ( + {}, + {"rejections": {"insufficient_memory": 3}}, + diagnosis(eligibleNodes=True), + diagnosis(eligibleNodes=-1), + diagnosis(eligibleNodes="0"), + diagnosis(eligibleNodes=16), + diagnosis(rejections=None), + diagnosis(rejections=["insufficient_memory"]), + diagnosis(rejections={"insufficient_memory": True}), + diagnosis(rejections={"insufficient_memory": -1}), + diagnosis(rejections={"insufficient_memory": 0}), + diagnosis(rejections={"insufficient_memory": "14"}), + diagnosis(rejections={"insufficient_memory": 16}), + diagnosis(rejections={"no_workers": 1}), + ): + with self.subTest(details=details): + error = self.convert(details) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.resource_shortages, ()) + self.assertEqual(error.reasons, ("unknown",)) + self.assertNotIn("insufficient memory", str(error)) + self.assertIs(error.scheduling_failure, details) + + def test_future_reasons_remain_inspectable_without_fabricated_advice(self): + error = self.convert(diagnosis(rejections={"future_filter": 15})) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, ("future_filter",)) + self.assertIn("other placement constraint", str(error)) + self.assertIn("cluster operator", " ".join(error.suggestions)) + + def test_bad_requested_resources_do_not_mask_a_valid_rejection(self): + for resources in (None, [], {"cpu_millis": True}, {"memory_bytes": -1}): + with self.subTest(resources=resources): + error = self.convert(diagnosis(requested=resources)) + self.assertIsInstance(error, InsufficientResourcesError) + self.assertIn("insufficient memory", str(error)) + self.assertNotIn("Requested:", str(error)) + + def test_only_authoritative_cancelled_scheduling_errors_are_classified(self): + for overrides in ( + {"retry": "same_operation", "outcome": "unknown"}, + {"retry": "after_backoff"}, + {"outcome": "terminal"}, + {"outcome": None}, + {"code": "RESOURCE_EXHAUSTED"}, + {"code": None}, + ): + with self.subTest(overrides=overrides): + error = self.convert(diagnosis(), **overrides) + self.assertNotIsInstance(error, SchedulingError) + self.assertIn("native scheduling diagnostics", str(error)) + for key, value in overrides.items(): + self.assertEqual(getattr(error, key), value) + for details in (None, [], "insufficient_memory"): + with self.subTest(details=details): + error = self.convert(details) + self.assertNotIsInstance(error, SchedulingError) + + def test_plain_timeouts_and_backend_limits_keep_existing_classification(self): + for native in ( + TimeoutError("transport timeout"), + EnvironmentHTTPError(504, {}, "initialization timeout"), + EnvironmentHTTPError(429, {}, "registry limit", code="RESOURCE_EXHAUSTED"), + ): + with self.subTest(native=native): + error = adx._convert_error("create sandbox", native) + self.assertNotIsInstance(error, SchedulingError) + self.assertIn(str(native), str(error)) + error = adx._convert_error("create sandbox", TimeoutError("transport timeout")) + self.assertIsInstance(error, TimeoutException) + + def test_public_scheduling_error_properties_are_safe_without_diagnostics(self): + error = SchedulingError("caller-supplied message") + self.assertEqual(error.reasons, ("unknown",)) + self.assertEqual(error.resource_shortages, ()) + self.assertEqual(str(error), "caller-supplied message") + + +class SchedulingTransportTest(unittest.TestCase): + def setUp(self): + self.config = BackendConfig( + api_endpoint=Endpoint("api.example", 443, "https", True), + gateway_endpoint=Endpoint("gateway.example", 8443, "https", True), + token="test-token", + ) + + def create_error(self, sse, **overrides): + calls = [] + context = { + "code": "DEADLINE_EXCEEDED", + "retry": "never", + "outcome": "not_started", + "requestId": "create-test", + "operationId": "operation-test", + "environmentId": "default-test", + "schedulingFailure": diagnosis(), + } + context.update(overrides) + + def reject(_client, request, **_kwargs): + calls.append(request) + payload = {"message": "native scheduling diagnostics", "error": context} + if sse: + payload.update(status="timeout", errorCode=504) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="event: final\ndata: " + json.dumps(payload) + "\n\n", + request=request, + ) + return httpx.Response(504, json=payload, request=request) + + # Exercise the native constructor, encoder and JSON/SSE parser; + # replace only network I/O and real backoff waiting. + with ( + patch.object(httpx.Client, "send", autospec=True, side_effect=reject), + patch("adx_environment._transport.time.sleep") as sleep, + self.assertRaises(BackendOperationError) as caught, + ): + adx.AdxBackend(self.config).create(_spec()) + return caught.exception, calls, sleep + + def test_pinned_client_json_and_sse_keep_single_attempt_and_full_context(self): + for sse in (False, True): + with self.subTest(sse=sse): + error, calls, sleep = self.create_error(sse) + self.assertIsInstance(error, InsufficientResourcesError) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].method, "POST") + sleep.assert_not_called() + self.assertIsInstance(error.__cause__, EnvironmentHTTPError) + self.assertEqual(error.scheduling_failure, diagnosis()) + self.assertEqual(error.status_code, 504) + self.assertEqual(error.code, "DEADLINE_EXCEEDED") + self.assertEqual(error.retry, "never") + self.assertEqual(error.outcome, "not_started") + self.assertEqual(error.request_id, "create-test") + self.assertEqual(error.operation_id, "operation-test") + self.assertEqual(error.environment_id, "default-test") + self.assertIn("native scheduling diagnostics", str(error.__cause__)) + + def test_uncertain_json_and_sse_keep_native_retries_and_request_identity(self): + for sse in (False, True): + with self.subTest(sse=sse): + error, calls, sleep = self.create_error( + sse, retry="same_operation", outcome="unknown" + ) + self.assertNotIsInstance(error, SchedulingError) + self.assertEqual(error.retry, "same_operation") + self.assertEqual(error.outcome, "unknown") + self.assertEqual(error.scheduling_failure, diagnosis()) + self.assertEqual(len(calls), 3) + self.assertEqual(sleep.call_count, 2) + for request in calls: + self.assertEqual(request.method, "POST") + self.assertEqual(request.content, calls[0].content) + self.assertEqual( + request.headers["x-request-id"], + calls[0].headers["x-request-id"], + ) + + def test_terminal_initialization_and_older_services_keep_native_errors(self): + for sse in (False, True): + for overrides in ({"outcome": "terminal"}, {"schedulingFailure": None}): + with self.subTest(sse=sse, overrides=overrides): + error, calls, sleep = self.create_error(sse, **overrides) + self.assertNotIsInstance(error, SchedulingError) + self.assertEqual(len(calls), 1) + sleep.assert_not_called() + self.assertIn("native scheduling diagnostics", str(error)) + + def test_wrapping_retains_secondary_context_and_scheduling_class(self): + native = native_error(diagnosis()) + native.cleanup_errors = (RuntimeError("cleanup failed"),) + native.first_error = RuntimeError("initial attempt") + native.failure = {"reason": "failure"} + native.probe_failure = {"kind": "probe"} + native.request_forwarded = False + native.command_id = "command-test" + native.command_result = SimpleNamespace(stdout="output") + with ( + patch.object(adx, "_OwnedSandbox", side_effect=native), + self.assertRaises(InsufficientResourcesError) as caught, + ): + adx.AdxBackend(self.config).create(_spec()) + public = caught.exception + self.assertIs(public.__cause__, native) + wrapped = adx._convert_error("restore snapshot", public) + self.assertIsInstance(wrapped, InsufficientResourcesError) + self.assertTrue(str(wrapped).startswith("restore snapshot failed:")) + for key in ( + "code", + "retry", + "outcome", + "request_id", + "operation_id", + "environment_id", + "status_code", + "scheduling_failure", + "cleanup_errors", + "first_error", + "failure", + "probe_failure", + "request_forwarded", + "command_id", + "command_result", + ): + self.assertEqual(getattr(wrapped, key), getattr(native, key, None), key) + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index fc783524..a0965ecc 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -40,6 +40,9 @@ def test_lightweight_imports_do_not_load_yuanrong(self): os.environ.pop('YR_HTTP_CONNECTION_NUM', None) import akernel_sdk import akernel_sdk.cli +from akernel_sdk import InsufficientResourcesError, SchedulingError +assert issubclass(InsufficientResourcesError, SchedulingError) +assert 'akernel_sdk._backends.adx' not in sys.modules assert 'yr' not in sys.modules assert 'yr_sandbox' not in sys.modules assert 'YR_HTTP_CONNECTION_NUM' not in os.environ @@ -104,6 +107,8 @@ def test_public_exports_are_minimal(self): "BackendNotInstalledError", "UnsupportedBackendFeatureError", "BackendOperationError", + "SchedulingError", + "InsufficientResourcesError", "AuthenticationException", "PermissionDeniedException", "NotFoundException",