From 8d60cb6bf0c106ac5b47d845352212a1da289921 Mon Sep 17 00:00:00 2001 From: wayne Date: Thu, 12 Nov 2026 18:00:31 +0800 Subject: [PATCH 1/2] feat(sdk): explain scheduling failures and resource shortages Expose SchedulingError and InsufficientResourcesError with readable resource requests, rejection reasons and recovery suggestions. Explain runtime, device, placement, worker and volume blockers alongside scalar shortages, without treating global capacity maxima as usable worker capacity. Classify only service-confirmed cancelled scheduling requests. Preserve native causes, diagnostics, identities and retry/outcome guidance, including the existing same-operation policy for uncertain results. Eligible workers and incomplete diagnoses must not imply a resource shortage. Cover the pinned client's JSON/SSE parsing, attempt counts and identities, public exports, diagnostic edge cases and installed distribution imports. Document the public exceptions and update the runtime example. Refs: #163 Signed-off-by: wayne --- .../actions/python-distributions/action.yml | 4 + AGENTS.md | 6 + sdk/python/README.md | 73 ++++ sdk/python/akernel_sdk/__init__.py | 4 + .../akernel_sdk/_backends/_scheduling.py | 187 +++++++++ sdk/python/akernel_sdk/_backends/adx.py | 26 +- sdk/python/akernel_sdk/_backends/errors.py | 39 ++ sdk/python/examples/sandbox_runtime.py | 27 +- .../tests/unit/test_scheduling_errors.py | 371 ++++++++++++++++++ sdk/python/tests/unit/test_types.py | 5 + 10 files changed, 732 insertions(+), 10 deletions(-) create mode 100644 sdk/python/akernel_sdk/_backends/_scheduling.py create mode 100644 sdk/python/tests/unit/test_scheduling_errors.py diff --git a/.github/actions/python-distributions/action.yml b/.github/actions/python-distributions/action.yml index fa3878cf..9a155ab9 100644 --- a/.github/actions/python-distributions/action.yml +++ b/.github/actions/python-distributions/action.yml @@ -61,9 +61,13 @@ runs: import adx_environment import akernel_sdk from akernel_sdk import ComposeProject, DockerRuntime, ServiceHandle, ServiceProcessHandle + from akernel_sdk import BackendOperationError, InsufficientResourcesError, SchedulingError from importlib.resources import files assert all(callable(value) for value in (ComposeProject, DockerRuntime, ServiceHandle, ServiceProcessHandle)) + assert issubclass(InsufficientResourcesError, SchedulingError) + assert issubclass(SchedulingError, BackendOperationError) + assert SchedulingError("smoke").reasons == ("unknown",) for helper in ("_image_bridge_guest.py", "_docker_network_guest.py", "_service_process_guest.py"): assert files("akernel_sdk").joinpath(helper).read_text() diff --git a/AGENTS.md b/AGENTS.md index 1a0ad9aa..efd195ba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -725,6 +725,12 @@ the pinned native PermissionDenied exception without inventing a missing status. `BackendOperationError` preserves optional structured service fields and the original cause; missing context remains `None`. Keep retry/outcome semantics backend-owned and document public error fields in `sdk/python/README.md`. +Confirmed cancelled queue requests with structured diagnostics use +`SchedulingError` and, when scalar shortages contribute with no eligible worker, +`InsufficientResourcesError`. Preserve raw diagnostics and native causes while +explaining all blockers; eligible workers or unusable diagnostics must not be +presented as confirmed resource shortages. Keep classification independent of +retry policy and do not infer usable capacity from global per-dimension maxima. Keep sandbox cleanup explicit through context managers or `kill()`, with observable, retryable deletion failures and workload exceptions preserved on diff --git a/sdk/python/README.md b/sdk/python/README.md index 6eab24e5..60fab1b8 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -1469,6 +1469,79 @@ original identity when querying or retrying. Do not infer execution or cleanup from a timeout message. `status_code` describes the service failure (including an SSE final event's error code); it need not equal the stream's HTTP status. +Confirmed scheduling cancellations with structured diagnostics raise +`SchedulingError`, a subclass of `BackendOperationError`. If no worker is +eligible and CPU, memory or disk shortages contribute, the exception is the +more specific `InsufficientResourcesError`, which also subclasses +`SchedulingError`. Existing `except BackendOperationError` handlers still work. +Both expose: + +- `reasons`: a tuple of positive worker rejection keys from the service, sorted + by key. Counts can overlap; their sum is not a worker count. Unknown future + rejection keys are retained. +- `resource_shortages`: contributing resource names (`"cpu"`, `"memory"`, + `"disk"`), in that order. Other constraints can also contribute. +- `suggestions`: a tuple of concrete recovery suggestions. The exception message + includes these suggestions, readable requested resources, and rejection counts. + +```python +from akernel_sdk import InsufficientResourcesError, Sandbox, SchedulingError + +try: + sandbox = Sandbox(cpu=2000, memory=8192, storage_mb=20480) +except InsufficientResourcesError as error: + print("Unavailable resources:", ", ".join(error.resource_shortages)) + print(error) # Summary and actions; this does not retry the request. + print("All placement blockers:", error.reasons) +except SchedulingError as error: + print(error) # Runtime, devices, placement, volumes, workers, or queue budget. +else: + sandbox.kill() +``` + +| Reported blocker | Action to consider | +| --- | --- | +| `insufficient_cpu`, `insufficient_memory`, `insufficient_disk` | Reduce the request, release idle sandboxes, or add suitable worker capacity. | +| `runtime_fit`, `device_fit` | Select a supported runtime or available accelerator model/count, or add compatible workers. | +| `node_available`, `no_workers` | Ask the operator to check worker health/registration and register healthy workers. | +| `node_affinity`, `placement_groups`, `environment_affinity`, `topology_spread` | Adjust placement constraints or add workers that satisfy them. | +| `waiting_for_volume`, `volume_topology` | Wait for the volume to be released or select workers with access to its storage. | + +For example, a mixed rejection produces a message like: + +```text +create sandbox failed: the cluster has insufficient resources for this request; +no worker satisfies all resource and placement requirements. +Requested: 2 CPU, 8 GiB memory, 20 GiB disk. +Worker rejections: insufficient memory (14), unsupported runtime (1). +Suggestions: Reduce the memory request, release idle sandboxes, or add workers +with more available memory. Select a supported runtime or add workers that +advertise the requested runtime. +``` + +When the snapshot still has eligible workers, `reasons` is `("queue_timeout",)` +and `resource_shortages` is empty: the queue budget expired, but resource shortage +is not established. Missing or unusable diagnostic fields produce +`reasons=("unknown",)` and retain the raw diagnostic for investigation. Older +services without a structured diagnosis keep their existing exception behavior. +Plain transport timeouts, initialization failures, backend rate limits and +uncertain outcomes do not become scheduling errors just because their message +mentions resources or a deadline. + +The service's `code`, `retry`, `outcome`, IDs and original `scheduling_failure` +are preserved; the native message remains in `__cause__`. No additional attempt, +backoff or queue budget is introduced. `retry="never"` means this queue wait is +finished; after addressing the blockers, callers may deliberately start a new +create request. Unknown outcomes keep their original same-operation semantics. + +Capacity and available maxima in the raw diagnostic span all workers, including +workers rejected by runtime or placement filters. Each resource maximum can +come from a different worker. They do not establish that a suitable worker can +run the request, or reliably distinguish temporary occupancy from an oversized +request on compatible workers. The default summary therefore presents the +requested resources and blockers without presenting those maxima as usable +capacity. Diagnostics describe the expiry snapshot, not capacity after scaling. + ### Client-managed Docker/Compose runtime diff --git a/sdk/python/akernel_sdk/__init__.py b/sdk/python/akernel_sdk/__init__.py index 8a934648..ffef49b7 100644 --- a/sdk/python/akernel_sdk/__init__.py +++ b/sdk/python/akernel_sdk/__init__.py @@ -30,11 +30,13 @@ BackendNotInstalledError, BackendOperationError, CommandExitException, + InsufficientResourcesError, InvalidBackendError, NotFoundException, PermissionDeniedException, RateLimitException, SandboxNotFoundException, + SchedulingError, ServiceBusyException, TimeoutException, UnsupportedBackendFeatureError, @@ -93,6 +95,8 @@ "BackendNotInstalledError", "UnsupportedBackendFeatureError", "BackendOperationError", + "SchedulingError", + "InsufficientResourcesError", "AuthenticationException", "PermissionDeniedException", "NotFoundException", diff --git a/sdk/python/akernel_sdk/_backends/_scheduling.py b/sdk/python/akernel_sdk/_backends/_scheduling.py new file mode 100644 index 00000000..505b5de2 --- /dev/null +++ b/sdk/python/akernel_sdk/_backends/_scheduling.py @@ -0,0 +1,187 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Explain a scheduling snapshot without inferring retry or execution state.""" + +from collections.abc import Mapping +from dataclasses import dataclass + +_RESOURCE_NAMES = {"cpu": "CPU", "memory": "memory", "disk": "disk"} +_PLACEMENT_HINT = "Relax placement constraints or add workers that satisfy them." +_OPERATOR_HINT = ( + "Inspect scheduling_failure with the cluster operator for remaining placement " + "constraints." +) +_BLOCKERS = { + "node_available": ( + "worker unavailable", + "Ask the cluster operator to check worker health, registration and " + "drain status.", + ), + "runtime_fit": ( + "unsupported runtime", + "Select a supported runtime or add workers that advertise the " + "requested runtime.", + ), + "device_fit": ( + "accelerator unavailable or incompatible", + "Request an available accelerator model/count or add compatible workers.", + ), + "node_affinity": ("node affinity mismatch", _PLACEMENT_HINT), + "placement_groups": ("placement group mismatch", _PLACEMENT_HINT), + "environment_affinity": ("environment affinity mismatch", _PLACEMENT_HINT), + "topology_spread": ("topology constraint", _PLACEMENT_HINT), + "waiting_for_volume": ( + "volume in use", + "Wait for the volume to be released or select a different volume.", + ), + "volume_topology": ( + "storage inaccessible from worker", + "Select workers with access to the required storage or adjust " + "volume placement.", + ), +} + + +@dataclass(frozen=True) +class SchedulingExplanation: + message: str + reasons: tuple[str, ...] + resource_shortages: tuple[str, ...] = () + suggestions: tuple[str, ...] = () + + +def _count(value: object) -> int | None: + # These are unsigned service counters/resources. Reject bool and malformed + # values rather than coercing them into a diagnosis or masking the failure. + if isinstance(value, int) and not isinstance(value, bool) and 0 <= value < 2**64: + return value + return None + + +def _bytes(value: int) -> str: + units = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB") + for power, unit in reversed(tuple(enumerate(units))): + scale = 1024**power + if value >= scale: + return f"{value / scale:g} {unit}" + return "0 B" + + +def _requested(details: Mapping[str, object]) -> str: + requested = details.get("requested") + if not isinstance(requested, Mapping): + return "" + parts = [] + for key, resource in ( + ("cpu_millis", "CPU"), + ("memory_bytes", "memory"), + ("disk_bytes", "disk"), + ): + value = _count(requested.get(key)) + if value is not None: + amount = f"{value / 1000:g}" if resource == "CPU" else _bytes(value) + parts.append(f"{amount} {resource}") + return " Requested: " + ", ".join(parts) + "." if parts else "" + + +def explain_scheduling_failure(details: object) -> SchedulingExplanation: + """Use only explicit current-snapshot evidence; retain raw details elsewhere.""" + unknown = SchedulingExplanation( + "the scheduling queue deadline expired without a usable placement diagnosis.", + ("unknown",), + suggestions=(_OPERATOR_HINT,), + ) + if not isinstance(details, Mapping): + return unknown + eligible = _count(details.get("eligibleNodes")) + nodes = _count(details.get("nodeCount")) + if eligible is None or (nodes is not None and eligible > nodes): + return unknown + if eligible > 0: + noun = "worker" if eligible == 1 else "workers" + return SchedulingExplanation( + f"the scheduling queue deadline expired with {eligible} eligible {noun}." + + _requested(details), + ("queue_timeout",), + suggestions=( + "Consider a larger schedule_timeout for a new create request and ask " + "the cluster operator to check scheduler load.", + ), + ) + rejections = details.get("rejections") + counts = ( + { + key: count + for key, value in rejections.items() + if isinstance(key, str) and key and (count := _count(value)) + } + if isinstance(rejections, Mapping) + else {} + ) + if nodes is not None and ( + any(count > nodes for key, count in counts.items() if key != "no_workers") + or (nodes > 0 and "no_workers" in counts) + ): + return unknown + if nodes == 0 or "no_workers" in counts: + hint = "Ask the cluster operator to register healthy workers." + return SchedulingExplanation( + "the cluster has no workers available for scheduling." + + _requested(details), + ("no_workers",), + suggestions=(hint,), + ) + if not counts: + return unknown + resources = tuple( + name for name in _RESOURCE_NAMES if f"insufficient_{name}" in counts + ) + if resources: + summary = ( + "the cluster has insufficient resources for this request; " + "no worker satisfies all resource and placement requirements." + ) + else: + summary = ( + "no worker satisfies the current availability or placement requirements." + ) + labels = [] + hints = [] + for resource in resources: + key = f"insufficient_{resource}" + name = _RESOURCE_NAMES[resource] + labels.append(f"insufficient {name} ({counts[key]})") + hints.append( + f"Reduce the {name} request, release idle sandboxes, or add workers " + f"with more available {name}." + ) + for reason, count in sorted(counts.items()): + if reason in {f"insufficient_{name}" for name in resources}: + continue + label, hint = _BLOCKERS.get( + reason, ("other placement constraint", _OPERATOR_HINT) + ) + labels.append(f"{label} ({count})") + hints.append(hint) + return SchedulingExplanation( + summary + + _requested(details) + + " Worker rejections: " + + ", ".join(labels) + + ".", + tuple(sorted(counts)), + resources, + tuple(dict.fromkeys(hints)), + ) diff --git a/sdk/python/akernel_sdk/_backends/adx.py b/sdk/python/akernel_sdk/_backends/adx.py index 71fa1934..6dc8a9c2 100644 --- a/sdk/python/akernel_sdk/_backends/adx.py +++ b/sdk/python/akernel_sdk/_backends/adx.py @@ -52,6 +52,7 @@ SnapshotInfo, TrafficPolicy, ) +from ._scheduling import explain_scheduling_failure from .base import ( Backend, BackendConfig, @@ -66,10 +67,12 @@ from .errors import ( AuthenticationException, BackendOperationError, + InsufficientResourcesError, NotFoundException, PermissionDeniedException, RateLimitException, SandboxNotFoundException, + SchedulingError, ServiceBusyException, TimeoutException, UnsupportedBackendFeatureError, @@ -292,6 +295,7 @@ def _convert_error(operation: str, error: Exception) -> BackendOperationError: TimeoutException, RateLimitException, ServiceBusyException, + SchedulingError, ), ): error_type = type(error) @@ -299,8 +303,28 @@ def _convert_error(operation: str, error: Exception) -> BackendOperationError: error_type = NotFoundException elif isinstance(error, (TimeoutError, httpx.TimeoutException)): error_type = TimeoutException + scheduling_failure = getattr(error, "scheduling_failure", None) + if ( + isinstance(scheduling_failure, dict) + and getattr(error, "code", None) == "DEADLINE_EXCEEDED" + and getattr(error, "retry", None) == "never" + and getattr(error, "outcome", None) == "not_started" + and status in (None, 504) + ): + explanation = explain_scheduling_failure(scheduling_failure) + error_type = ( + InsufficientResourcesError + if explanation.resource_shortages + else SchedulingError + ) + message = f"{operation} failed: {error}" + if issubclass(error_type, SchedulingError): + explanation = explain_scheduling_failure(scheduling_failure) + message = f"{operation} failed: {explanation.message}" + if explanation.suggestions: + message += " Suggestions: " + " ".join(explanation.suggestions) return error_type( - f"{operation} failed: {error}", + message, code=getattr(error, "code", None), retry=getattr(error, "retry", None), outcome=getattr(error, "outcome", None), diff --git a/sdk/python/akernel_sdk/_backends/errors.py b/sdk/python/akernel_sdk/_backends/errors.py index 29796779..86214db8 100644 --- a/sdk/python/akernel_sdk/_backends/errors.py +++ b/sdk/python/akernel_sdk/_backends/errors.py @@ -16,6 +16,8 @@ from typing import TYPE_CHECKING, Any +from ._scheduling import explain_scheduling_failure + if TYPE_CHECKING: from ..types import CommandResult @@ -111,6 +113,43 @@ class ServiceBusyException(BackendOperationError): """The service is temporarily unavailable; inspect retry/outcome context.""" +class SchedulingError(BackendOperationError): + """A service-confirmed scheduling cancellation with placement diagnostics. + + ``reasons`` contains positive rejection keys when no worker is eligible, + ``queue_timeout`` when workers are eligible, or ``unknown`` for unusable + diagnostics. ``resource_shortages`` lists contributing CPU/memory/disk + dimensions; other constraints may also contribute. ``suggestions`` offers + recovery advice without authorizing a retry or changing service context. + Original diagnostics remain in ``scheduling_failure`` and the native error + in ``__cause__``. Rejection counts can overlap and are not a node total. + """ + + @property + def reasons(self) -> tuple[str, ...]: + """Reported rejection keys, or the queue_timeout/unknown classification.""" + return explain_scheduling_failure(self.scheduling_failure).reasons + + @property + def resource_shortages(self) -> tuple[str, ...]: + """Contributing resources: cpu, memory and/or disk; empty when unconfirmed.""" + return explain_scheduling_failure(self.scheduling_failure).resource_shortages + + @property + def suggestions(self) -> tuple[str, ...]: + """Actions to consider before deliberately creating a new request.""" + return explain_scheduling_failure(self.scheduling_failure).suggestions + + +class InsufficientResourcesError(SchedulingError): + """No worker is eligible and CPU, memory or disk shortages contribute. + + Runtime, placement or volume constraints may also contribute; inspect + ``reasons`` and ``suggestions`` rather than assuming resources are the + sole blocker. The snapshot does not predict capacity after scaling. + """ + + class CommandExitException(BackendOperationError): """An explicitly checked command exited unsuccessfully.""" diff --git a/sdk/python/examples/sandbox_runtime.py b/sdk/python/examples/sandbox_runtime.py index 27036cc8..1c26725c 100644 --- a/sdk/python/examples/sandbox_runtime.py +++ b/sdk/python/examples/sandbox_runtime.py @@ -22,20 +22,29 @@ """ import os +import sys -from akernel_sdk import Sandbox +from akernel_sdk import Sandbox, SchedulingError def run(runtime: str | None) -> None: - sandbox = ( - Sandbox(cpu=1000, memory=2048) - if runtime is None - else Sandbox( - runtime=runtime, - cpu=1000, - memory=2048, + try: + sandbox = ( + Sandbox(cpu=1000, memory=2048) + if runtime is None + else Sandbox( + runtime=runtime, + cpu=1000, + memory=2048, + ) ) - ) + except SchedulingError as error: + print(error, file=sys.stderr) # Includes blockers and recovery suggestions. + if error.resource_shortages: + print("Unavailable resources:", error.resource_shortages, file=sys.stderr) + print("Request ID:", error.request_id, file=sys.stderr) + # Preserve the failed result; changing runtime or retrying is caller policy. + raise with sandbox: result = sandbox.commands.run("uname -s") assert result.exit_code == 0, result.stderr diff --git a/sdk/python/tests/unit/test_scheduling_errors.py b/sdk/python/tests/unit/test_scheduling_errors.py new file mode 100644 index 00000000..d27fcd3a --- /dev/null +++ b/sdk/python/tests/unit/test_scheduling_errors.py @@ -0,0 +1,371 @@ +# Copyright (c) 2026 Ant Group Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Actionable scheduling failures retain authoritative retry/outcome semantics.""" + +import json +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import httpx +from adx_environment._transport import EnvironmentHTTPError + +from akernel_sdk import ( + BackendOperationError, + InsufficientResourcesError, + SchedulingError, + TimeoutException, +) +from akernel_sdk._addresses import Endpoint +from akernel_sdk._backends import adx +from akernel_sdk._backends.base import BackendConfig +from tests.unit.test_adx_backend import _spec + + +def diagnosis(**overrides): + result = { + "nodeCount": 15, + "eligibleNodes": 0, + "requested": { + "cpu_millis": 2000, + "memory_bytes": 8 * 1024**3, + "disk_bytes": 20 * 1024**3, + }, + "maxCapacity": {"memory_bytes": 32 * 1024**3}, + "maxAvailable": {"memory_bytes": 32 * 1024**3}, + "rejections": {"insufficient_memory": 14, "runtime_fit": 1}, + } + result.update(overrides) + return result + + +def native_error(details, **overrides): + context = { + "code": "DEADLINE_EXCEEDED", + "retry": "never", + "outcome": "not_started", + "request_id": "create-test", + "environment_id": "default-test", + "scheduling_failure": details, + } + context.update(overrides) + return EnvironmentHTTPError(504, {}, "native scheduling diagnostics", **context) + + +class SchedulingErrorTest(unittest.TestCase): + def convert(self, details, **overrides): + return adx._convert_error("create sandbox", native_error(details, **overrides)) + + def test_mixed_memory_runtime_failure_explains_all_blockers(self): + details = diagnosis() + error = self.convert(details) + self.assertIsInstance(error, InsufficientResourcesError) + self.assertIsInstance(error, SchedulingError) + self.assertIsInstance(error, BackendOperationError) + self.assertEqual(error.resource_shortages, ("memory",)) + self.assertEqual(error.reasons, ("insufficient_memory", "runtime_fit")) + text = str(error) + for phrase in ( + "insufficient resources", + "2 CPU", + "8 GiB memory", + "20 GiB disk", + "insufficient memory (14)", + "unsupported runtime (1)", + "release idle sandboxes", + "requested runtime", + ): + self.assertIn(phrase, text) + self.assertNotIn("32 GiB", text) + self.assertNotIn("native scheduling diagnostics", text) + self.assertIs(error.scheduling_failure, details) + + def test_each_scalar_shortage_is_actionable_and_preserves_units(self): + for resource in ("cpu", "memory", "disk"): + with self.subTest(resource=resource): + error = self.convert( + diagnosis( + rejections={f"insufficient_{resource}": 15}, + requested={ + "cpu_millis": 1500, + "memory_bytes": 512 * 1024**2, + "disk_bytes": 1536 * 1024**2, + }, + ) + ) + self.assertIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.resource_shortages, (resource,)) + self.assertIn("1.5 CPU, 512 MiB memory, 1.5 GiB disk", str(error)) + self.assertTrue( + any(resource in hint.lower() for hint in error.suggestions) + ) + + def test_nonresource_blockers_have_specific_guidance(self): + for reason, advice in ( + ("runtime_fit", "runtime"), + ("device_fit", "accelerator"), + ("node_available", "health"), + ("node_affinity", "placement"), + ("placement_groups", "placement"), + ("environment_affinity", "placement"), + ("topology_spread", "placement"), + ("waiting_for_volume", "volume"), + ("volume_topology", "storage"), + ): + with self.subTest(reason=reason): + error = self.convert(diagnosis(rejections={reason: 15})) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, (reason,)) + self.assertEqual(error.resource_shortages, ()) + self.assertTrue(any(advice in hint for hint in error.suggestions)) + self.assertNotIn("insufficient resources", str(error)) + + def test_no_workers_is_distinct_from_insufficient_capacity(self): + for reasons in ({}, {"no_workers": 1}): + with self.subTest(rejections=reasons): + error = self.convert(diagnosis(nodeCount=0, rejections=reasons)) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, ("no_workers",)) + self.assertIn("no workers", str(error)) + self.assertIn("register", " ".join(error.suggestions)) + + def test_eligible_workers_do_not_imply_cluster_shortage(self): + error = self.convert(diagnosis(eligibleNodes=1)) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, ("queue_timeout",)) + self.assertEqual(error.resource_shortages, ()) + self.assertIn("1 eligible worker", str(error)) + self.assertIn("schedule_timeout", " ".join(error.suggestions)) + self.assertNotIn("insufficient memory", str(error)) + + def test_incomplete_or_malformed_diagnoses_do_not_invent_shortages(self): + for details in ( + {}, + {"rejections": {"insufficient_memory": 3}}, + diagnosis(eligibleNodes=True), + diagnosis(eligibleNodes=-1), + diagnosis(eligibleNodes="0"), + diagnosis(eligibleNodes=16), + diagnosis(rejections=None), + diagnosis(rejections=["insufficient_memory"]), + diagnosis(rejections={"insufficient_memory": True}), + diagnosis(rejections={"insufficient_memory": -1}), + diagnosis(rejections={"insufficient_memory": 0}), + diagnosis(rejections={"insufficient_memory": "14"}), + diagnosis(rejections={"insufficient_memory": 16}), + diagnosis(rejections={"no_workers": 1}), + ): + with self.subTest(details=details): + error = self.convert(details) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.resource_shortages, ()) + self.assertEqual(error.reasons, ("unknown",)) + self.assertNotIn("insufficient memory", str(error)) + self.assertIs(error.scheduling_failure, details) + + def test_future_reasons_remain_inspectable_without_fabricated_advice(self): + error = self.convert(diagnosis(rejections={"future_filter": 15})) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, ("future_filter",)) + self.assertIn("other placement constraint", str(error)) + self.assertIn("cluster operator", " ".join(error.suggestions)) + + def test_bad_requested_resources_do_not_mask_a_valid_rejection(self): + for resources in (None, [], {"cpu_millis": True}, {"memory_bytes": -1}): + with self.subTest(resources=resources): + error = self.convert(diagnosis(requested=resources)) + self.assertIsInstance(error, InsufficientResourcesError) + self.assertIn("insufficient memory", str(error)) + self.assertNotIn("Requested:", str(error)) + + def test_only_authoritative_cancelled_scheduling_errors_are_classified(self): + for overrides in ( + {"retry": "same_operation", "outcome": "unknown"}, + {"retry": "after_backoff"}, + {"outcome": "terminal"}, + {"outcome": None}, + {"code": "RESOURCE_EXHAUSTED"}, + {"code": None}, + ): + with self.subTest(overrides=overrides): + error = self.convert(diagnosis(), **overrides) + self.assertNotIsInstance(error, SchedulingError) + self.assertIn("native scheduling diagnostics", str(error)) + for key, value in overrides.items(): + self.assertEqual(getattr(error, key), value) + for details in (None, [], "insufficient_memory"): + with self.subTest(details=details): + error = self.convert(details) + self.assertNotIsInstance(error, SchedulingError) + + def test_plain_timeouts_and_backend_limits_keep_existing_classification(self): + for native in ( + TimeoutError("transport timeout"), + EnvironmentHTTPError(504, {}, "initialization timeout"), + EnvironmentHTTPError(429, {}, "registry limit", code="RESOURCE_EXHAUSTED"), + ): + with self.subTest(native=native): + error = adx._convert_error("create sandbox", native) + self.assertNotIsInstance(error, SchedulingError) + self.assertIn(str(native), str(error)) + error = adx._convert_error("create sandbox", TimeoutError("transport timeout")) + self.assertIsInstance(error, TimeoutException) + + def test_public_scheduling_error_properties_are_safe_without_diagnostics(self): + error = SchedulingError("caller-supplied message") + self.assertEqual(error.reasons, ("unknown",)) + self.assertEqual(error.resource_shortages, ()) + self.assertEqual(str(error), "caller-supplied message") + + +class SchedulingTransportTest(unittest.TestCase): + def setUp(self): + self.config = BackendConfig( + api_endpoint=Endpoint("api.example", 443, "https", True), + gateway_endpoint=Endpoint("gateway.example", 8443, "https", True), + token="test-token", + ) + + def create_error(self, sse, **overrides): + calls = [] + context = { + "code": "DEADLINE_EXCEEDED", + "retry": "never", + "outcome": "not_started", + "requestId": "create-test", + "operationId": "operation-test", + "environmentId": "default-test", + "schedulingFailure": diagnosis(), + } + context.update(overrides) + + def reject(_client, request, **_kwargs): + calls.append(request) + payload = {"message": "native scheduling diagnostics", "error": context} + if sse: + payload.update(status="timeout", errorCode=504) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="event: final\ndata: " + json.dumps(payload) + "\n\n", + request=request, + ) + return httpx.Response(504, json=payload, request=request) + + # Exercise the native constructor, encoder and JSON/SSE parser; + # replace only network I/O and real backoff waiting. + with ( + patch.object(httpx.Client, "send", autospec=True, side_effect=reject), + patch("adx_environment._transport.time.sleep") as sleep, + self.assertRaises(BackendOperationError) as caught, + ): + adx.AdxBackend(self.config).create(_spec()) + return caught.exception, calls, sleep + + def test_pinned_client_json_and_sse_keep_single_attempt_and_full_context(self): + for sse in (False, True): + with self.subTest(sse=sse): + error, calls, sleep = self.create_error(sse) + self.assertIsInstance(error, InsufficientResourcesError) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].method, "POST") + sleep.assert_not_called() + self.assertIsInstance(error.__cause__, EnvironmentHTTPError) + self.assertEqual(error.scheduling_failure, diagnosis()) + self.assertEqual(error.status_code, 504) + self.assertEqual(error.code, "DEADLINE_EXCEEDED") + self.assertEqual(error.retry, "never") + self.assertEqual(error.outcome, "not_started") + self.assertEqual(error.request_id, "create-test") + self.assertEqual(error.operation_id, "operation-test") + self.assertEqual(error.environment_id, "default-test") + self.assertIn("native scheduling diagnostics", str(error.__cause__)) + + def test_uncertain_json_and_sse_keep_native_retries_and_request_identity(self): + for sse in (False, True): + with self.subTest(sse=sse): + error, calls, sleep = self.create_error( + sse, retry="same_operation", outcome="unknown" + ) + self.assertNotIsInstance(error, SchedulingError) + self.assertEqual(error.retry, "same_operation") + self.assertEqual(error.outcome, "unknown") + self.assertEqual(error.scheduling_failure, diagnosis()) + self.assertEqual(len(calls), 3) + self.assertEqual(sleep.call_count, 2) + for request in calls: + self.assertEqual(request.method, "POST") + self.assertEqual(request.content, calls[0].content) + self.assertEqual( + request.headers["x-request-id"], + calls[0].headers["x-request-id"], + ) + + def test_terminal_initialization_and_older_services_keep_native_errors(self): + for sse in (False, True): + for overrides in ({"outcome": "terminal"}, {"schedulingFailure": None}): + with self.subTest(sse=sse, overrides=overrides): + error, calls, sleep = self.create_error(sse, **overrides) + self.assertNotIsInstance(error, SchedulingError) + self.assertEqual(len(calls), 1) + sleep.assert_not_called() + self.assertIn("native scheduling diagnostics", str(error)) + + def test_wrapping_retains_secondary_context_and_scheduling_class(self): + native = native_error(diagnosis()) + native.cleanup_errors = (RuntimeError("cleanup failed"),) + native.first_error = RuntimeError("initial attempt") + native.failure = {"reason": "failure"} + native.probe_failure = {"kind": "probe"} + native.request_forwarded = False + native.command_id = "command-test" + native.command_result = SimpleNamespace(stdout="output") + with ( + patch.object(adx, "_OwnedSandbox", side_effect=native), + self.assertRaises(InsufficientResourcesError) as caught, + ): + adx.AdxBackend(self.config).create(_spec()) + public = caught.exception + self.assertIs(public.__cause__, native) + wrapped = adx._convert_error("restore snapshot", public) + self.assertIsInstance(wrapped, InsufficientResourcesError) + self.assertTrue(str(wrapped).startswith("restore snapshot failed:")) + for key in ( + "code", + "retry", + "outcome", + "request_id", + "operation_id", + "environment_id", + "status_code", + "scheduling_failure", + "cleanup_errors", + "first_error", + "failure", + "probe_failure", + "request_forwarded", + "command_id", + "command_result", + ): + self.assertEqual(getattr(wrapped, key), getattr(native, key, None), key) + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk/python/tests/unit/test_types.py b/sdk/python/tests/unit/test_types.py index fc783524..a0965ecc 100644 --- a/sdk/python/tests/unit/test_types.py +++ b/sdk/python/tests/unit/test_types.py @@ -40,6 +40,9 @@ def test_lightweight_imports_do_not_load_yuanrong(self): os.environ.pop('YR_HTTP_CONNECTION_NUM', None) import akernel_sdk import akernel_sdk.cli +from akernel_sdk import InsufficientResourcesError, SchedulingError +assert issubclass(InsufficientResourcesError, SchedulingError) +assert 'akernel_sdk._backends.adx' not in sys.modules assert 'yr' not in sys.modules assert 'yr_sandbox' not in sys.modules assert 'YR_HTTP_CONNECTION_NUM' not in os.environ @@ -104,6 +107,8 @@ def test_public_exports_are_minimal(self): "BackendNotInstalledError", "UnsupportedBackendFeatureError", "BackendOperationError", + "SchedulingError", + "InsufficientResourcesError", "AuthenticationException", "PermissionDeniedException", "NotFoundException", From 7aca1416ec5263f0c617aceecee186e7469c3f28 Mon Sep 17 00:00:00 2001 From: wayne Date: Thu, 12 Nov 2026 22:11:00 +0800 Subject: [PATCH 2/2] chore(sdk): prepare 0.2.0rc7 Document the retained scheduling diagnostic schema and its resource units, overlapping rejection counts and service-side extension boundary. Cover the deliberate unknown diagnosis when workers exist but none is eligible and no rejection reasons are reported. Advance the SDK package version to 0.2.0rc7 without changing the classifier, ADX dependency or retry/outcome semantics. Validate the full SDK quality gate and independently installed wheel and source distributions. Signed-off-by: wayne --- sdk/python/README.md | 27 ++++++++++++++++++- sdk/python/pyproject.toml | 2 +- .../tests/unit/test_scheduling_errors.py | 13 +++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/sdk/python/README.md b/sdk/python/README.md index 60fab1b8..09f6e586 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -1474,7 +1474,29 @@ Confirmed scheduling cancellations with structured diagnostics raise eligible and CPU, memory or disk shortages contribute, the exception is the more specific `InsufficientResourcesError`, which also subclasses `SchedulingError`. Existing `except BackendOperationError` handlers still work. -Both expose: + +The service's `error.schedulingFailure` object is retained as +`error.scheduling_failure`, with its original field names and values: + +| Field | Meaning | +| --- | --- | +| `snapshotRevision` | Revision of the scheduling snapshot at queue expiry; retained for diagnostics. | +| `nodeCount` | Number of workers known to the scheduler. Zero establishes `no_workers`. | +| `eligibleNodes` | Number passing all hard placement checks in that snapshot. A positive value produces `queue_timeout`. | +| `requested` | Requested resources: `cpu_millis` in millicores, `memory_bytes` and `disk_bytes` in bytes. | +| `maxCapacity`, `maxAvailable` | Resource objects with the same keys/units. Maxima span workers; available values already subtract reservations. | +| `rejections` | Map of rejection keys to worker counts. Counts may overlap; zero counts do not establish blockers. | + +Counters and resource amounts use unsigned 64-bit integers; booleans, negative +values and numeric strings are not coerced into valid evidence. Missing or +unusable eligibility data cannot establish a resource shortage; classification +also requires valid positive resource-rejection counts. Additional fields and +future rejection keys remain available in the original object. +Keep schema extensions aligned with ADX's `platform/api/http/error-contract.md`; +adding compatible-worker capacity diagnostics requires a service-side contract +before the SDK can distinguish temporary occupancy from an oversized request. + +The scheduling exceptions expose: - `reasons`: a tuple of positive worker rejection keys from the service, sorted by key. Counts can overlap; their sum is not a worker count. Unknown future @@ -1524,6 +1546,9 @@ and `resource_shortages` is empty: the queue budget expired, but resource shorta is not established. Missing or unusable diagnostic fields produce `reasons=("unknown",)` and retain the raw diagnostic for investigation. Older services without a structured diagnosis keep their existing exception behavior. +In particular, `nodeCount > 0`, `eligibleNodes == 0` and an empty `rejections` +map produce `unknown`: workers exist, but the snapshot does not explain the +rejection. This case establishes neither `no_workers` nor a resource shortage. Plain transport timeouts, initialization failures, backend rate limits and uncertain outcomes do not become scheduling errors just because their message mentions resources or a deadline. diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 4c0499c6..f871bfa5 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "akernel-sdk" -version = "0.2.0rc6" +version = "0.2.0rc7" description = "Python SDK and CLI for AKernel remote sandboxes" readme = "README.md" license = "Apache-2.0" diff --git a/sdk/python/tests/unit/test_scheduling_errors.py b/sdk/python/tests/unit/test_scheduling_errors.py index d27fcd3a..544fb7d5 100644 --- a/sdk/python/tests/unit/test_scheduling_errors.py +++ b/sdk/python/tests/unit/test_scheduling_errors.py @@ -153,6 +153,19 @@ def test_eligible_workers_do_not_imply_cluster_shortage(self): self.assertIn("schedule_timeout", " ".join(error.suggestions)) self.assertNotIn("insufficient memory", str(error)) + def test_zero_eligible_workers_without_rejections_remains_unexplained(self): + details = diagnosis(nodeCount=15, eligibleNodes=0, rejections={}) + error = self.convert(details) + self.assertIsInstance(error, SchedulingError) + self.assertNotIsInstance(error, InsufficientResourcesError) + self.assertEqual(error.reasons, ("unknown",)) + self.assertEqual(error.resource_shortages, ()) + self.assertIs(error.scheduling_failure, details) + self.assertIn("without a usable placement diagnosis", str(error)) + self.assertIn("cluster operator", " ".join(error.suggestions)) + self.assertNotIn("no workers", str(error)) + self.assertEqual((error.retry, error.outcome), ("never", "not_started")) + def test_incomplete_or_malformed_diagnoses_do_not_invent_shortages(self): for details in ( {},