Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/actions/python-distributions/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,13 @@ runs:
import adx_environment
import akernel_sdk
from akernel_sdk import ComposeProject, DockerRuntime, ServiceHandle, ServiceProcessHandle
from akernel_sdk import BackendOperationError, InsufficientResourcesError, SchedulingError
from importlib.resources import files

assert all(callable(value) for value in (ComposeProject, DockerRuntime, ServiceHandle, ServiceProcessHandle))
assert issubclass(InsufficientResourcesError, SchedulingError)
assert issubclass(SchedulingError, BackendOperationError)
assert SchedulingError("smoke").reasons == ("unknown",)
for helper in ("_image_bridge_guest.py", "_docker_network_guest.py", "_service_process_guest.py"):
assert files("akernel_sdk").joinpath(helper).read_text()

Expand Down
6 changes: 6 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,12 @@ the pinned native PermissionDenied exception without inventing a missing status.
`BackendOperationError` preserves optional structured service fields and the
original cause; missing context remains `None`. Keep retry/outcome semantics
backend-owned and document public error fields in `sdk/python/README.md`.
Confirmed cancelled queue requests with structured diagnostics use
`SchedulingError` and, when scalar shortages contribute with no eligible worker,
`InsufficientResourcesError`. Preserve raw diagnostics and native causes while
explaining all blockers; eligible workers or unusable diagnostics must not be
presented as confirmed resource shortages. Keep classification independent of
retry policy and do not infer usable capacity from global per-dimension maxima.

Keep sandbox cleanup explicit through context managers or `kill()`, with
observable, retryable deletion failures and workload exceptions preserved on
Expand Down
98 changes: 98 additions & 0 deletions sdk/python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1469,6 +1469,104 @@ original identity when querying or retrying. Do not infer execution or cleanup
from a timeout message. `status_code` describes the service failure (including
an SSE final event's error code); it need not equal the stream's HTTP status.

Confirmed scheduling cancellations with structured diagnostics raise
`SchedulingError`, a subclass of `BackendOperationError`. If no worker is
eligible and CPU, memory or disk shortages contribute, the exception is the
more specific `InsufficientResourcesError`, which also subclasses
`SchedulingError`. Existing `except BackendOperationError` handlers still work.

The service's `error.schedulingFailure` object is retained as
`error.scheduling_failure`, with its original field names and values:

| Field | Meaning |
| --- | --- |
| `snapshotRevision` | Revision of the scheduling snapshot at queue expiry; retained for diagnostics. |
| `nodeCount` | Number of workers known to the scheduler. Zero establishes `no_workers`. |
| `eligibleNodes` | Number passing all hard placement checks in that snapshot. A positive value produces `queue_timeout`. |
| `requested` | Requested resources: `cpu_millis` in millicores, `memory_bytes` and `disk_bytes` in bytes. |
| `maxCapacity`, `maxAvailable` | Resource objects with the same keys/units. Maxima span workers; available values already subtract reservations. |
| `rejections` | Map of rejection keys to worker counts. Counts may overlap; zero counts do not establish blockers. |

Counters and resource amounts use unsigned 64-bit integers; booleans, negative
values and numeric strings are not coerced into valid evidence. Missing or
unusable eligibility data cannot establish a resource shortage; classification
also requires valid positive resource-rejection counts. Additional fields and
future rejection keys remain available in the original object.
Keep schema extensions aligned with ADX's `platform/api/http/error-contract.md`;
adding compatible-worker capacity diagnostics requires a service-side contract
before the SDK can distinguish temporary occupancy from an oversized request.

The scheduling exceptions expose:

- `reasons`: a tuple of positive worker rejection keys from the service, sorted
by key. Counts can overlap; their sum is not a worker count. Unknown future
rejection keys are retained.
- `resource_shortages`: contributing resource names (`"cpu"`, `"memory"`,
`"disk"`), in that order. Other constraints can also contribute.
- `suggestions`: a tuple of concrete recovery suggestions. The exception message
includes these suggestions, readable requested resources, and rejection counts.

```python
from akernel_sdk import InsufficientResourcesError, Sandbox, SchedulingError

try:
sandbox = Sandbox(cpu=2000, memory=8192, storage_mb=20480)
except InsufficientResourcesError as error:
print("Unavailable resources:", ", ".join(error.resource_shortages))
print(error) # Summary and actions; this does not retry the request.
print("All placement blockers:", error.reasons)
except SchedulingError as error:
print(error) # Runtime, devices, placement, volumes, workers, or queue budget.
else:
sandbox.kill()
```

| Reported blocker | Action to consider |
| --- | --- |
| `insufficient_cpu`, `insufficient_memory`, `insufficient_disk` | Reduce the request, release idle sandboxes, or add suitable worker capacity. |
| `runtime_fit`, `device_fit` | Select a supported runtime or available accelerator model/count, or add compatible workers. |
| `node_available`, `no_workers` | Ask the operator to check worker health/registration and register healthy workers. |
| `node_affinity`, `placement_groups`, `environment_affinity`, `topology_spread` | Adjust placement constraints or add workers that satisfy them. |
| `waiting_for_volume`, `volume_topology` | Wait for the volume to be released or select workers with access to its storage. |

For example, a mixed rejection produces a message like:

```text
create sandbox failed: the cluster has insufficient resources for this request;
no worker satisfies all resource and placement requirements.
Requested: 2 CPU, 8 GiB memory, 20 GiB disk.
Worker rejections: insufficient memory (14), unsupported runtime (1).
Suggestions: Reduce the memory request, release idle sandboxes, or add workers
with more available memory. Select a supported runtime or add workers that
advertise the requested runtime.
```

When the snapshot still has eligible workers, `reasons` is `("queue_timeout",)`
and `resource_shortages` is empty: the queue budget expired, but resource shortage
is not established. Missing or unusable diagnostic fields produce
`reasons=("unknown",)` and retain the raw diagnostic for investigation. Older
services without a structured diagnosis keep their existing exception behavior.
In particular, `nodeCount > 0`, `eligibleNodes == 0` and an empty `rejections`
map produce `unknown`: workers exist, but the snapshot does not explain the
rejection. This case establishes neither `no_workers` nor a resource shortage.
Plain transport timeouts, initialization failures, backend rate limits and
uncertain outcomes do not become scheduling errors just because their message
mentions resources or a deadline.

The service's `code`, `retry`, `outcome`, IDs and original `scheduling_failure`
are preserved; the native message remains in `__cause__`. No additional attempt,
backoff or queue budget is introduced. `retry="never"` means this queue wait is
finished; after addressing the blockers, callers may deliberately start a new
create request. Unknown outcomes keep their original same-operation semantics.

Capacity and available maxima in the raw diagnostic span all workers, including
workers rejected by runtime or placement filters. Each resource maximum can
come from a different worker. They do not establish that a suitable worker can
run the request, or reliably distinguish temporary occupancy from an oversized
request on compatible workers. The default summary therefore presents the
requested resources and blockers without presenting those maxima as usable
capacity. Diagnostics describe the expiry snapshot, not capacity after scaling.


### Client-managed Docker/Compose runtime

Expand Down
4 changes: 4 additions & 0 deletions sdk/python/akernel_sdk/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,13 @@
BackendNotInstalledError,
BackendOperationError,
CommandExitException,
InsufficientResourcesError,
InvalidBackendError,
NotFoundException,
PermissionDeniedException,
RateLimitException,
SandboxNotFoundException,
SchedulingError,
ServiceBusyException,
TimeoutException,
UnsupportedBackendFeatureError,
Expand Down Expand Up @@ -93,6 +95,8 @@
"BackendNotInstalledError",
"UnsupportedBackendFeatureError",
"BackendOperationError",
"SchedulingError",
"InsufficientResourcesError",
"AuthenticationException",
"PermissionDeniedException",
"NotFoundException",
Expand Down
187 changes: 187 additions & 0 deletions sdk/python/akernel_sdk/_backends/_scheduling.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
# Copyright (c) 2026 Ant Group Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Explain a scheduling snapshot without inferring retry or execution state."""

from collections.abc import Mapping
from dataclasses import dataclass

_RESOURCE_NAMES = {"cpu": "CPU", "memory": "memory", "disk": "disk"}
_PLACEMENT_HINT = "Relax placement constraints or add workers that satisfy them."
_OPERATOR_HINT = (
"Inspect scheduling_failure with the cluster operator for remaining placement "
"constraints."
)
_BLOCKERS = {
"node_available": (
"worker unavailable",
"Ask the cluster operator to check worker health, registration and "
"drain status.",
),
"runtime_fit": (
"unsupported runtime",
"Select a supported runtime or add workers that advertise the "
"requested runtime.",
),
"device_fit": (
"accelerator unavailable or incompatible",
"Request an available accelerator model/count or add compatible workers.",
),
"node_affinity": ("node affinity mismatch", _PLACEMENT_HINT),
"placement_groups": ("placement group mismatch", _PLACEMENT_HINT),
"environment_affinity": ("environment affinity mismatch", _PLACEMENT_HINT),
"topology_spread": ("topology constraint", _PLACEMENT_HINT),
"waiting_for_volume": (
"volume in use",
"Wait for the volume to be released or select a different volume.",
),
"volume_topology": (
"storage inaccessible from worker",
"Select workers with access to the required storage or adjust "
"volume placement.",
),
}


@dataclass(frozen=True)
class SchedulingExplanation:
message: str
reasons: tuple[str, ...]
resource_shortages: tuple[str, ...] = ()
suggestions: tuple[str, ...] = ()


def _count(value: object) -> int | None:
# These are unsigned service counters/resources. Reject bool and malformed
# values rather than coercing them into a diagnosis or masking the failure.
if isinstance(value, int) and not isinstance(value, bool) and 0 <= value < 2**64:
return value
return None


def _bytes(value: int) -> str:
units = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB")
for power, unit in reversed(tuple(enumerate(units))):
scale = 1024**power
if value >= scale:
return f"{value / scale:g} {unit}"
return "0 B"


def _requested(details: Mapping[str, object]) -> str:
requested = details.get("requested")
if not isinstance(requested, Mapping):
return ""
parts = []
for key, resource in (
("cpu_millis", "CPU"),
("memory_bytes", "memory"),
("disk_bytes", "disk"),
):
value = _count(requested.get(key))
if value is not None:
amount = f"{value / 1000:g}" if resource == "CPU" else _bytes(value)
parts.append(f"{amount} {resource}")
return " Requested: " + ", ".join(parts) + "." if parts else ""


def explain_scheduling_failure(details: object) -> SchedulingExplanation:
"""Use only explicit current-snapshot evidence; retain raw details elsewhere."""
unknown = SchedulingExplanation(
"the scheduling queue deadline expired without a usable placement diagnosis.",
("unknown",),
suggestions=(_OPERATOR_HINT,),
)
if not isinstance(details, Mapping):
return unknown
eligible = _count(details.get("eligibleNodes"))
nodes = _count(details.get("nodeCount"))
if eligible is None or (nodes is not None and eligible > nodes):
return unknown
if eligible > 0:
noun = "worker" if eligible == 1 else "workers"
return SchedulingExplanation(
f"the scheduling queue deadline expired with {eligible} eligible {noun}."
+ _requested(details),
("queue_timeout",),
suggestions=(
"Consider a larger schedule_timeout for a new create request and ask "
"the cluster operator to check scheduler load.",
),
)
rejections = details.get("rejections")
counts = (
{
key: count
for key, value in rejections.items()
if isinstance(key, str) and key and (count := _count(value))
}
if isinstance(rejections, Mapping)
else {}
)
if nodes is not None and (
any(count > nodes for key, count in counts.items() if key != "no_workers")
or (nodes > 0 and "no_workers" in counts)
):
return unknown
if nodes == 0 or "no_workers" in counts:
hint = "Ask the cluster operator to register healthy workers."
return SchedulingExplanation(
"the cluster has no workers available for scheduling."
+ _requested(details),
("no_workers",),
suggestions=(hint,),
)
if not counts:
return unknown
resources = tuple(
name for name in _RESOURCE_NAMES if f"insufficient_{name}" in counts
)
if resources:
summary = (
"the cluster has insufficient resources for this request; "
"no worker satisfies all resource and placement requirements."
)
else:
summary = (
"no worker satisfies the current availability or placement requirements."
)
labels = []
hints = []
for resource in resources:
key = f"insufficient_{resource}"
name = _RESOURCE_NAMES[resource]
labels.append(f"insufficient {name} ({counts[key]})")
hints.append(
f"Reduce the {name} request, release idle sandboxes, or add workers "
f"with more available {name}."
)
for reason, count in sorted(counts.items()):
if reason in {f"insufficient_{name}" for name in resources}:
continue
label, hint = _BLOCKERS.get(
reason, ("other placement constraint", _OPERATOR_HINT)
)
labels.append(f"{label} ({count})")
hints.append(hint)
return SchedulingExplanation(
summary
+ _requested(details)
+ " Worker rejections: "
+ ", ".join(labels)
+ ".",
tuple(sorted(counts)),
resources,
tuple(dict.fromkeys(hints)),
)
Loading
Loading