Skip to content

Commit a97f95e

Browse files
committed
fix(selfhosted): send worker identity on heartbeat
Sync-Source-Commit: 3a50fd6cf984cc31bd147844160553bb3b0d539e Hand-Written-Reason: No Ark-APIs provenance marker; treated as a hand-written source commit. Release-Version: 0.11.0
1 parent 88f041f commit a97f95e

8 files changed

Lines changed: 21 additions & 35 deletions

File tree

‎src/arkruntime/resources/environments/work.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ def heartbeat(
102102
environment_id: str,
103103
work_id: str,
104104
*,
105+
worker_id: str = "",
105106
expected_last_heartbeat: str = "",
106107
desired_ttl_seconds: int = 0,
107108
extra_headers=None,
@@ -118,7 +119,7 @@ def heartbeat(
118119
_path(environment_id, f"{quote(work_id, safe='')}/heartbeat"),
119120
options=make_request_options(
120121
query=query,
121-
extra_headers=extra_headers,
122+
extra_headers=_worker_headers(worker_id, extra_headers),
122123
timeout=timeout,
123124
),
124125
cast_to=HeartbeatWorkResponse,
@@ -200,6 +201,7 @@ async def heartbeat(
200201
environment_id: str,
201202
work_id: str,
202203
*,
204+
worker_id: str = "",
203205
expected_last_heartbeat: str = "",
204206
desired_ttl_seconds: int = 0,
205207
extra_headers=None,
@@ -216,7 +218,7 @@ async def heartbeat(
216218
_path(environment_id, f"{quote(work_id, safe='')}/heartbeat"),
217219
options=make_request_options(
218220
query=query,
219-
extra_headers=extra_headers,
221+
extra_headers=_worker_headers(worker_id, extra_headers),
220222
timeout=timeout,
221223
),
222224
cast_to=HeartbeatWorkResponse,

‎src/arkruntime/selfhosted/client.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ def heartbeat_work(
9191
environment_id: str,
9292
work_id: str,
9393
*,
94+
worker_id: str = "",
9495
expected_last_heartbeat: str,
9596
desired_ttl_seconds: int = 30,
9697
) -> HeartbeatResponse:
@@ -101,6 +102,7 @@ def heartbeat_work(
101102
response = self.client.environments.work.heartbeat(
102103
environment_id,
103104
work_id,
105+
worker_id=worker_id,
104106
expected_last_heartbeat=(expected_last_heartbeat or EXPECTED_LAST_HEARTBEAT_NO_HEARTBEAT),
105107
desired_ttl_seconds=desired_ttl_seconds,
106108
timeout=max(1.0, min(float(desired_ttl_seconds or 30) / 2, 30.0)),

‎src/arkruntime/selfhosted/worker.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ def _handle_item(self, work: _ClaimedWork) -> None:
332332
cause = heartbeat_cause["value"]
333333
if _should_stop_item(cause):
334334
reason = None if cause == "stop_requested" else stop_reason
335-
# 外围关闭且 heartbeat 未给出其他原因时,需要由平台重新排队。
335+
# 外围关闭且 heartbeat 未给出其他原因时,记录 worker 异常退出。
336336
if cause == "" and self._stop.is_set():
337337
reason = WorkStopReason.worker_abnormal
338338
try:
@@ -360,6 +360,7 @@ def _heartbeat_loop(self, work: _ClaimedWork, stop: threading.Event, done: threa
360360
resp = self.api.heartbeat_work(
361361
work.environment_id,
362362
work.id,
363+
worker_id=self.options.worker_id,
363364
expected_last_heartbeat=last,
364365
desired_ttl_seconds=int(ttl),
365366
)

‎src/arkruntime/types/environment/env_config.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,6 @@ class EnvConfig(BaseModel):
4141
"""
4242
沙箱启动阶段执行的初始化脚本。
4343
"""
44-
active_recovery: Optional[bool] = None
45-
"""
46-
是否启用 Work 积极恢复;仅 self_hosted Environment 支持,省略时等同于 false。该配置
47-
仅允许在 Environment 级设置,不允许通过 Session override/upgrade 覆盖。开启后,平台在
48-
Work 已 stopped、仍有未闭合 agent.tool_use 且 recovery_count 少于 5 次时将 Work 重新排队,
49-
不按 stop_reason 排除。只有 Worker 上报 completed 且全部 tool_use 已闭合后才会清零计数。
50-
"""
5144
tos: Optional[TosConfig] = None
5245
"""
5346
Environment outputs 的 TOS 存储配置。

‎src/arkruntime/types/environment/stop_work_body.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,5 @@ class StopWorkBody(BaseModel):
2727
强制停止原因;仅允许与 `force=true` 一起提交,省略时按 `others` 记录。
2828
Worker 正常完成时传 `completed`,异常退出时传 `worker_abnormal`;用户或管理员操作分别传
2929
`user_cancelled` 或 `admin_stopped`,无法归类时传 `others`。`lease_expired` 仅由平台在租约
30-
过期时写入,不应通过 StopWork 上报。该字段用于停止诊断,并在 Worker 上报 `completed`
31-
且全部 tool_use 已闭合时作为清零恢复次数的信号;平台不会按停止原因排除积极恢复。
30+
过期时写入,不应通过 StopWork 上报。该字段仅用于停止诊断;强停不会自动重新排队。
3231
"""

‎src/arkruntime/types/environment/work_item.py‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,8 @@ class WorkItem(BaseModel):
7171
"""
7272
stop_reason: Optional[WorkStopReason] = None
7373
"""
74-
最近一次停止原因。首次停止前省略;state 为 stopping / stopped 时返回本次停止原因;
75-
自动恢复到 queued / starting / active 后仍保留上一次停止原因,当前生命周期状态以
76-
`state` 为准。
77-
"""
78-
recovery_count: Optional[int] = None
79-
"""
80-
自上次成功完成且全部 tool_use 已闭合后,平台已自动将本 Work 重新排队的次数;
81-
首次执行或字段省略时按 0 处理。手动 Ensure 不会清零该计数。
74+
最近一次停止原因。首次停止前省略;state 为 stopping / stopped 时返回本次停止原因。
75+
原 owner 通过 Heartbeat 将 `stopped + lease_expired` 恢复为 active 时清除此字段。
8276
"""
8377
type: Literal["work"]
8478
"""

‎tests/selfhosted/test_client.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ def test_poll_work_preserves_nested_session_data() -> None:
6262
"environment_id": "env-20260813132936-wq8d4",
6363
"state": "queued",
6464
"stop_reason": "worker_abnormal",
65-
"recovery_count": 5,
6665
"type": "work",
6766
}
6867

@@ -81,7 +80,6 @@ def handle(request: httpx.Request) -> httpx.Response:
8180
assert item.data.id == work["data"]["id"]
8281
assert item.data.type == "session"
8382
assert item.stop_reason == WorkStopReason.worker_abnormal
84-
assert item.recovery_count == 5
8583
assert work_session_id(item) == work["data"]["id"]
8684

8785

@@ -113,7 +111,7 @@ def handle(request: httpx.Request) -> httpx.Response:
113111
client.close()
114112

115113

116-
def test_heartbeat_uses_generated_response_model() -> None:
114+
def test_heartbeat_uses_generated_response_model_and_worker_identity() -> None:
117115
heartbeat = {
118116
"last_heartbeat": "2026-08-24T10:00:00Z",
119117
"lease_extended": True,
@@ -123,13 +121,15 @@ def test_heartbeat_uses_generated_response_model() -> None:
123121
}
124122

125123
def handle(request: httpx.Request) -> httpx.Response:
124+
assert request.headers["Ark-Worker-ID"] == "worker-1"
126125
return httpx.Response(httpx.codes.OK, json=heartbeat)
127126

128127
client = _ark_client(httpx.MockTransport(handle))
129128
try:
130129
response = ClientAPI(client).heartbeat_work(
131130
"env-1",
132131
"work-1",
132+
worker_id="worker-1",
133133
expected_last_heartbeat="NO_HEARTBEAT",
134134
desired_ttl_seconds=30,
135135
)

‎tests/selfhosted/test_worker.py‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,7 @@ def test_fatal_ack_error_force_stops_work_and_continues() -> None:
7676

7777
assert poller.next() is None
7878
assert poller.error is None
79-
assert api.stops == [
80-
("env-1", "work-1", {"force": True, "reason": WorkStopReason.others})
81-
]
79+
assert api.stops == [("env-1", "work-1", {"force": True, "reason": WorkStopReason.others})]
8280

8381

8482
def test_poller_does_not_swallow_programming_errors() -> None:
@@ -104,8 +102,10 @@ def test_default_worker_id_is_unique_per_worker() -> None:
104102
class _StoppingHeartbeatAPI:
105103
def __init__(self) -> None:
106104
self.stops = []
105+
self.heartbeat_worker_ids = []
107106

108107
def heartbeat_work(self, environment_id, work_id, **kwargs):
108+
self.heartbeat_worker_ids.append(kwargs.get("worker_id"))
109109
return HeartbeatResponse(
110110
last_heartbeat="2026-08-24T10:00:00Z",
111111
state="stopping",
@@ -136,6 +136,7 @@ def test_heartbeat_stop_cancels_session_runner(tmp_path) -> None:
136136

137137
assert time.monotonic() - started < 1
138138
assert api.stops == [("env-1", "work-1", {"force": True})]
139+
assert api.heartbeat_worker_ids == [worker.options.worker_id]
139140

140141

141142
class _ActiveHeartbeatAPI(_StoppingHeartbeatAPI):
@@ -164,9 +165,7 @@ def test_worker_failure_stops_work_as_abnormal(tmp_path) -> None:
164165
with pytest.raises(ValueError, match="worker failed"):
165166
worker.handle_item(HandleItemOptions(work_id="work-1", environment_id="env-1", session_id="session-1"))
166167

167-
assert api.stops == [
168-
("env-1", "work-1", {"force": True, "reason": WorkStopReason.worker_abnormal})
169-
]
168+
assert api.stops == [("env-1", "work-1", {"force": True, "reason": WorkStopReason.worker_abnormal})]
170169

171170

172171
class _TerminatedWorkerAPI(_FailingWorkerAPI):
@@ -183,9 +182,7 @@ def test_session_termination_stops_work_as_completed(tmp_path) -> None:
183182

184183
worker.handle_item(HandleItemOptions(work_id="work-1", environment_id="env-1", session_id="session-1"))
185184

186-
assert api.stops == [
187-
("env-1", "work-1", {"force": True, "reason": WorkStopReason.completed})
188-
]
185+
assert api.stops == [("env-1", "work-1", {"force": True, "reason": WorkStopReason.completed})]
189186

190187

191188
def test_worker_close_stops_work_as_abnormal(tmp_path) -> None:
@@ -198,9 +195,7 @@ def test_worker_close_stops_work_as_abnormal(tmp_path) -> None:
198195

199196
worker.handle_item(HandleItemOptions(work_id="work-1", environment_id="env-1", session_id="session-1"))
200197

201-
assert api.stops == [
202-
("env-1", "work-1", {"force": True, "reason": WorkStopReason.worker_abnormal})
203-
]
198+
assert api.stops == [("env-1", "work-1", {"force": True, "reason": WorkStopReason.worker_abnormal})]
204199

205200

206201
class _LeaseLostHeartbeatAPI(_StoppingHeartbeatAPI):

0 commit comments

Comments
 (0)