Skip to content

test(e2e): retry a cold-start chat request and report why it failed #155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rominf wants to merge 1 commit into
base: main
Choose a base branch
from
+115 −78
Open

test(e2e): retry a cold-start chat request and report why it failed #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 102 additions & 33 deletions tests/e2e-cucumber/tests/e2e.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -661,43 +661,112 @@
inference_timeout_secs().max(world.serve_timeout_override.unwrap_or(0))
}

pub async fn send_chat(world: &mut E2eWorld) {
let endpoint = world.endpoint.as_ref().expect("no endpoint configured");

let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(inference_timeout_for(world)))
.build()
.expect("failed to build HTTP client");
/// A transport failure spelled out well enough to triage from a CI log alone.
///
/// `reqwest::Error`'s `Display` prints only its KIND — a client timeout and a
/// refused/reset connection both read as "error sending request for url (…)",
/// with the real cause reachable only through `source()`. That ambiguity cost a
/// full log archaeology once (a 10s client timeout that read as a dead server),
/// so classify the error and unwind the source chain into the panic message.
fn describe_request_error(error: &reqwest::Error) -> String {
use std::error::Error as _;

let kind = if error.is_timeout() {
" [client timeout — the harness gave up, the server may still be working]"
} else if error.is_connect() {
" [connect failure — nothing accepted the connection]"
} else {
""
};
let mut detail = format!("{error}{kind}");
let mut source = error.source();
while let Some(cause) = source {
detail.push_str(&format!("\n caused by: {cause}"));

Check failure on line 684 in tests/e2e-cucumber/tests/e2e.rs

View workflow job for this annotation

GitHub Actions / clippy 

`format!(..)` appended to existing `String`
source = cause.source();
}
detail
}

/// Discover the served model id over `/models` and POST one chat completion to
/// it, returning the decoded response. `tools` is merged into the request body
/// for the tool-definitions scenario.
///
/// Retries once on a transport failure, but only for a scenario expected to
/// PASS — the same rule the serve relaunch uses. The FIRST inference after a
/// serve pays a cold start (weights paged in on demand), and on a busy runner
/// that has exceeded the flat inference timeout even though the steady-state
/// request on the very same host takes ~1s: run 30614673685 (Strix-Windows)
/// failed this at exactly 10.000s while the next scenario's chat answered in
/// 1.4s. The aborted attempt still leaves the model resident, so the retry runs
/// warm. A known-bug scenario keeps its single attempt so hang detection stays
/// as prompt as `inference_timeout_secs` documents.
pub async fn request_chat_completion(
world: &mut E2eWorld,
prompt: &str,
tools: Option<serde_json::Value>,
) -> serde_json::Value {
let endpoint = world.endpoint.clone().expect("no endpoint configured");
let timeout_secs = inference_timeout_for(world);
let attempts = if world.expect_xfail { 1 } else { 2 };
let models_url = format!("{endpoint}/models");
let resp: serde_json::Value = client
.get(&models_url)
.send()
.await
.unwrap_or_else(|e| panic!("GET {models_url} failed: {e}"))
.json()
.await
.unwrap_or_else(|e| panic!("GET {models_url} returned non-JSON: {e}"));
let model = resp["data"][0]["id"]
.as_str()
.unwrap_or_else(|| panic!("no model id in response: {resp}"))
.to_string();
world.discovered_model = Some(model.clone());

let chat_url = format!("{endpoint}/chat/completions");
let chat_resp: serde_json::Value = client
.post(&chat_url)
.json(&serde_json::json!({
let mut diagnostics = Vec::new();

for attempt in 1..=attempts {
// A fresh client per attempt: the pooled connection of a timed-out
// attempt is not worth reusing.
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(timeout_secs))
.build()
.expect("failed to build HTTP client");

let models: serde_json::Value = client
.get(&models_url)
.send()
.await
.unwrap_or_else(|e| panic!("GET {models_url} failed: {}", describe_request_error(&e)))
.json()
.await
.unwrap_or_else(|e| panic!("GET {models_url} returned non-JSON: {e}"));
let model = models["data"][0]["id"]
.as_str()
.unwrap_or_else(|| panic!("no model id in response: {models}"))
.to_string();
world.discovered_model = Some(model.clone());

let mut body = serde_json::json!({
"model": model,
"messages": [{"role": "user", "content": "Hello"}]
}))
.send()
.await
.unwrap_or_else(|e| panic!("POST {chat_url} failed: {e}"))
.json()
.await
.unwrap_or_else(|e| panic!("POST {chat_url} returned non-JSON: {e}"));
world.chat_response = Some(chat_resp);
"messages": [{"role": "user", "content": prompt}]
});
if let Some(tools) = tools.clone() {
body["tools"] = tools;
}

let started = std::time::Instant::now();
match client.post(&chat_url).json(&body).send().await {
Ok(response) => {
return response
.json()
.await
.unwrap_or_else(|e| panic!("POST {chat_url} returned non-JSON: {e}"));
}
Err(error) => diagnostics.push(format!(
"attempt {attempt} failed after {:.1}s: {}",
started.elapsed().as_secs_f64(),
describe_request_error(&error)
)),
}
}

panic!(
"POST {chat_url} failed after {attempts} attempt(s) of {timeout_secs}s each:\n{}",
diagnostics.join("\n")
);
}

pub async fn send_chat(world: &mut E2eWorld) {
let response = request_chat_completion(world, "Hello", None).await;
world.chat_response = Some(response);
}

// ── Runner ─────────────────────────────────────────────────────────
Expand Down
58 changes: 13 additions & 45 deletions tests/e2e-cucumber/tests/e2e/chat_steps.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,51 +37,19 @@ async fn user_checks_services(world: &mut E2eWorld) {

#[when("a chat request with tool definitions is sent")]
async fn send_chat_with_tools(world: &mut E2eWorld) {
let endpoint = world.endpoint.as_ref().expect("no endpoint configured");

let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(
crate::inference_timeout_for(world),
))
.build()
.expect("failed to build HTTP client");

let models_url = format!("{endpoint}/models");
let resp: serde_json::Value = client
.get(&models_url)
.send()
.await
.unwrap_or_else(|e| panic!("GET {models_url} failed: {e}"))
.json()
.await
.unwrap_or_else(|e| panic!("GET {models_url} returned non-JSON: {e}"));
let model = resp["data"][0]["id"]
.as_str()
.unwrap_or_else(|| panic!("no model id in response: {resp}"))
.to_string();

let chat_url = format!("{endpoint}/chat/completions");
let chat_resp: serde_json::Value = client
.post(&chat_url)
.json(&serde_json::json!({
"model": model,
"messages": [{"role": "user", "content": "What GPUs are available?"}],
"tools": [{
"type": "function",
"function": {
"name": "gpu_status",
"description": "Get GPU status",
"parameters": {"type": "object", "properties": {}}
}
}]
}))
.send()
.await
.unwrap_or_else(|e| panic!("POST {chat_url} failed: {e}"))
.json()
.await
.unwrap_or_else(|e| panic!("POST {chat_url} returned non-JSON: {e}"));
world.chat_response = Some(chat_resp);
// Same discover-then-POST path as a plain chat (including its cold-start
// retry and transport diagnostics), with a tool definition attached.
let tools = serde_json::json!([{
"type": "function",
"function": {
"name": "gpu_status",
"description": "Get GPU status",
"parameters": {"type": "object", "properties": {}}
}
}]);
let response =
crate::request_chat_completion(world, "What GPUs are available?", Some(tools)).await;
world.chat_response = Some(response);
}

#[when("the user sends a chat message")]
Expand Down
Loading