diff --git a/crates/lance-context-api/src/lib.rs b/crates/lance-context-api/src/lib.rs index fc1d131..4b19b30 100644 --- a/crates/lance-context-api/src/lib.rs +++ b/crates/lance-context-api/src/lib.rs @@ -663,6 +663,16 @@ pub struct AddRolloutRequest { #[serde(default = "default_content_type")] pub content_type: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_input_string: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_output_string: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rationale: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub problem_text: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_metadata: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub input_tokens: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub output_tokens: Option>, @@ -742,6 +752,16 @@ pub struct RolloutRecordDto { pub content: Option, pub content_type: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_input_string: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_output_string: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rationale: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub problem_text: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_metadata: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub input_tokens: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub output_tokens: Option>, diff --git a/crates/lance-context-core/src/api_impl.rs b/crates/lance-context-core/src/api_impl.rs index 9f26cb8..09de0ec 100644 --- a/crates/lance-context-core/src/api_impl.rs +++ b/crates/lance-context-core/src/api_impl.rs @@ -456,6 +456,11 @@ fn rollout_record_from_add_request(r: &AddRolloutRequest) -> RolloutRecord { created_at: r.created_at.unwrap_or_else(Utc::now), content: r.content.clone(), content_type: r.content_type.clone(), + model_input_string: r.model_input_string.clone(), + model_output_string: r.model_output_string.clone(), + rationale: r.rationale.clone(), + problem_text: r.problem_text.clone(), + user_metadata: r.user_metadata.clone(), input_tokens: r.input_tokens.clone(), output_tokens: r.output_tokens.clone(), num_input_tokens: r.num_input_tokens, @@ -498,6 +503,11 @@ pub fn rollout_record_to_dto(r: RolloutRecord) -> RolloutRecordDto { created_at: r.created_at, content: r.content, content_type: r.content_type, + model_input_string: r.model_input_string, + model_output_string: r.model_output_string, + rationale: r.rationale, + problem_text: r.problem_text, + user_metadata: r.user_metadata, input_tokens: r.input_tokens, output_tokens: r.output_tokens, num_input_tokens: r.num_input_tokens, diff --git a/crates/lance-context-core/src/rollout.rs b/crates/lance-context-core/src/rollout.rs index a6db415..bc68b5f 100644 --- a/crates/lance-context-core/src/rollout.rs +++ b/crates/lance-context-core/src/rollout.rs @@ -45,6 +45,16 @@ pub struct RolloutRecord { pub content: Option, pub content_type: String, + // Oversized message fields offloaded via the claim-check write path. Each is + // its own nullable, individually-projectable column (not packed into + // `content`/`binary_payload`) so a reader can select one without + // materializing the rest. + pub model_input_string: Option, + pub model_output_string: Option, + pub rationale: Option, + pub problem_text: Option, + pub user_metadata: Option, + // Tokens. pub input_tokens: Option>, pub output_tokens: Option>, diff --git a/crates/lance-context-core/src/rollout_store.rs b/crates/lance-context-core/src/rollout_store.rs index 1a34b42..b2ed3da 100644 --- a/crates/lance-context-core/src/rollout_store.rs +++ b/crates/lance-context-core/src/rollout_store.rs @@ -1408,6 +1408,11 @@ impl RolloutStore { let mut created_at_builder = TimestampMicrosecondBuilder::with_capacity(records.len()); let mut content_builder = LargeStringBuilder::new(); let mut content_type_builder = StringBuilder::new(); + let mut model_input_string_builder = LargeStringBuilder::new(); + let mut model_output_string_builder = LargeStringBuilder::new(); + let mut rationale_builder = LargeStringBuilder::new(); + let mut problem_text_builder = LargeStringBuilder::new(); + let mut user_metadata_builder = LargeStringBuilder::new(); let mut input_tokens_builder = ListBuilder::new(Int32Builder::new()); let mut output_tokens_builder = ListBuilder::new(Int32Builder::new()); let mut num_input_tokens_builder = Int32Builder::new(); @@ -1442,6 +1447,11 @@ impl RolloutStore { created_at_builder.append_value(record.created_at.timestamp_micros()); content_builder.append_option(record.content.as_deref()); content_type_builder.append_value(&record.content_type); + model_input_string_builder.append_option(record.model_input_string.as_deref()); + model_output_string_builder.append_option(record.model_output_string.as_deref()); + rationale_builder.append_option(record.rationale.as_deref()); + problem_text_builder.append_option(record.problem_text.as_deref()); + user_metadata_builder.append_option(record.user_metadata.as_deref()); append_i32_list(&mut input_tokens_builder, record.input_tokens.as_deref()); append_i32_list(&mut output_tokens_builder, record.output_tokens.as_deref()); num_input_tokens_builder.append_option(record.num_input_tokens); @@ -1521,6 +1531,26 @@ impl RolloutStore { "content_type".to_string(), Arc::new(content_type_builder.finish()), ); + arrays_by_name.insert( + "model_input_string".to_string(), + Arc::new(model_input_string_builder.finish()), + ); + arrays_by_name.insert( + "model_output_string".to_string(), + Arc::new(model_output_string_builder.finish()), + ); + arrays_by_name.insert( + "rationale".to_string(), + Arc::new(rationale_builder.finish()), + ); + arrays_by_name.insert( + "problem_text".to_string(), + Arc::new(problem_text_builder.finish()), + ); + arrays_by_name.insert( + "user_metadata".to_string(), + Arc::new(user_metadata_builder.finish()), + ); arrays_by_name.insert( "input_tokens".to_string(), Arc::new(input_tokens_builder.finish()), @@ -1698,6 +1728,12 @@ pub fn rollout_schema() -> Schema { // Message content. Field::new("content", DataType::LargeUtf8, true), Field::new("content_type", DataType::Utf8, false), + // Claim-check offloaded message fields. + Field::new("model_input_string", DataType::LargeUtf8, true), + Field::new("model_output_string", DataType::LargeUtf8, true), + Field::new("rationale", DataType::LargeUtf8, true), + Field::new("problem_text", DataType::LargeUtf8, true), + Field::new("user_metadata", DataType::LargeUtf8, true), // Tokens. list_field("input_tokens", DataType::Int32), list_field("output_tokens", DataType::Int32), @@ -1788,6 +1824,13 @@ fn batch_to_rollout_records(batch: &RecordBatch) -> LanceResult(batch, "created_at")?; let content_array = column_as_optional::(batch, "content"); let content_type_array = column_as::(batch, "content_type")?; + let model_input_string_array = + column_as_optional::(batch, "model_input_string"); + let model_output_string_array = + column_as_optional::(batch, "model_output_string"); + let rationale_array = column_as_optional::(batch, "rationale"); + let problem_text_array = column_as_optional::(batch, "problem_text"); + let user_metadata_array = column_as_optional::(batch, "user_metadata"); let input_tokens_array = column_as_optional::(batch, "input_tokens"); let output_tokens_array = column_as_optional::(batch, "output_tokens"); let num_input_tokens_array = column_as_optional::(batch, "num_input_tokens"); @@ -1863,6 +1906,11 @@ fn batch_to_rollout_records(batch: &RecordBatch) -> LanceResult RolloutRecord { created_at: chrono::Utc::now(), content: Some("x".to_string()), content_type: "text/plain".to_string(), + model_input_string: None, + model_output_string: None, + rationale: None, + problem_text: None, + user_metadata: None, input_tokens: None, output_tokens: None, num_input_tokens: None, diff --git a/crates/lance-context-master/src/routes.rs b/crates/lance-context-master/src/routes.rs index 34b2e2e..82283ba 100644 --- a/crates/lance-context-master/src/routes.rs +++ b/crates/lance-context-master/src/routes.rs @@ -511,6 +511,11 @@ mod tests { created_at: Utc::now(), content: Some("answer".to_string()), content_type: if with_blob { "image/png" } else { "text/plain" }.to_string(), + model_input_string: None, + model_output_string: None, + rationale: None, + problem_text: None, + user_metadata: None, input_tokens: None, output_tokens: Some(vec![1, 2]), num_input_tokens: None, diff --git a/crates/lance-context-master/src/scheduler.rs b/crates/lance-context-master/src/scheduler.rs index 319f39c..fbc5508 100644 --- a/crates/lance-context-master/src/scheduler.rs +++ b/crates/lance-context-master/src/scheduler.rs @@ -761,6 +761,11 @@ mod tests { created_at: Utc.timestamp_micros(1_700_000_000_000_000).unwrap(), content: Some("x".to_string()), content_type: "text/plain".to_string(), + model_input_string: None, + model_output_string: None, + rationale: None, + problem_text: None, + user_metadata: None, input_tokens: None, output_tokens: None, num_input_tokens: None, diff --git a/crates/lance-context-server/src/routes/rollouts.rs b/crates/lance-context-server/src/routes/rollouts.rs index 2ece5aa..a85c470 100644 --- a/crates/lance-context-server/src/routes/rollouts.rs +++ b/crates/lance-context-server/src/routes/rollouts.rs @@ -530,6 +530,11 @@ fn rollout_record_from_add_request(r: &AddRolloutRequest) -> RolloutRecord { created_at: r.created_at.unwrap_or_else(Utc::now), content: r.content.clone(), content_type: r.content_type.clone(), + model_input_string: r.model_input_string.clone(), + model_output_string: r.model_output_string.clone(), + rationale: r.rationale.clone(), + problem_text: r.problem_text.clone(), + user_metadata: r.user_metadata.clone(), input_tokens: r.input_tokens.clone(), output_tokens: r.output_tokens.clone(), num_input_tokens: r.num_input_tokens, @@ -571,6 +576,11 @@ fn rollout_record_to_dto(r: RolloutRecord) -> RolloutRecordDto { created_at: r.created_at, content: r.content, content_type: r.content_type, + model_input_string: r.model_input_string, + model_output_string: r.model_output_string, + rationale: r.rationale, + problem_text: r.problem_text, + user_metadata: r.user_metadata, input_tokens: r.input_tokens, output_tokens: r.output_tokens, num_input_tokens: r.num_input_tokens, diff --git a/specs/rollout-schema-design.md b/specs/rollout-schema-design.md index 346377d..b8b16f9 100644 --- a/specs/rollout-schema-design.md +++ b/specs/rollout-schema-design.md @@ -100,6 +100,16 @@ One row per **message** in a rollout (assistant turn, tool call, grade, or artif | `content` | `LargeUtf8` | message text | | `content_type` | `Utf8` | MIME type | +**Claim-check offloaded message fields** *(oversized rollout-message fields, each an individually-projectable nullable column)* + +| Column | Arrow type | Purpose | +|---|---|---| +| `model_input_string` | `LargeUtf8` | rendered model prompt string | +| `model_output_string` | `LargeUtf8` | raw model completion string | +| `rationale` | `LargeUtf8` | grader rationale | +| `problem_text` | `LargeUtf8` | source problem text | +| `user_metadata` | `LargeUtf8` | harness-supplied per-message metadata blob | + **Tokens** *(first-class variable-length arrays)* | Column | Arrow type | Purpose |