Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions .github/scripts/test-rust-workspaces.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ check_cex_products() {
cargo tree --manifest-path "$root/rust_hft/data-pipelines/Cargo.toml" \
-p hft-cex-data-preparation --locked --edges normal --prefix none --color never \
--format '{p}|{f}|' >"$work/cex-preparation.tree"
if grep -Eq '^(alpha-|burn |ort |hft-(research-platform|research-dispatch-io|research-ml|cex-research-worker|execution-adapter-[a-z-]+) )' "$work/cex-preparation.tree"; then
if grep -Eq '^(alpha-|burn(-[a-z-]+)? |ort |hft-(research-platform|research-dispatch-io|research-ml|cex-research-worker|execution-adapter-[a-z-]+) )' "$work/cex-preparation.tree"; then
cex_graph_failure 'data preparation must exclude research execution, scheduling and trading' "$work/cex-preparation.tree"
fi

Expand All @@ -73,8 +73,13 @@ check_cex_products() {
if ! grep -q '^hft-research-ml ' "$work/cex-worker.tree"; then
cex_graph_failure 'worker must include hft-research-ml' "$work/cex-worker.tree"
fi
if ! grep -q '^burn ' "$work/cex-worker.tree"; then
cex_graph_failure 'worker must include Burn' "$work/cex-worker.tree"
for package in burn-core burn-autodiff burn-ndarray burn-nn burn-optim; do
if ! grep -q "^$package " "$work/cex-worker.tree"; then
cex_graph_failure "worker must include $package" "$work/cex-worker.tree"
fi
done
if grep -Eq '^burn(-candle|-cuda|-rocm|-wgpu|-router)? ' "$work/cex-worker.tree"; then
cex_graph_failure 'worker must use direct CPU components without the backend facade' "$work/cex-worker.tree"
fi
check_cex_feature worker "$work/cex-worker.tree" alpha-harness scientific enabled
check_cex_feature worker "$work/cex-worker.tree" alpha-engine kernel enabled
Expand Down Expand Up @@ -133,9 +138,15 @@ for profile in default db full; do
echo 'prediction research pulls the CEX input, evaluator or control chain' >&2; exit 1
fi
# Prove the full graph includes science and export, beyond the db-only graph.
if [[ $profile == full ]] &&
{ ! grep -q '^burn ' "$work/prediction.tree" || ! grep -q '^polars ' "$work/prediction.tree"; }; then
echo 'prediction full graph is missing training or columnar export dependencies' >&2; exit 1
if [[ $profile == full ]]; then
for package in burn-core burn-autodiff burn-ndarray burn-nn burn-optim polars; do
if ! grep -q "^$package " "$work/prediction.tree"; then
echo "prediction full graph is missing $package" >&2; exit 1
fi
done
if grep -Eq '^burn(-candle|-cuda|-rocm|-wgpu|-router)? ' "$work/prediction.tree"; then
echo 'prediction training must use direct CPU components' >&2; exit 1
fi
fi
done
for profile in default researcher-verification control,gateway,publisher,researcher-verification; do
Expand Down Expand Up @@ -170,7 +181,7 @@ fi
for features in formula-strategy,binance full; do
cargo tree --manifest-path "$root/rust_hft/runtime/Cargo.toml" -p hft-live \
--no-default-features --features "$features" --locked --edges normal --prefix none >"$work/live.tree"
if grep -E '^(alpha-(domain|engine|store|onnx-evaluator|harness) |burn |hft-(collector|research-platform|research-ml) )' "$work/live.tree"; then
if grep -E '^(alpha-(domain|engine|store|onnx-evaluator|harness) |burn(-[a-z-]+)? |hft-(collector|research-platform|research-ml) )' "$work/live.tree"; then
echo 'live imports research control, evaluation or training' >&2; exit 1
fi
done
Expand All @@ -179,7 +190,7 @@ for selection in 'data-pipelines/Cargo.toml hft-collector' 'data-pipelines/Cargo
read -r manifest package <<<"$selection"
cargo tree --manifest-path "$root/rust_hft/$manifest" -p "$package" \
--locked --edges normal --prefix none >"$work/depth.tree"
if grep -E '^(hft-engine |alpha-(domain|engine|onnx-evaluator|harness) |burn |ort |hft-(research-ml|research-platform|execution-adapter-[a-z-]+) )' "$work/depth.tree"; then
if grep -E '^(hft-engine |alpha-(domain|engine|onnx-evaluator|harness) |burn(-[a-z-]+)? |ort |hft-(research-ml|research-platform|execution-adapter-[a-z-]+) )' "$work/depth.tree"; then
echo 'depth acquisition imports runtime engine, science or execution authority' >&2; exit 1
fi
done
Expand All @@ -201,7 +212,7 @@ for package in hft-research-artifacts hft-research-dispatch-io hft-prediction-re
case $package in hft-research-artifacts|hft-research-dispatch-io) owner=shared ;; *) owner=prediction-markets ;; esac
cargo tree --manifest-path "$root/rust_hft/$owner/Cargo.toml" \
-p "$package" --locked --edges normal --prefix none >"$work/worker.tree"
if grep -E '^(alpha-(domain|engine|harness|onnx-evaluator) |burn |ort |hft-(collector|backtest|research-platform|research-ml|execution-adapter-[a-z-]+) )' "$work/worker.tree"; then
if grep -E '^(alpha-(domain|engine|harness|onnx-evaluator) |burn(-[a-z-]+)? |ort |hft-(collector|backtest|research-platform|research-ml|execution-adapter-[a-z-]+) )' "$work/worker.tree"; then
echo 'Prediction/artifact worker imports acquisition, CEX research, control, training or execution' >&2; exit 1
fi
if [[ $package == hft-research-artifacts || $package == hft-prediction-research-worker ]] &&
Expand All @@ -211,7 +222,7 @@ for package in hft-research-artifacts hft-research-dispatch-io hft-prediction-re
done
cargo tree --manifest-path "$root/rust_hft/runtime/Cargo.toml" \
-p hft-strategy-probability-reversal --locked --edges normal --prefix none >"$work/probability.tree"
if grep -E '^(alpha-(domain|engine|harness|store|onnx-evaluator) |burn |ort |sqlx-|axum |reqwest |hft-(collector|research-platform|research-ml|execution-adapter-[a-z-]+) )' "$work/probability.tree"; then
if grep -E '^(alpha-(domain|engine|harness|store|onnx-evaluator) |burn(-[a-z-]+)? |ort |sqlx-|axum |reqwest |hft-(collector|research-platform|research-ml|execution-adapter-[a-z-]+) )' "$work/probability.tree"; then
echo 'fixed probability strategy imports science, control, transport or execution' >&2; exit 1
fi
printf 'workspace ownership, scoped commands, feature rejection and thin closures passed\n'
2 changes: 1 addition & 1 deletion rust_hft/alpha-harness/app/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ serde_json = { workspace = true }
sha2 = { workspace = true }
tempfile = "3"
tokio = { workspace = true, features = ["macros", "rt", "sync"] }
zip = { version = "6", default-features = false, features = ["deflate"] }
zip = { version = "9.0.5", default-features = false, features = ["deflate"] }

[[bin]]
name = "alpha-harness"
Expand Down
4 changes: 2 additions & 2 deletions rust_hft/alpha-harness/app/src/mission_campaign.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7234,7 +7234,7 @@ pub(crate) mod tests {
.compression_method(zip::CompressionMethod::Deflated);
for index in 0..source.len() {
let mut entry = source.by_index(index).unwrap();
let name = entry.name().to_string();
let name = entry.name().unwrap().into_owned();
if mutation == 0 && name == block_entry {
continue;
}
Expand Down Expand Up @@ -9208,7 +9208,7 @@ pub(crate) mod tests {
let mut entry = archive.by_index(index).unwrap();
let mut bytes = Vec::new();
entry.read_to_end(&mut bytes).unwrap();
entries.insert(entry.name().to_string(), bytes);
entries.insert(entry.name().unwrap().into_owned(), bytes);
}
mutate(&mut entries);
let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ fn actual_archive_entries_with_limit(
if entry.is_dir() {
continue;
}
let path = entry.name().to_string();
let path = entry.name()?.into_owned();
if entry.enclosed_name().is_none()
|| path.len() > 256
|| path.contains("..")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ pub(crate) fn verify_artifact_inventory(
let mut actual = BTreeSet::new();
for index in 0..archive.len() {
let entry = archive.by_index(index)?;
if entry.is_dir() || !actual.insert(entry.name().to_owned()) {
if entry.is_dir() || !actual.insert(entry.name()?.into_owned()) {
bail!("unexpected sequence archive entry");
}
}
Expand Down
37 changes: 26 additions & 11 deletions rust_hft/alpha-harness/app/src/mission_runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4722,6 +4722,8 @@ fn decode_verified_execution_report(
None => {
if archive
.file_names()
.collect::<zip::result::ZipResult<Vec<_>>>()?
.into_iter()
.any(|name| name == "results/native-prepared-inputs.json")
{
bail!("native result readback requires its independently bound finalized request");
Expand Down Expand Up @@ -4810,6 +4812,8 @@ fn decode_verified_execution_report(
{
let unexpected_backtest = archive
.file_names()
.collect::<zip::result::ZipResult<Vec<_>>>()?
.into_iter()
.any(|path| path == format!("results/{name}-supervised-backtest.json"));
if baseline.is_some() || candidate.is_some() || unexpected_backtest {
bail!("published result contains a model outside its declared scope");
Expand Down Expand Up @@ -5288,14 +5292,18 @@ fn validate_model_metric_readback(
candidates: &std::collections::BTreeMap<&str, CexResearchContentRefV1>,
native: Option<&crate::mission_campaign::prepared_inputs::VerifiedNativeCampaignPreparedInputs>,
) -> anyhow::Result<()> {
let calendar_artifacts_present = archive.file_names().any(|name| {
matches!(
name,
"results/calendar-validation.json"
| "results/calendar-prediction-diagnostics.json"
| "results/calendar-validation-event-replay-receipt.json"
)
});
let calendar_artifacts_present = archive
.file_names()
.collect::<zip::result::ZipResult<Vec<_>>>()?
.into_iter()
.any(|name| {
matches!(
name.as_ref(),
"results/calendar-validation.json"
| "results/calendar-prediction-diagnostics.json"
| "results/calendar-validation-event-replay-receipt.json"
)
});
if calendar_artifacts_present
&& (mission.spec.evaluation_protocol.calendar.is_none() || selection.is_none())
{
Expand Down Expand Up @@ -5517,7 +5525,9 @@ fn load_native_archive_input(
.context("native archive is missing its actual prepared collection")?;
let names = archive
.file_names()
.map(str::to_owned)
.collect::<zip::result::ZipResult<Vec<_>>>()?
.into_iter()
.map(|name| name.into_owned())
.collect::<std::collections::BTreeSet<_>>();
if names.len() != archive.len() {
bail!("native archive contains duplicate entry names");
Expand Down Expand Up @@ -5672,7 +5682,12 @@ fn read_optional_bundle_bytes(
name: &str,
max_bytes: u64,
) -> anyhow::Result<Option<Vec<u8>>> {
if !archive.file_names().any(|entry| entry == name) {
if !archive
.file_names()
.collect::<zip::result::ZipResult<Vec<_>>>()?
.into_iter()
.any(|entry| entry == name)
{
return Ok(None);
}
read_bundle_bytes(archive, name, max_bytes).map(Some)
Expand Down Expand Up @@ -9924,7 +9939,7 @@ pub(crate) mod tests {
let mut entry = archive.by_index(index).unwrap();
let mut bytes = Vec::new();
entry.read_to_end(&mut bytes).unwrap();
entries.push((entry.name().to_string(), bytes));
entries.push((entry.name().unwrap().into_owned(), bytes));
}
entries
}
Expand Down
45 changes: 22 additions & 23 deletions rust_hft/data-pipelines/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion rust_hft/native/libduckdb-sys/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,6 @@ optional = true

[build-dependencies.zip]
optional = true
version = "6"
version = "9.0.5"
features = ["deflate"]
default-features = false
5 changes: 3 additions & 2 deletions rust_hft/native/libduckdb-sys/MONDAY-PATCH.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ compilation do not compile these dependencies. Set both the feature and
feature, that environment variable fails with an actionable error.
Bundled mode retains precedence over the download environment variable.

The owning research-core and data-pipelines manifests select this patch.
Prediction keeps its distinct locked upstream version.
The research-core, data-pipelines, and prediction-markets manifests select this patch.
The optional downloader uses zip 9.0.5 or later, including the extraction fixes
for RUSTSEC-2026-0367, RUSTSEC-2026-0368, and RUSTSEC-2026-0369.
The native cache identity binds every patch file, features, dependency checksums,
compiler, ABI, flags and profiles. Only this exact native path is retained during
cache cleanup. Application path dependencies are still removed.
Expand Down
Loading
Loading