diff --git a/Cargo.lock b/Cargo.lock index 87c19a0..fa48844 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2474,6 +2474,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.228" @@ -2528,6 +2534,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2879,6 +2894,47 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" version = "0.5.2" @@ -3143,11 +3199,13 @@ dependencies = [ "rkyv", "rustc-hash", "salsa", + "semver", "serde", "serde_json", "syn", "tempfile", "thiserror", + "toml", "tracing", "uuid", "which", @@ -3188,7 +3246,6 @@ dependencies = [ "tracing", "venus-core", "venus-sync", - "windows-sys 0.59.0", ] [[package]] @@ -3553,6 +3610,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winsafe" version = "0.0.19" diff --git a/Cargo.toml b/Cargo.toml index 3a7d9d4..fe66e2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,8 @@ venus-sync = { path = "crates/venus-sync", version = "0" } venus-server = { path = "crates/venus-server", version = "0" } # Parsing and AST +toml = "0.8" +semver = "1.0" syn = { version = "2.0", features = [ "full", "parsing", diff --git a/crates/venus-core/Cargo.toml b/crates/venus-core/Cargo.toml index bee9692..e80179f 100644 --- a/crates/venus-core/Cargo.toml +++ b/crates/venus-core/Cargo.toml @@ -7,6 +7,8 @@ repository.workspace = true description = "Core engine for Venus reactive notebook environment" [dependencies] +toml.workspace = true +semver.workspace = true syn = { workspace = true, features = ["full", "parsing", "visit", "extra-traits"] } quote.workspace = true proc-macro2.workspace = true diff --git a/crates/venus-core/src/compile/cargo_generator.rs b/crates/venus-core/src/compile/cargo_generator.rs index b8b1d8c..589dc15 100644 --- a/crates/venus-core/src/compile/cargo_generator.rs +++ b/crates/venus-core/src/compile/cargo_generator.rs @@ -148,20 +148,17 @@ pub fn generate_cargo_toml( fn format_dependency(toml: &mut String, dep: &ExternalDependency, notebook_dir: Option<&Path>) { if let Some(path) = &dep.path { // Convert relative paths to absolute if notebook_dir is provided - let abs_path = if path.is_relative() { + let canonical_path = if path.is_relative() { notebook_dir .map(|dir| dir.join(path)) .and_then(|p| p.canonicalize().ok()) - .unwrap_or_else(|| path.clone()) } else { - path.clone() + None }; + let abs_path = canonical_path.as_deref().unwrap_or(path); - toml.push_str(&format!( - "{} = {{ path = \"{}\" }}\n", - dep.name, - abs_path.display() - )); + let path = toml::Value::String(abs_path.to_string_lossy().into_owned()).to_string(); + toml.push_str(&format!("{} = {{ path = {path} }}\n", dep.name)); } else if let Some(version) = &dep.version { if dep.features.is_empty() { toml.push_str(&format!("{} = \"{}\"\n", dep.name, version)); @@ -275,6 +272,31 @@ mod tests { assert!(toml.contains("local_crate = { path = \"/absolute/path/to/crate\" }")); } + #[test] + fn path_dependencies_preserve_native_toml_strings() { + for path in [ + r"C:\notebooks\local", + r"\\?\C:\notebooks\local", + r"\\server\share\local", + r"\\?\UNC\server\share\local", + r"local\component", + r#"/notebooks/quoted "component"/local"#, + ] { + let deps = [ExternalDependency { + name: "local_crate".into(), + version: None, + features: vec![], + path: Some(PathBuf::from(path)), + }]; + let source = generate_cargo_toml(&ManifestConfig::default(), &deps, false, None); + let manifest: toml::Table = toml::from_str(&source).unwrap(); + assert_eq!( + manifest["dependencies"]["local_crate"]["path"].as_str(), + Some(path) + ); + } + } + #[test] fn test_skips_duplicate_serde() { let deps = vec![ExternalDependency { diff --git a/crates/venus-core/src/compile/cell/build.rs b/crates/venus-core/src/compile/cell/build.rs new file mode 100644 index 0000000..162d2ae --- /dev/null +++ b/crates/venus-core/src/compile/cell/build.rs @@ -0,0 +1,174 @@ +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::graph::CellInfo; + +use super::super::errors::{CompileError, ErrorMapper}; +use super::super::types::{dylib_extension, dylib_prefix}; +use super::compiler::CellCompiler; + +impl CellCompiler { + /// Compile wrapper code to a dynamic library. + pub(super) fn compile_to_dylib( + &self, + cell: &CellInfo, + wrapper_code: &str, + source_hash: u64, + deps_hash: u64, + ) -> std::result::Result> { + let build_dir = self.config.cell_build_dir(); + fs::create_dir_all(&build_dir).map_err(|e| { + CompileError::simple(format!("Failed to create build directory: {}", e)) + })?; + + // Write wrapper source + let src_file = build_dir.join(format!("{}.rs", cell.name)); + fs::write(&src_file, wrapper_code) + .map_err(|e| CompileError::simple(format!("Failed to write source: {}", e)))?; + + // Source and dependency hashes distinguish loaded library versions. + let dylib_name = format!( + "{}cell_{}_{:x}_{:x}.{}", + dylib_prefix(), + cell.name, + source_hash, + deps_hash, + dylib_extension() + ); + let dylib_path = build_dir.join(&dylib_name); + + // Clean up old dylibs for this cell (they accumulate with different hashes) + let cell_prefix = format!("{}cell_{}_", dylib_prefix(), cell.name); + if let Ok(entries) = fs::read_dir(&build_dir) { + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with(&cell_prefix) && name_str != dylib_name { + let _ = fs::remove_file(entry.path()); + } + } + } + + // Build rustc command + let mut cmd = Command::new(self.toolchain.rustc_path()); + + cmd.arg(&src_file) + .arg("--crate-type=cdylib") + .arg("--edition=2021") + .arg("-o") + .arg(&dylib_path) + .arg("--error-format=json"); + + // Add Cranelift backend if available and configured + if self.config.use_cranelift && self.toolchain.has_cranelift() { + cmd.args(self.toolchain.cranelift_flags()); + } + + // Optimization level + cmd.arg(format!("-Copt-level={}", self.config.opt_level)); + + // Debug info + if self.config.debug_info { + cmd.arg("-g"); + } + + // Link against universe rlib for compilation + if let Some(universe_dylib) = &self.universe_path { + // The universe build directory contains both cdylib and rlib + // We need the rlib for rustc compilation and cdylib for runtime + let universe_build_dir = universe_dylib.parent().unwrap_or(universe_dylib); + let target_release_dir = universe_build_dir.join("target").join("release"); + let deps_dir = target_release_dir.join("deps"); + + // Add search paths for dependencies + cmd.arg("-L").arg(&target_release_dir); + cmd.arg("-L").arg(&deps_dir); + + // Find and link the universe rlib using --extern + let rlib_path = target_release_dir.join("libvenus_universe.rlib"); + if rlib_path.exists() { + cmd.arg("--extern") + .arg(format!("venus_universe={}", rlib_path.display())); + } else { + // Fallback: try to find it in deps + if let Ok(entries) = std::fs::read_dir(&deps_dir) { + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with("libvenus_universe-") && name_str.ends_with(".rlib") + { + cmd.arg("--extern") + .arg(format!("venus_universe={}", entry.path().display())); + break; + } + } + } + } + + // Add rpath for runtime linking (Unix-like systems) + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + // Runtime links against cdylib in the universe build dir + cmd.arg(format!( + "-Clink-arg=-Wl,-rpath,{}", + universe_build_dir.display() + )); + } + + // On macOS, fix the universe dylib install_name so the dynamic + // linker can resolve it via rpath. Raw rustc sets install_name to + // the bare filename, but @rpath/ prefix is needed for rpath lookup. + #[cfg(target_os = "macos")] + { + let universe_filename = universe_dylib + .file_name() + .unwrap_or_default() + .to_string_lossy(); + let desired_install_name = format!("@rpath/{universe_filename}"); + + // Check if we need to fix install_name (cargo sets it correctly, + // but direct rustc compilation does not) + let output = Command::new("otool") + .args(["-D", &universe_dylib.to_string_lossy()]) + .output(); + if let Ok(output) = output { + let stdout = String::from_utf8_lossy(&output.stdout); + if !stdout.contains("@rpath") { + let _ = Command::new("install_name_tool") + .args([ + "-id", + &desired_install_name, + &universe_dylib.to_string_lossy(), + ]) + .status(); + } + } + } + } + + // Extra flags + cmd.args(&self.config.extra_rustc_flags); + + // Run compilation + let output = cmd + .output() + .map_err(|e| CompileError::simple(format!("Failed to run rustc: {}", e)))?; + + if output.status.success() { + Ok(dylib_path) + } else { + // Parse errors + let stderr = String::from_utf8_lossy(&output.stderr); + let mapper = ErrorMapper::new(cell.source_file.clone()); + let errors = mapper.parse_rustc_output(&stderr); + + if errors.is_empty() { + // Fallback if JSON parsing failed + Err(CompileError::simple_rendered(stderr.to_string())) + } else { + Err(errors) + } + } + } +} diff --git a/crates/venus-core/src/compile/cell.rs b/crates/venus-core/src/compile/cell/compiler.rs similarity index 58% rename from crates/venus-core/src/compile/cell.rs rename to crates/venus-core/src/compile/cell/compiler.rs index 962dd19..cd81f3a 100644 --- a/crates/venus-core/src/compile/cell.rs +++ b/crates/venus-core/src/compile/cell/compiler.rs @@ -7,27 +7,25 @@ use std::collections::hash_map::DefaultHasher; use std::fs; use std::hash::{Hash, Hasher}; use std::path::PathBuf; -use std::process::Command; use std::time::Instant; use crate::graph::CellInfo; -use super::errors::ErrorMapper; -use super::toolchain::ToolchainManager; -use super::types::{ +use super::super::toolchain::ToolchainManager; +use super::super::types::{ CompilationResult, CompiledCell, CompilerConfig, dylib_extension, dylib_prefix, }; /// Compiles individual cells to dynamic libraries. pub struct CellCompiler { /// Compiler configuration - config: CompilerConfig, + pub(super) config: CompilerConfig, /// Toolchain manager - toolchain: ToolchainManager, + pub(super) toolchain: ToolchainManager, /// Path to the universe library (for linking) - universe_path: Option, + pub(super) universe_path: Option, } impl CellCompiler { @@ -60,8 +58,8 @@ impl CellCompiler { // Generate wrapper code let wrapper_code = self.generate_wrapper(cell); - // Compile - include source_hash in dylib name to force reload on changes - match self.compile_to_dylib(cell, &wrapper_code, source_hash) { + // Compile with source and dependency identity. + match self.compile_to_dylib(cell, &wrapper_code, source_hash, deps_hash) { Ok(dylib_path) => { let compile_time = start.elapsed().as_millis() as u64; @@ -290,172 +288,6 @@ impl CellCompiler { code } - /// Compile wrapper code to a dynamic library. - fn compile_to_dylib( - &self, - cell: &CellInfo, - wrapper_code: &str, - source_hash: u64, - ) -> std::result::Result> { - let build_dir = self.config.cell_build_dir(); - fs::create_dir_all(&build_dir).map_err(|e| { - super::CompileError::simple(format!("Failed to create build directory: {}", e)) - })?; - - // Write wrapper source - let src_file = build_dir.join(format!("{}.rs", cell.name)); - fs::write(&src_file, wrapper_code) - .map_err(|e| super::CompileError::simple(format!("Failed to write source: {}", e)))?; - - // Output path - include hash to force dlopen to reload on changes - // (Linux caches shared libraries by path, so we need unique paths) - let dylib_name = format!( - "{}cell_{}_{:x}.{}", - dylib_prefix(), - cell.name, - source_hash, - dylib_extension() - ); - let dylib_path = build_dir.join(&dylib_name); - - // Clean up old dylibs for this cell (they accumulate with different hashes) - let cell_prefix = format!("{}cell_{}_", dylib_prefix(), cell.name); - if let Ok(entries) = fs::read_dir(&build_dir) { - for entry in entries.flatten() { - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - if name_str.starts_with(&cell_prefix) && name_str != dylib_name { - let _ = fs::remove_file(entry.path()); - } - } - } - - // Build rustc command - let mut cmd = Command::new(self.toolchain.rustc_path()); - - cmd.arg(&src_file) - .arg("--crate-type=cdylib") - .arg("--edition=2021") - .arg("-o") - .arg(&dylib_path) - .arg("--error-format=json"); - - // Add Cranelift backend if available and configured - if self.config.use_cranelift && self.toolchain.has_cranelift() { - for flag in self.toolchain.cranelift_flags() { - cmd.arg(&flag); - } - } - - // Optimization level - cmd.arg(format!("-Copt-level={}", self.config.opt_level)); - - // Debug info - if self.config.debug_info { - cmd.arg("-g"); - } - - // Link against universe rlib for compilation - if let Some(universe_dylib) = &self.universe_path { - // The universe build directory contains both cdylib and rlib - // We need the rlib for rustc compilation and cdylib for runtime - let universe_build_dir = universe_dylib.parent().unwrap_or(universe_dylib); - let target_release_dir = universe_build_dir.join("target").join("release"); - let deps_dir = target_release_dir.join("deps"); - - // Add search paths for dependencies - cmd.arg("-L").arg(&target_release_dir); - cmd.arg("-L").arg(&deps_dir); - - // Find and link the universe rlib using --extern - let rlib_path = target_release_dir.join("libvenus_universe.rlib"); - if rlib_path.exists() { - cmd.arg("--extern") - .arg(format!("venus_universe={}", rlib_path.display())); - } else { - // Fallback: try to find it in deps - if let Ok(entries) = std::fs::read_dir(&deps_dir) { - for entry in entries.flatten() { - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - if name_str.starts_with("libvenus_universe-") && name_str.ends_with(".rlib") - { - cmd.arg("--extern") - .arg(format!("venus_universe={}", entry.path().display())); - break; - } - } - } - } - - // Add rpath for runtime linking (Unix-like systems) - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - // Runtime links against cdylib in the universe build dir - cmd.arg(format!( - "-Clink-arg=-Wl,-rpath,{}", - universe_build_dir.display() - )); - } - - // On macOS, fix the universe dylib install_name so the dynamic - // linker can resolve it via rpath. Raw rustc sets install_name to - // the bare filename, but @rpath/ prefix is needed for rpath lookup. - #[cfg(target_os = "macos")] - { - let universe_filename = universe_dylib - .file_name() - .unwrap_or_default() - .to_string_lossy(); - let desired_install_name = format!("@rpath/{universe_filename}"); - - // Check if we need to fix install_name (cargo sets it correctly, - // but direct rustc compilation does not) - let output = Command::new("otool") - .args(["-D", &universe_dylib.to_string_lossy()]) - .output(); - if let Ok(output) = output { - let stdout = String::from_utf8_lossy(&output.stdout); - if !stdout.contains("@rpath") { - let _ = Command::new("install_name_tool") - .args([ - "-id", - &desired_install_name, - &universe_dylib.to_string_lossy(), - ]) - .status(); - } - } - } - } - - // Extra flags - for flag in &self.config.extra_rustc_flags { - cmd.arg(flag); - } - - // Run compilation - let output = cmd - .output() - .map_err(|e| super::CompileError::simple(format!("Failed to run rustc: {}", e)))?; - - if output.status.success() { - Ok(dylib_path) - } else { - // Parse errors - let stderr = String::from_utf8_lossy(&output.stderr); - let mapper = ErrorMapper::new(cell.source_file.clone()); - let errors = mapper.parse_rustc_output(&stderr); - - if errors.is_empty() { - // Fallback if JSON parsing failed - Err(super::CompileError::simple_rendered(stderr.to_string())) - } else { - Err(errors) - } - } - } - /// Hash the source code. fn hash_source(&self, source: &str) -> u64 { let mut hasher = DefaultHasher::new(); @@ -478,10 +310,10 @@ impl CellCompiler { // Read cache metadata let meta_file = self.cache_meta_path(&cell.name); if let Ok(meta) = fs::read_to_string(&meta_file) { - let lines: Vec<&str> = meta.lines().collect(); - if lines.len() >= 2 + let mut lines = meta.lines(); + if let (Some(cached_src), Some(cached_deps)) = (lines.next(), lines.next()) && let (Ok(cached_src), Ok(cached_deps)) = - (lines[0].parse::(), lines[1].parse::()) + (cached_src.parse::(), cached_deps.parse::()) && cached_src == source_hash && cached_deps == deps_hash { @@ -535,89 +367,5 @@ impl CellCompiler { } #[cfg(test)] -mod tests { - use super::*; - use crate::graph::{CellId, Dependency, SourceSpan}; - - fn make_test_cell() -> CellInfo { - CellInfo { - id: CellId::new(0), - name: "test_cell".to_string(), - display_name: "test_cell".to_string(), - dependencies: vec![], - return_type: "i32".to_string(), - doc_comment: None, - source_code: "pub fn test_cell() -> i32 { 42 }".to_string(), - source_file: PathBuf::from("test.rs"), - span: SourceSpan { - start_line: 1, - start_col: 0, - end_line: 1, - end_col: 30, - }, - } - } - - #[test] - fn test_generate_wrapper_simple() { - let config = CompilerConfig::default(); - let toolchain = ToolchainManager::new().unwrap(); - let compiler = CellCompiler::new(config, toolchain); - - let cell = make_test_cell(); - let wrapper = compiler.generate_wrapper(&cell); - - assert!(wrapper.contains("venus_cell_test_cell")); - assert!(wrapper.contains("pub fn test_cell() -> i32")); - assert!(wrapper.contains("#[no_mangle]")); - } - - #[test] - fn test_generate_wrapper_with_deps() { - let config = CompilerConfig::default(); - let toolchain = ToolchainManager::new().unwrap(); - let compiler = CellCompiler::new(config, toolchain); - - let cell = CellInfo { - id: CellId::new(1), - name: "process".to_string(), - display_name: "process".to_string(), - dependencies: vec![Dependency { - param_name: "config".to_string(), - param_type: "Config".to_string(), - is_ref: true, - is_mut: false, - }], - return_type: "Output".to_string(), - doc_comment: None, - source_code: "pub fn process(config: &Config) -> Output { todo!() }".to_string(), - source_file: PathBuf::from("test.rs"), - span: SourceSpan { - start_line: 5, - start_col: 0, - end_line: 5, - end_col: 50, - }, - }; - - let wrapper = compiler.generate_wrapper(&cell); - - assert!(wrapper.contains("config_ptr: *const u8")); - assert!(wrapper.contains("config_len: usize")); - assert!(wrapper.contains("rkyv::access")); - } - - #[test] - fn test_hash_source() { - let config = CompilerConfig::default(); - let toolchain = ToolchainManager::new().unwrap(); - let compiler = CellCompiler::new(config, toolchain); - - let hash1 = compiler.hash_source("fn foo() {}"); - let hash2 = compiler.hash_source("fn foo() {}"); - let hash3 = compiler.hash_source("fn bar() {}"); - - assert_eq!(hash1, hash2); - assert_ne!(hash1, hash3); - } -} +#[path = "tests.rs"] +mod tests; diff --git a/crates/venus-core/src/compile/cell/mod.rs b/crates/venus-core/src/compile/cell/mod.rs new file mode 100644 index 0000000..b7d2a80 --- /dev/null +++ b/crates/venus-core/src/compile/cell/mod.rs @@ -0,0 +1,6 @@ +//! Cell compilation and dependency-aware dynamic library builds. + +mod build; +mod compiler; + +pub use compiler::CellCompiler; diff --git a/crates/venus-core/src/compile/cell/tests.rs b/crates/venus-core/src/compile/cell/tests.rs new file mode 100644 index 0000000..ea720c4 --- /dev/null +++ b/crates/venus-core/src/compile/cell/tests.rs @@ -0,0 +1,152 @@ +use super::*; +use crate::graph::{CellId, Dependency, SourceSpan}; +use std::path::Path; +use std::process::Command; + +fn build_universe(toolchain: &ToolchainManager, build_dir: &Path, value: i32) { + let release_dir = build_dir.join("target/release"); + fs::create_dir_all(release_dir.join("deps")).unwrap(); + let source = build_dir.join("universe.rs"); + fs::write(&source, format!("pub fn value() -> i32 {{ {value} }}")).unwrap(); + let output = Command::new(toolchain.rustc_path()) + .arg(&source) + .args([ + "--crate-name=venus_universe", + "--crate-type=rlib", + "--edition=2021", + "-o", + ]) + .arg(release_dir.join("libvenus_universe.rlib")) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn dependency_changes_load_distinct_libraries_while_retaining_previous_values() { + let directory = tempfile::tempdir().unwrap(); + let config = CompilerConfig { + build_dir: directory.path().join("build"), + cache_dir: directory.path().join("cache"), + use_cranelift: false, + ..CompilerConfig::default() + }; + let toolchain = ToolchainManager::new().unwrap(); + let universe_dir = config.universe_build_dir(); + build_universe(&toolchain, &universe_dir, 40); + let universe_path = universe_dir.join(format!( + "{}venus_universe.{}", + dylib_prefix(), + dylib_extension() + )); + let compiler = CellCompiler::new(config, toolchain).with_universe(universe_path); + let cell = make_test_cell(); + let source_hash = compiler.hash_source(&cell.source_code); + let wrapper = "extern crate venus_universe;\n#[no_mangle]\npub extern \"C\" fn cell_value() -> i32 { venus_universe::value() }"; + let first_path = compiler + .compile_to_dylib(&cell, wrapper, source_hash, 1) + .unwrap(); + unsafe { + let first_library = libloading::Library::new(&first_path).unwrap(); + let first_value: libloading::Symbol i32> = + first_library.get(b"cell_value").unwrap(); + assert_eq!(first_value(), 40); + + build_universe(&compiler.toolchain, &universe_dir, 80); + let second_path = compiler + .compile_to_dylib(&cell, wrapper, source_hash, 2) + .unwrap(); + assert_ne!(first_path, second_path); + let second_library = libloading::Library::new(&second_path).unwrap(); + let second_value: libloading::Symbol i32> = + second_library.get(b"cell_value").unwrap(); + assert_eq!(second_value(), 80); + assert_eq!(first_value(), 40); + } +} + +fn make_test_cell() -> CellInfo { + CellInfo { + id: CellId::new(0), + name: "test_cell".to_string(), + display_name: "test_cell".to_string(), + dependencies: vec![], + return_type: "i32".to_string(), + doc_comment: None, + source_code: "pub fn test_cell() -> i32 { 42 }".to_string(), + source_file: PathBuf::from("test.rs"), + span: SourceSpan { + start_line: 1, + start_col: 0, + end_line: 1, + end_col: 30, + }, + } +} + +#[test] +fn test_generate_wrapper_simple() { + let config = CompilerConfig::default(); + let toolchain = ToolchainManager::new().unwrap(); + let compiler = CellCompiler::new(config, toolchain); + + let cell = make_test_cell(); + let wrapper = compiler.generate_wrapper(&cell); + + assert!(wrapper.contains("venus_cell_test_cell")); + assert!(wrapper.contains("pub fn test_cell() -> i32")); + assert!(wrapper.contains("#[no_mangle]")); +} + +#[test] +fn test_generate_wrapper_with_deps() { + let config = CompilerConfig::default(); + let toolchain = ToolchainManager::new().unwrap(); + let compiler = CellCompiler::new(config, toolchain); + + let cell = CellInfo { + id: CellId::new(1), + name: "process".to_string(), + display_name: "process".to_string(), + dependencies: vec![Dependency { + param_name: "config".to_string(), + param_type: "Config".to_string(), + is_ref: true, + is_mut: false, + }], + return_type: "Output".to_string(), + doc_comment: None, + source_code: "pub fn process(config: &Config) -> Output { todo!() }".to_string(), + source_file: PathBuf::from("test.rs"), + span: SourceSpan { + start_line: 5, + start_col: 0, + end_line: 5, + end_col: 50, + }, + }; + + let wrapper = compiler.generate_wrapper(&cell); + + assert!(wrapper.contains("config_ptr: *const u8")); + assert!(wrapper.contains("config_len: usize")); + assert!(wrapper.contains("rkyv::access")); +} + +#[test] +fn test_hash_source() { + let config = CompilerConfig::default(); + let toolchain = ToolchainManager::new().unwrap(); + let compiler = CellCompiler::new(config, toolchain); + + let hash1 = compiler.hash_source("fn foo() {}"); + let hash2 = compiler.hash_source("fn foo() {}"); + let hash3 = compiler.hash_source("fn bar() {}"); + + assert_eq!(hash1, hash2); + assert_ne!(hash1, hash3); +} diff --git a/crates/venus-core/src/compile/dependency_parser.rs b/crates/venus-core/src/compile/dependency_parser.rs index 3095fc5..2f55867 100644 --- a/crates/venus-core/src/compile/dependency_parser.rs +++ b/crates/venus-core/src/compile/dependency_parser.rs @@ -94,57 +94,52 @@ impl DependencyParser { /// //! ``` /// ``` pub fn parse(&mut self, source: &str) -> &[ExternalDependency] { - self.dependencies.clear(); - - let mut in_cargo_block = false; - let mut in_dependencies = false; - let mut toml_content = String::new(); - - for line in source.lines() { - let trimmed = line.trim(); - - // Check for cargo block markers - if trimmed.starts_with("//!") { - let content = trimmed.trim_start_matches("//!").trim(); - - if content == "```cargo" { - in_cargo_block = true; - continue; - } - - if content == "```" && in_cargo_block { - in_cargo_block = false; - in_dependencies = false; - continue; - } - - if in_cargo_block { - if content == "[dependencies]" { - in_dependencies = true; - continue; - } - - if content.starts_with('[') { - in_dependencies = false; - continue; - } - - if in_dependencies && !content.is_empty() { - toml_content.push_str(content); - toml_content.push('\n'); - } - } - } - } - - // Parse the TOML content - if !toml_content.is_empty() { - self.parse_toml_dependencies(&toml_content); - } - + self.dependencies = + super::notebook_manifest::notebook_document(source, std::path::Path::new("notebook")) + .ok() + .and_then(|mut document| { + document + .remove("dependencies") + .and_then(|value| match value { + toml::Value::Table(table) => Some(table), + _ => None, + }) + }) + .map(|dependencies| { + dependencies + .iter() + .map(|(name, value)| ExternalDependency { + name: name.clone(), + version: value + .as_str() + .or_else(|| value.get("version").and_then(toml::Value::as_str)) + .map(str::to_string), + features: value + .get("features") + .and_then(toml::Value::as_array) + .map(|features| { + features + .iter() + .filter_map(toml::Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(), + path: value + .get("path") + .and_then(toml::Value::as_str) + .map(PathBuf::from), + }) + .collect() + }) + .unwrap_or_default(); &self.dependencies } + pub(super) fn set_dependencies(&mut self, dependencies: Vec) { + self.dependencies = dependencies; + } + /// Get the parsed dependencies. pub fn dependencies(&self) -> &[ExternalDependency] { &self.dependencies @@ -156,85 +151,6 @@ impl DependencyParser { self.dependencies.hash(&mut hasher); hasher.finish() } - - /// Parse TOML-format dependencies. - fn parse_toml_dependencies(&mut self, toml: &str) { - for line in toml.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - - // Parse: name = "version" or name = { version = "...", ... } - if let Some((name, value)) = line.split_once('=') { - let name = name.trim().to_string(); - let value = value.trim(); - - let dep = if value.starts_with('"') { - // Simple version: name = "1.0" - let version = value.trim_matches('"').to_string(); - ExternalDependency { - name, - version: Some(version), - features: Vec::new(), - path: None, - } - } else if value.starts_with('{') { - // Table format: name = { version = "1.0", features = [...] } - Self::parse_table_dependency(name, value) - } else { - continue; - }; - - self.dependencies.push(dep); - } - } - } - - /// Parse a table-format dependency. - fn parse_table_dependency(name: String, value: &str) -> ExternalDependency { - let mut version = None; - let mut features = Vec::new(); - let mut path = None; - - // Simple parser for inline tables - let content = value.trim_start_matches('{').trim_end_matches('}'); - - for part in content.split(',') { - let part = part.trim(); - if let Some((key, val)) = part.split_once('=') { - let key = key.trim(); - let val = val.trim(); - - match key { - "version" => { - version = Some(val.trim_matches('"').to_string()); - } - "path" => { - path = Some(PathBuf::from(val.trim_matches('"'))); - } - "features" => { - // Parse array: ["feat1", "feat2"] - let arr = val.trim_start_matches('[').trim_end_matches(']'); - for feat in arr.split(',') { - let feat = feat.trim().trim_matches('"'); - if !feat.is_empty() { - features.push(feat.to_string()); - } - } - } - _ => {} - } - } - } - - ExternalDependency { - name, - version, - features, - path, - } - } } impl Default for DependencyParser { diff --git a/crates/venus-core/src/compile/mod.rs b/crates/venus-core/src/compile/mod.rs index ffc615a..cc48625 100644 --- a/crates/venus-core/src/compile/mod.rs +++ b/crates/venus-core/src/compile/mod.rs @@ -24,6 +24,7 @@ mod cell; mod definition_processor; mod dependency_parser; mod errors; +mod notebook_manifest; mod production; mod source_processor; mod toolchain; @@ -34,6 +35,7 @@ pub use cargo_generator::{ManifestConfig, ReleaseProfile, generate_cargo_toml}; pub use cell::CellCompiler; pub use dependency_parser::{DependencyParser, ExternalDependency}; pub use errors::{CompileError, ErrorMapper}; +pub use notebook_manifest::find_notebook_manifest; pub use production::ProductionBuilder; pub use source_processor::NotebookSourceProcessor; pub use toolchain::ToolchainManager; diff --git a/crates/venus-core/src/compile/notebook_manifest/context.rs b/crates/venus-core/src/compile/notebook_manifest/context.rs new file mode 100644 index 0000000..2e6d9f0 --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/context.rs @@ -0,0 +1,187 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use toml::{Table, Value}; + +use crate::error::{Error, Result}; + +/// Find the nearest Cargo manifest above a notebook. +pub fn find_notebook_manifest(notebook_path: &Path) -> Option { + let path = absolute(notebook_path).ok()?; + path.parent()? + .ancestors() + .map(|dir| dir.join("Cargo.toml")) + .find(|path| path.is_file()) +} + +#[derive(Debug, Clone)] +pub(in crate::compile) struct NotebookContext { + pub directory: PathBuf, + pub manifest: Option, +} + +impl NotebookContext { + pub fn for_notebook(path: &Path) -> Result { + let path = absolute(path)?; + let directory = path + .parent() + .ok_or_else(|| manifest_error(&path, "Notebook has no parent directory"))? + .to_path_buf(); + Ok(Self { + directory, + manifest: find_notebook_manifest(&path), + }) + } + + pub fn from_manifest(manifest: Option) -> Self { + let directory = manifest + .as_ref() + .and_then(|path| path.parent()) + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + Self { + directory, + manifest, + } + } + + pub(super) fn read(&self) -> Result> { + let Some(path) = &self.manifest else { + return Ok(None); + }; + let path = absolute(path)?; + let document = read_document(&path)?; + let workspace_path = owning_workspace(&path, &document)?; + let workspace = match workspace_path { + Some(root) if root == path => Some((root, document.clone())), + Some(root) => { + let document = read_document(&root)?; + Some((root, document)) + } + None => None, + }; + Ok(Some(ParentManifest { + path, + document, + workspace, + })) + } +} + +pub(super) struct ParentManifest { + pub path: PathBuf, + pub document: Table, + pub workspace: Option<(PathBuf, Table)>, +} + +fn owning_workspace(path: &Path, document: &Table) -> Result> { + if document.contains_key("workspace") { + return Ok(Some(path.to_path_buf())); + } + if let Some(location) = document + .get("package") + .and_then(Value::as_table) + .and_then(|package| package.get("workspace")) + { + let location = location + .as_str() + .ok_or_else(|| manifest_error(path, "package.workspace must be a directory path"))?; + let root = path + .parent() + .unwrap_or(Path::new(".")) + .join(location) + .join("Cargo.toml"); + let root = absolute(&root)?; + if !read_document(&root)?.contains_key("workspace") { + return Err(manifest_error( + &root, + "package.workspace does not identify a workspace root", + )); + } + return Ok(Some(root)); + } + for ancestor in path + .parent() + .and_then(Path::parent) + .into_iter() + .flat_map(Path::ancestors) + { + let root = ancestor.join("Cargo.toml"); + if root.is_file() && read_document(&root)?.contains_key("workspace") { + let output = Command::new("cargo") + .args([ + "locate-project", + "--workspace", + "--message-format", + "plain", + "--manifest-path", + ]) + .arg(path) + .output() + .map_err(|error| { + manifest_error( + path, + &format!( + "Cannot discover owning workspace: {error}. Check Cargo installation" + ), + ) + })?; + if !output.status.success() { + return Err(manifest_error( + path, + &format!( + "Cannot discover owning workspace: {}. Correct the manifest or workspace membership", + String::from_utf8_lossy(&output.stderr).trim() + ), + )); + } + let workspace = PathBuf::from(String::from_utf8_lossy(&output.stdout).trim()); + return if workspace == path { + Ok(None) + } else { + Ok(Some(absolute(&workspace)?)) + }; + } + } + Ok(None) +} + +pub(super) fn absolute(path: &Path) -> Result { + let path = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir()?.join(path) + }; + Ok(path.canonicalize().unwrap_or(path)) +} + +pub(super) fn read_document(path: &Path) -> Result { + let source = fs::read_to_string(path) + .map_err(|error| manifest_error(path, &format!("Cannot read manifest: {error}")))?; + let document: Table = toml::from_str(&source) + .map_err(|error| manifest_error(path, &format!("Cannot parse manifest: {error}")))?; + for section in [ + "package", + "workspace", + "dependencies", + "target", + "features", + "patch", + ] { + if document.get(section).is_some_and(|value| !value.is_table()) { + return Err(manifest_error(path, &format!("{section} must be a table"))); + } + } + Ok(document) +} + +pub(super) fn manifest_error(path: &Path, message: &str) -> Error { + Error::Compilation { + cell_id: None, + message: format!( + "Cargo context '{}': {message}. Correct the manifest dependency declarations", + path.display() + ), + } +} diff --git a/crates/venus-core/src/compile/notebook_manifest/exports.rs b/crates/venus-core/src/compile/notebook_manifest/exports.rs new file mode 100644 index 0000000..4d1ad52 --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/exports.rs @@ -0,0 +1,138 @@ +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::hash::{Hash, Hasher}; + +use toml::Value; + +use super::resolve::ResolvedManifest; + +impl ResolvedManifest { + pub fn reexports(&self) -> String { + let mut aliases: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + for (alias, dependency) in &self.dependencies { + aliases + .entry(alias) + .or_default() + .insert(self.dependency_gate(alias, dependency, None)); + } + for (target, table) in &self.targets { + if let Some(dependencies) = table.get("dependencies").and_then(Value::as_table) { + for (alias, dependency) in dependencies { + aliases + .entry(alias) + .or_default() + .insert(self.dependency_gate(alias, dependency, Some(target))); + } + } + } + let mut source = String::new(); + for (alias, gates) in aliases { + if matches!(alias, "rkyv" | "serde_json" | "venus") { + continue; + } + if !gates.contains("") { + source.push_str(&format!( + "#[cfg(any({}))]\n", + gates.into_iter().collect::>().join(", ") + )); + } + source.push_str(&format!("pub use {};\n", alias.replace('-', "_"))); + } + source + } + + pub fn build_script(&self) -> String { + let mut source = String::from( + "fn main() {\n let target = std::env::var(\"TARGET\").unwrap_or_default();\n", + ); + for target in self + .targets + .keys() + .filter(|target| !target.starts_with("cfg(")) + { + let cfg = target_cfg(target); + source.push_str(&format!(" println!(\"cargo:rustc-check-cfg=cfg({cfg})\");\n if target == {target:?} {{ println!(\"cargo:rustc-cfg={cfg}\"); }}\n")); + } + source.push_str("}\n"); + source + } + + fn dependency_gate(&self, alias: &str, dependency: &Value, target: Option<&str>) -> String { + let mut gates = Vec::new(); + if let Some(target) = target { + gates.push( + if let Some(expression) = target + .strip_prefix("cfg(") + .and_then(|target| target.strip_suffix(')')) + { + expression.into() + } else { + target_cfg(target) + }, + ); + } + if dependency.get("optional").and_then(Value::as_bool) == Some(true) { + let implicit = !self.features.contains_key(alias) + && !self + .features + .values() + .filter_map(Value::as_array) + .flatten() + .filter_map(Value::as_str) + .any(|feature| feature.strip_prefix("dep:") == Some(alias)); + let mut activators = BTreeSet::new(); + if implicit { + activators.insert(alias.to_string()); + } + for feature in self.features.keys() { + if self.activates(feature, alias, implicit, &mut HashSet::new()) { + activators.insert(feature.clone()); + } + } + gates.push(format!( + "any({})", + activators + .iter() + .map(|feature| format!("feature = {feature:?}")) + .collect::>() + .join(", ") + )); + } + match gates.len() { + 0 => String::new(), + 1 => gates.remove(0), + _ => format!("all({})", gates.join(", ")), + } + } + + fn activates( + &self, + feature: &str, + alias: &str, + implicit: bool, + visited: &mut HashSet, + ) -> bool { + if !visited.insert(feature.into()) { + return false; + } + self.features + .get(feature) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .any(|member| { + member.strip_prefix("dep:") == Some(alias) + || (implicit && member == alias) + || member + .strip_prefix(alias) + .is_some_and(|suffix| suffix.starts_with('/')) + || self.activates(member, alias, implicit, visited) + }) + } +} + +fn target_cfg(target: &str) -> String { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + target.hash(&mut hasher); + format!("venus_target_{:x}", hasher.finish()) +} diff --git a/crates/venus-core/src/compile/notebook_manifest/feature_graph.rs b/crates/venus-core/src/compile/notebook_manifest/feature_graph.rs new file mode 100644 index 0000000..21c4b86 --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/feature_graph.rs @@ -0,0 +1,160 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use toml::{Table, Value}; + +use crate::error::Result; + +use super::context::manifest_error; +use super::resolve::ResolvedManifest; + +impl ResolvedManifest { + pub fn optional_aliases(&self) -> BTreeSet { + self.dependencies + .iter() + .chain( + self.targets + .values() + .filter_map(|target| target.get("dependencies").and_then(Value::as_table)) + .flat_map(Table::iter), + ) + .filter(|(_, value)| value.get("optional").and_then(Value::as_bool) == Some(true)) + .map(|(alias, _)| alias.clone()) + .collect() + } + + pub fn normalize_required_feature(&mut self, alias: &str, was_optional: bool) { + let activation = format!("dep:{alias}"); + let weak_forward = format!("{alias}?/"); + let explicit_activation = self + .features + .values() + .filter_map(Value::as_array) + .flatten() + .filter_map(Value::as_str) + .any(|member| member == activation); + if was_optional && !explicit_activation { + self.features + .entry(alias) + .or_insert_with(|| Value::Array(Vec::new())); + } + for feature in self + .features + .iter_mut() + .map(|(_, value)| value) + .filter_map(Value::as_array_mut) + { + feature.retain(|member| member.as_str() != Some(activation.as_str())); + for member in feature { + if let Some(forward) = member + .as_str() + .and_then(|member| member.strip_prefix(weak_forward.as_str())) + { + *member = Value::String(format!("{alias}/{forward}")); + } + } + } + } + + pub fn prune_omitted_features(&mut self, parent: &Table) { + let mut omitted = BTreeSet::new(); + for table in std::iter::once(parent).chain( + parent + .get("target") + .and_then(Value::as_table) + .into_iter() + .flat_map(Table::values) + .filter_map(Value::as_table), + ) { + for section in ["build-dependencies", "dev-dependencies"] { + if let Some(dependencies) = table.get(section).and_then(Value::as_table) { + omitted.extend(dependencies.keys().cloned()); + } + } + } + for (alias, _) in self.dependencies.iter().chain( + self.targets + .values() + .filter_map(|target| target.get("dependencies").and_then(Value::as_table)) + .flat_map(Table::iter), + ) { + omitted.remove(alias); + } + let nodes = self.features.keys().cloned().collect::>(); + for feature in self + .features + .iter_mut() + .map(|(_, value)| value) + .filter_map(Value::as_array_mut) + { + feature.retain(|member| { + let Some(member) = member.as_str() else { + return true; + }; + let alias = member + .strip_prefix("dep:") + .unwrap_or(member) + .split('/') + .next() + .unwrap_or(member) + .trim_end_matches('?'); + !omitted.contains(alias) + || (!member.starts_with("dep:") + && !member.contains('/') + && nodes.contains(member)) + }); + } + } + + pub fn check_sources(&self, origin: &Path) -> Result<()> { + let mut sources = BTreeMap::new(); + for (alias, dependency) in self.dependencies.iter().chain( + self.targets + .values() + .filter_map(|target| target.get("dependencies").and_then(Value::as_table)) + .flat_map(Table::iter), + ) { + let source = [ + "package", + "path", + "git", + "rev", + "branch", + "tag", + "registry", + "registry-index", + ] + .iter() + .map(|key| { + ( + *key, + if *key == "package" { + Some(Value::String( + dependency + .get(*key) + .and_then(Value::as_str) + .unwrap_or(alias) + .into(), + )) + } else { + dependency.get(*key).cloned() + }, + ) + }) + .collect::>(); + if sources + .get(alias) + .is_some_and(|existing| existing != &source) + { + return Err(manifest_error( + origin, + &format!( + "Dependency alias '{alias}' selects different packages or sources across targets. Use distinct aliases or matching sources" + ), + )); + } + sources.insert(alias, source); + } + Ok(()) + } +} diff --git a/crates/venus-core/src/compile/notebook_manifest/mod.rs b/crates/venus-core/src/compile/notebook_manifest/mod.rs new file mode 100644 index 0000000..0761d95 --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/mod.rs @@ -0,0 +1,16 @@ +//! Cargo dependency context for generated notebook projects. + +mod context; +mod exports; +mod feature_graph; +mod render; +mod resolve; +mod runtime; +mod spec; +#[cfg(test)] +mod tests; + +pub(super) use context::NotebookContext; +pub use context::find_notebook_manifest; +pub(super) use resolve::ResolvedManifest; +pub(super) use spec::notebook_document; diff --git a/crates/venus-core/src/compile/notebook_manifest/render.rs b/crates/venus-core/src/compile/notebook_manifest/render.rs new file mode 100644 index 0000000..310ca1b --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/render.rs @@ -0,0 +1,82 @@ +use toml::{Table, Value}; + +use crate::compile::ManifestConfig; +use crate::error::Result; + +use super::context::manifest_error; +use super::resolve::ResolvedManifest; + +impl ResolvedManifest { + pub fn render(&self, config: &ManifestConfig<'_>) -> Result { + let mut manifest = self.sections(); + manifest.insert( + "package".into(), + Value::Table(Table::from_iter([ + ("name".into(), Value::String(config.name.into())), + ("version".into(), Value::String(config.version.into())), + ("edition".into(), Value::String(config.edition.into())), + ])), + ); + if let Some(types) = config.lib_crate_types { + manifest.insert( + "lib".into(), + Value::Table(Table::from_iter([( + "crate-type".into(), + Value::Array( + types + .iter() + .map(|name| Value::String((*name).into())) + .collect(), + ), + )])), + ); + } + if let Some(profile) = &config.release_profile { + manifest.insert( + "profile".into(), + Value::Table(Table::from_iter([( + "release".into(), + Value::Table(Table::from_iter([ + ("opt-level".into(), Value::Integer(profile.opt_level.into())), + ("lto".into(), Value::Boolean(profile.lto)), + ( + "codegen-units".into(), + Value::Integer(profile.codegen_units.into()), + ), + ("panic".into(), Value::String(profile.panic.into())), + ])), + )])), + ); + } + if config.standalone_workspace { + manifest.insert("workspace".into(), Value::Table(Table::new())); + } + toml::to_string(&manifest).map_err(|error| { + manifest_error( + std::path::Path::new(config.name), + &format!("Cannot render generated manifest: {error}"), + ) + }) + } + + pub fn canonical(&self) -> String { + Value::Table(self.sections()).to_string() + } + + fn sections(&self) -> Table { + let mut sections = Table::from_iter([( + "dependencies".into(), + Value::Table(self.dependencies.clone()), + )]); + for (name, table) in [ + ("target", &self.targets), + ("features", &self.features), + ("patch", &self.patches), + ] { + if !table.is_empty() { + sections.insert(name.into(), Value::Table(table.clone())); + } + } + sections + } +} diff --git a/crates/venus-core/src/compile/notebook_manifest/resolve.rs b/crates/venus-core/src/compile/notebook_manifest/resolve.rs new file mode 100644 index 0000000..6344e67 --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/resolve.rs @@ -0,0 +1,333 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use toml::{Table, Value}; + +use crate::compile::ExternalDependency; +use crate::error::Result; + +use super::context::{NotebookContext, ParentManifest, manifest_error}; +use super::spec::{dependency_table, notebook_document, rebase_path, string_array}; + +#[derive(Debug, Clone, Default)] +pub(in crate::compile) struct ResolvedManifest { + pub dependencies: Table, + pub targets: Table, + pub features: Table, + pub patches: Table, +} + +impl ResolvedManifest { + pub fn resolve(context: &NotebookContext, source: &str) -> Result { + let parent = context.read()?; + let mut resolved = Self::default(); + if let Some(parent) = &parent { + let directory = parent.path.parent().unwrap_or(Path::new(".")); + let virtual_root = !parent.document.contains_key("package") + && parent.document.contains_key("workspace"); + let dependencies = if virtual_root { + parent + .document + .get("workspace") + .and_then(|workspace| workspace.get("dependencies")) + } else { + parent.document.get("dependencies") + }; + resolved.dependencies = + resolve_dependencies(dependencies, directory, &parent.path, Some(parent))?; + resolved.targets = resolve_targets( + parent.document.get("target"), + directory, + &parent.path, + Some(parent), + )?; + resolved.features = resolve_features(parent.document.get("features"), &parent.path)?; + let (patch_document, patch_origin) = parent + .workspace + .as_ref() + .map(|(path, document)| (document, path)) + .unwrap_or((&parent.document, &parent.path)); + resolved.patches = resolve_patches(patch_document.get("patch"), patch_origin)?; + } + let previous_optional = resolved.optional_aliases(); + let notebook = notebook_document(source, &context.directory)?; + let dependencies = resolve_dependencies( + notebook.get("dependencies"), + &context.directory, + &context.directory, + parent.as_ref(), + )?; + // A notebook alias replaces its parent declaration across target sections. + for alias in dependencies.keys() { + for target in resolved + .targets + .iter_mut() + .map(|(_, value)| value) + .filter_map(Value::as_table_mut) + { + if let Some(dependencies) = + target.get_mut("dependencies").and_then(Value::as_table_mut) + { + dependencies.remove(alias); + } + } + } + resolved.dependencies.extend(dependencies); + let targets = resolve_targets( + notebook.get("target"), + &context.directory, + &context.directory, + parent.as_ref(), + )?; + for (target, mut entries) in targets { + let entries = entries + .as_table_mut() + .and_then(|entries| entries.remove("dependencies")) + .and_then(|value| match value { + Value::Table(table) => Some(table), + _ => None, + }) + .unwrap_or_default(); + let existing = resolved + .targets + .entry(target) + .or_insert_with(|| Value::Table(Table::new())); + if let Some(existing) = existing.as_table_mut() { + let dependencies = existing + .entry("dependencies") + .or_insert_with(|| Value::Table(Table::new())); + if let Some(dependencies) = dependencies.as_table_mut() { + dependencies.extend(entries); + } + } + } + resolved.features.extend(resolve_features( + notebook.get("features"), + &context.directory, + )?); + for (registry, patches) in resolve_patches(notebook.get("patch"), &context.directory)? { + let existing = resolved + .patches + .entry(registry) + .or_insert_with(|| Value::Table(Table::new())); + if let (Some(existing), Value::Table(patches)) = (existing.as_table_mut(), patches) { + existing.extend(patches); + } + } + let remaining_optional = resolved.optional_aliases(); + let declared = resolved + .dependencies + .keys() + .chain( + resolved + .targets + .values() + .filter_map(|target| target.get("dependencies").and_then(Value::as_table)) + .flat_map(Table::keys), + ) + .cloned() + .collect::>(); + for alias in previous_optional { + if declared.contains(&alias) && !remaining_optional.contains(&alias) { + resolved.normalize_required_feature(&alias, true); + } + } + if let Some(parent) = &parent { + resolved.prune_omitted_features(&parent.document); + } + resolved.check_sources(&context.directory)?; + Ok(resolved) + } + + pub fn external_dependencies(&self) -> Vec { + let mut dependencies = self.dependencies.iter().collect::>(); + for target in self.targets.values().filter_map(Value::as_table) { + if let Some(target_deps) = target.get("dependencies").and_then(Value::as_table) { + for (alias, value) in target_deps { + dependencies.entry(alias).or_insert(value); + } + } + } + dependencies + .into_iter() + .map(|(name, value)| ExternalDependency { + name: name.clone(), + version: value + .get("version") + .and_then(Value::as_str) + .map(str::to_string), + features: value + .get("features") + .and_then(Value::as_array) + .map(|features| { + features + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(), + path: value.get("path").and_then(Value::as_str).map(PathBuf::from), + }) + .collect() + } + + pub fn has_local_paths(&self) -> bool { + fn contains_path(value: &Value) -> bool { + match value { + Value::Table(table) => { + table.contains_key("path") || table.values().any(contains_path) + } + _ => false, + } + } + [&self.dependencies, &self.targets, &self.patches] + .iter() + .any(|table| table.values().any(contains_path)) + } +} + +fn resolve_dependencies( + value: Option<&Value>, + directory: &Path, + origin: &Path, + parent: Option<&ParentManifest>, +) -> Result
{ + let Some(value) = value else { + return Ok(Table::new()); + }; + let dependencies = value + .as_table() + .ok_or_else(|| manifest_error(origin, "dependencies must be a table"))?; + dependencies.iter().map(|(alias, value)| { + let mut dependency = dependency_table(value, alias, origin)?; + let mut path_directory = directory; + if dependency.remove("workspace").is_some() { + let (workspace_path, workspace) = parent.and_then(|parent| parent.workspace.as_ref()) + .ok_or_else(|| manifest_error(origin, &format!("Dependency '{alias}' inherits without an owning workspace")))?; + let base = workspace.get("workspace").and_then(|workspace| workspace.get("dependencies")).and_then(|dependencies| dependencies.get(alias)) + .ok_or_else(|| manifest_error(origin, &format!("Dependency '{alias}' is missing from workspace.dependencies in '{}'", workspace_path.display())))?; + let mut inherited = dependency_table(base, alias, workspace_path)?; + if inherited.contains_key("workspace") || inherited.get("optional").and_then(Value::as_bool) == Some(true) { + return Err(manifest_error(workspace_path, &format!("workspace.dependencies.{alias} cannot inherit or be optional"))); + } + for key in dependency.keys() { + if !matches!(key.as_str(), "features" | "optional" | "default-features" | "default_features") { + return Err(manifest_error(origin, &format!("Inherited dependency '{alias}' cannot override '{key}'"))); + } + } + if let Some(extra) = dependency.remove("features") { + let features = inherited.entry("features").or_insert_with(|| Value::Array(Vec::new())); + if let Some(features) = features.as_array_mut() { + features.extend(string_array(&extra, origin, alias)?.iter().cloned()); + } + } + let defaults = inherited.get("default-features").or_else(|| inherited.get("default_features")).and_then(Value::as_bool).unwrap_or(true); + if defaults && dependency.get("default-features").or_else(|| dependency.get("default_features")).and_then(Value::as_bool) == Some(false) { + let edition = parent.and_then(|parent| parent.document.get("package")) + .and_then(|package| package.get("edition")); + let edition = edition.and_then(Value::as_str).or_else(|| { + if edition.and_then(|edition| edition.get("workspace")).and_then(Value::as_bool) == Some(true) { + workspace.get("workspace").and_then(|workspace| workspace.get("package")).and_then(|package| package.get("edition")).and_then(Value::as_str) + } else { None } + }); + if edition == Some("2024") { + return Err(manifest_error(origin, &format!("Inherited dependency '{alias}' cannot disable workspace default features in edition 2024. Disable defaults in workspace.dependencies"))); + } + dependency.remove("default-features"); + dependency.remove("default_features"); + } + inherited.extend(dependency); + dependency = inherited; + path_directory = workspace_path.parent().unwrap_or(directory); + } + if !["version", "path", "git"].iter().any(|key| dependency.contains_key(*key)) { + return Err(manifest_error(origin, &format!("Dependency '{alias}' has no version, path, or Git source"))); + } + if dependency.contains_key("git") && dependency.contains_key("path") { + return Err(manifest_error(origin, &format!("Dependency '{alias}' selects both Git and path sources"))); + } + rebase_path(&mut dependency, path_directory, alias, origin)?; + // Feature order does not change Cargo semantics. + if let Some(features) = dependency.get_mut("features").and_then(Value::as_array_mut) { features.sort_by(|a, b| a.as_str().cmp(&b.as_str())); features.dedup(); } + Ok((alias.clone(), Value::Table(dependency))) + }).collect() +} + +fn resolve_targets( + value: Option<&Value>, + directory: &Path, + origin: &Path, + parent: Option<&ParentManifest>, +) -> Result
{ + let Some(value) = value else { + return Ok(Table::new()); + }; + let targets = value + .as_table() + .ok_or_else(|| manifest_error(origin, "target must be a table"))?; + targets + .iter() + .map(|(target, value)| { + if target.starts_with("cfg(") { + syn::parse_str::(target).map_err(|error| { + manifest_error( + origin, + &format!("Target condition '{target}' is malformed: {error}"), + ) + })?; + } + let table = value.as_table().ok_or_else(|| { + manifest_error(origin, &format!("target.{target} must be a table")) + })?; + let dependencies = + resolve_dependencies(table.get("dependencies"), directory, origin, parent)?; + Ok(( + target.clone(), + Value::Table(Table::from_iter([( + "dependencies".into(), + Value::Table(dependencies), + )])), + )) + }) + .collect() +} + +fn resolve_features(value: Option<&Value>, origin: &Path) -> Result
{ + let Some(value) = value else { + return Ok(Table::new()); + }; + let features = value + .as_table() + .ok_or_else(|| manifest_error(origin, "features must be a table"))?; + let mut features = features.clone(); + for (name, value) in &mut features { + string_array(value, origin, &format!("Feature '{name}'"))?; + if let Some(values) = value.as_array_mut() { + values.sort_by(|a, b| a.as_str().cmp(&b.as_str())); + values.dedup(); + } + } + Ok(features) +} + +fn resolve_patches(value: Option<&Value>, origin: &Path) -> Result
{ + let Some(value) = value else { + return Ok(Table::new()); + }; + let patches = value + .as_table() + .ok_or_else(|| manifest_error(origin, "patch must be a table"))?; + patches + .iter() + .map(|(registry, entries)| { + let directory = if origin.is_dir() { + origin + } else { + origin.parent().unwrap_or(Path::new(".")) + }; + let entries = resolve_dependencies(Some(entries), directory, origin, None)?; + Ok((registry.clone(), Value::Table(entries))) + }) + .collect() +} diff --git a/crates/venus-core/src/compile/notebook_manifest/runtime.rs b/crates/venus-core/src/compile/notebook_manifest/runtime.rs new file mode 100644 index 0000000..75d1a99 --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/runtime.rs @@ -0,0 +1,172 @@ +use std::path::Path; + +use semver::{Version, VersionReq}; +use toml::{Table, Value}; + +use crate::compile::CompilerConfig; +use crate::error::Result; + +use super::context::{absolute, manifest_error}; +use super::resolve::ResolvedManifest; + +impl ResolvedManifest { + pub fn with_runtime(&self, config: &CompilerConfig, origin: &Path) -> Result { + let mut resolved = self.clone(); + let mut venus = Table::new(); + if let Some(path) = &config.venus_crate_path { + venus.insert( + "path".into(), + Value::String(absolute(path)?.to_string_lossy().into_owned()), + ); + } else { + venus.insert("version".into(), Value::String("0.1".into())); + } + let runtime = [ + ("venus", venus, "0.1", Vec::<&str>::new()), + ( + "rkyv", + Table::from_iter([("version".into(), Value::String("0.8".into()))]), + "0.8", + vec!["std", "bytecheck"], + ), + ( + "serde_json", + Table::from_iter([("version".into(), Value::String("1.0".into()))]), + "1.0", + Vec::new(), + ), + ]; + for (alias, mut required, version, required_features) in runtime { + let mut features = required_features + .into_iter() + .map(|feature| Value::String(feature.into())) + .collect::>(); + if let Some(user) = self.dependencies.get(alias) { + check_reserved(alias, user, &required, version, origin)?; + append_features(user, &mut features); + retain_version(user, &mut required); + } + let mut was_optional = self + .dependencies + .get(alias) + .and_then(|user| user.get("optional")) + .and_then(Value::as_bool) + == Some(true); + for target in resolved + .targets + .iter_mut() + .map(|(_, value)| value) + .filter_map(Value::as_table_mut) + { + if let Some(user) = target + .get_mut("dependencies") + .and_then(Value::as_table_mut) + .and_then(|dependencies| dependencies.get_mut(alias)) + { + check_reserved(alias, user, &required, version, origin)?; + was_optional |= user.get("optional").and_then(Value::as_bool) == Some(true); + let mut target_required = required.clone(); + let mut target_features = Vec::new(); + append_features(user, &mut target_features); + retain_version(user, &mut target_required); + if !target_features.is_empty() { + target_required.insert("features".into(), Value::Array(target_features)); + } + *user = Value::Table(target_required); + } + } + resolved.normalize_required_feature(alias, was_optional); + features.sort_by(|a, b| a.as_str().cmp(&b.as_str())); + features.dedup(); + if !features.is_empty() { + required.insert("features".into(), Value::Array(features)); + } + resolved + .dependencies + .insert(alias.into(), Value::Table(required)); + } + Ok(resolved) + } +} + +fn append_features(user: &Value, features: &mut Vec) { + if let Some(extra) = user.get("features").and_then(Value::as_array) { + features.extend(extra.iter().cloned()); + } +} + +fn check_reserved( + alias: &str, + user: &Value, + required: &Table, + version: &str, + origin: &Path, +) -> Result<()> { + let incompatible = || { + manifest_error( + origin, + &format!( + "Dependency alias '{alias}' conflicts with the required Venus runtime dependency" + ), + ) + }; + if user + .get("package") + .and_then(Value::as_str) + .is_some_and(|package| package != alias) + { + return Err(incompatible()); + } + for key in ["path", "git", "registry", "registry-index"] { + if user + .get(key) + .is_some_and(|value| key != "path" || required.get(key) != Some(value)) + { + return Err(incompatible()); + } + } + if let Some(requirement) = user.get("version").and_then(Value::as_str) { + let requirement = VersionReq::parse(requirement).map_err(|_| incompatible())?; + let runtime = VersionReq::parse(version).map_err(|_| incompatible())?; + let base = Version::parse(&format!("{version}.0")).map_err(|_| incompatible())?; + let base_minor = base.minor; + let mut candidates = vec![base]; + for comparator in &requirement.comparators { + let candidate = Version::new( + comparator.major, + comparator.minor.unwrap_or(base_minor), + comparator.patch.unwrap_or(0), + ); + candidates.push(Version::new( + candidate.major, + candidate.minor, + candidate.patch.saturating_add(1), + )); + candidates.push(Version::new( + candidate.major, + candidate.minor.saturating_add(1), + 0, + )); + candidates.push(candidate); + } + if !candidates + .iter() + .any(|candidate| runtime.matches(candidate) && requirement.matches(candidate)) + { + return Err(incompatible()); + } + } + Ok(()) +} + +fn retain_version(user: &Value, required: &mut Table) { + if let (Some(user), Some(runtime)) = ( + user.get("version").and_then(Value::as_str), + required.get("version").and_then(Value::as_str), + ) { + required.insert( + "version".into(), + Value::String(format!("{runtime}, {user}")), + ); + } +} diff --git a/crates/venus-core/src/compile/notebook_manifest/spec.rs b/crates/venus-core/src/compile/notebook_manifest/spec.rs new file mode 100644 index 0000000..99bae51 --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/spec.rs @@ -0,0 +1,142 @@ +use std::path::Path; + +use toml::{Table, Value}; + +use crate::error::Result; + +use super::context::manifest_error; + +pub(in crate::compile) fn notebook_document(source: &str, origin: &Path) -> Result
{ + let mut content = String::new(); + let mut in_block = false; + for line in source.lines() { + let Some(line) = line.trim().strip_prefix("//!") else { + continue; + }; + let line = line.trim_start(); + if line.trim() == "```cargo" { + if in_block { + return Err(manifest_error(origin, "Nested cargo fence")); + } + in_block = true; + } else if line.trim() == "```" && in_block { + in_block = false; + content.push('\n'); + } else if in_block { + content.push_str(line); + content.push('\n'); + } + } + if in_block { + return Err(manifest_error(origin, "Unclosed cargo fence")); + } + toml::from_str(&content).map_err(|error| { + manifest_error( + origin, + &format!("Cannot parse notebook cargo block: {error}"), + ) + }) +} + +pub(super) fn dependency_table(value: &Value, alias: &str, origin: &Path) -> Result
{ + let table = match value { + Value::String(version) => { + Table::from_iter([("version".into(), Value::String(version.clone()))]) + } + Value::Table(table) => table.clone(), + _ => { + return Err(manifest_error( + origin, + &format!("Dependency '{alias}' must be a version string or table"), + )); + } + }; + for key in [ + "version", + "path", + "package", + "git", + "branch", + "tag", + "rev", + "registry", + "registry-index", + ] { + if table.get(key).is_some_and(|value| !value.is_str()) { + return Err(manifest_error( + origin, + &format!("Dependency '{alias}'.{key} must be a string"), + )); + } + } + for key in [ + "workspace", + "optional", + "default-features", + "default_features", + ] { + if table.get(key).is_some_and(|value| !value.is_bool()) { + return Err(manifest_error( + origin, + &format!("Dependency '{alias}'.{key} must be a boolean"), + )); + } + } + if let Some(features) = table.get("features") { + string_array(features, origin, &format!("Dependency '{alias}'.features"))?; + } + if let Some(version) = table.get("version").and_then(Value::as_str) { + semver::VersionReq::parse(version).map_err(|error| { + manifest_error( + origin, + &format!("Dependency '{alias}' has invalid version '{version}': {error}"), + ) + })?; + } + if table + .get("workspace") + .is_some_and(|value| value.as_bool() != Some(true)) + { + return Err(manifest_error( + origin, + &format!("Dependency '{alias}'.workspace must be true"), + )); + } + Ok(table) +} + +pub(super) fn string_array<'a>( + value: &'a Value, + origin: &Path, + label: &str, +) -> Result<&'a Vec> { + value + .as_array() + .filter(|values| values.iter().all(Value::is_str)) + .ok_or_else(|| manifest_error(origin, &format!("{label} must be an array of strings"))) +} + +pub(super) fn rebase_path( + table: &mut Table, + directory: &Path, + alias: &str, + origin: &Path, +) -> Result<()> { + if let Some(path) = table.get("path").and_then(Value::as_str) { + let path = directory.join(path); + let path = path.canonicalize().map_err(|error| { + manifest_error( + origin, + &format!( + "Cannot resolve dependency '{alias}' path '{}': {error}", + path.display() + ), + ) + })?; + table.insert( + "path".into(), + Value::String(path.to_string_lossy().into_owned()), + ); + } + Ok(()) +} diff --git a/crates/venus-core/src/compile/notebook_manifest/tests/features.rs b/crates/venus-core/src/compile/notebook_manifest/tests/features.rs new file mode 100644 index 0000000..e74916c --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/tests/features.rs @@ -0,0 +1,194 @@ +use super::fixtures::*; +use crate::compile::CompilerConfig; +use std::fs; +use toml::Value; + +#[test] +fn optional_exports_follow_activation_and_forwarding_features() { + let dir = tempfile::tempdir().unwrap(); + let resolved = resolve( + &dir.path().join("notebook.rs"), + r#" +//! ```cargo +//! [dependencies] +//! hidden = { version = "1", optional = true } +//! implicit = { version = "1", optional = true } +//! forwarded = { version = "1", optional = true } +//! [features] +//! default = ["outer"] +//! outer = ["activate"] +//! activate = ["dep:hidden"] +//! weak = ["hidden?/enabled"] +//! forward = ["forwarded/enabled"] +//! ``` +"#, + ); + let exports = resolved.reexports(); + let gate = exports + .lines() + .zip(exports.lines().skip(1)) + .find(|(_, line)| *line == "pub use hidden;") + .unwrap() + .0; + assert!(gate.contains("feature = \"activate\"")); + assert!(gate.contains("feature = \"outer\"")); + assert!(gate.contains("feature = \"default\"")); + assert!(!gate.contains("feature = \"hidden\"")); + assert!(!gate.contains("feature = \"weak\"")); + assert!(exports.contains("feature = \"implicit\"")); + assert!(exports.contains("feature = \"forward\"")); +} + +#[test] +fn reserved_dependencies_keep_compatible_features_and_reject_conflicting_packages() { + let dir = tempfile::tempdir().unwrap(); + let config = CompilerConfig { + venus_crate_path: None, + ..CompilerConfig::default() + }; + let resolved = resolve( + &dir.path().join("notebook.rs"), + r#" +//! ```cargo +//! [dependencies] +//! rkyv = { version = "0.8.12", features = ["alloc"], default-features = false, optional = true } +//! serde_json = { version = "1.0.100", features = ["raw_value"], optional = true } +//! [features] +//! default = ["json"] +//! json = ["dep:serde_json", "serde_json?/raw_value"] +//! archive = ["rkyv"] +//! ``` +"#, + ) + .with_runtime(&config, dir.path()) + .unwrap(); + assert_eq!( + resolved.dependencies["rkyv"]["features"] + .as_array() + .unwrap() + .len(), + 3 + ); + assert!(resolved.dependencies["rkyv"].get("optional").is_none()); + assert!(resolved.features.contains_key("rkyv")); + assert_eq!( + resolved.features["json"].as_array().unwrap(), + &[Value::String("serde_json/raw_value".into())] + ); + let conflicting = resolve( + &dir.path().join("notebook.rs"), + "//! ```cargo\n//! [dependencies]\n//! rkyv = { package = \"unrelated\", version = \"0.8\" }\n//! ```", + ); + assert!(conflicting.with_runtime(&config, dir.path()).is_err()); + let conflicting = resolve( + &dir.path().join("notebook.rs"), + "//! ```cargo\n//! [dependencies]\n//! rkyv = \"0.9\"\n//! ```", + ); + assert!(conflicting.with_runtime(&config, dir.path()).is_err()); +} + +#[test] +fn edition_2021_keeps_workspace_defaults_when_member_disables_them() { + let (dir, notebook) = workspace(); + let root = dir.path().join("Cargo.toml"); + fs::write( + &root, + fs::read_to_string(&root) + .unwrap() + .replace("default-features = false, ", ""), + ) + .unwrap(); + let member = dir.path().join("member/Cargo.toml"); + fs::write( + &member, + fs::read_to_string(&member).unwrap().replace( + "workspace = true", + "workspace = true\ndefault-features = false", + ), + ) + .unwrap(); + let resolved = resolve(¬ebook, ""); + assert!( + resolved.dependencies["internal-alias"] + .get("default-features") + .is_none() + ); +} + +#[test] +fn omitted_build_dependency_features_do_not_break_runtime_manifest() { + let dir = tempfile::tempdir().unwrap(); + package( + dir.path(), + "app", + "app", + r#" +[build-dependencies] +cc = { version = "1", optional = true } +[features] +native = ["dep:cc", "cc/parallel", "cc?/parallel"] +"#, + ); + let resolved = resolve(&dir.path().join("app/notebook.rs"), ""); + assert!(resolved.features["native"].as_array().unwrap().is_empty()); + assert!(!resolved.dependencies.contains_key("cc")); +} + +#[test] +fn mandatory_notebook_override_retains_valid_parent_activation_graph() { + let dir = tempfile::tempdir().unwrap(); + package( + dir.path(), + "app", + "app", + r#" +[dependencies] +helper = { version = "1", optional = true } +[features] +activate = ["dep:helper"] +weak = ["helper?/enabled"] +"#, + ); + let resolved = resolve( + &dir.path().join("app/notebook.rs"), + "//! ```cargo\n//! [dependencies]\n//! helper = \"1\"\n//! ```", + ); + assert!(resolved.features["activate"].as_array().unwrap().is_empty()); + assert_eq!( + resolved.features["weak"].as_array().unwrap(), + &[Value::String("helper/enabled".into())] + ); + assert!(resolved.reexports().contains("pub use helper;")); + assert!(!resolved.reexports().contains("#[cfg")); +} + +#[test] +fn mandatory_target_override_retains_valid_parent_activation_graph() { + let dir = tempfile::tempdir().unwrap(); + package( + dir.path(), + "app", + "app", + r#" +[target.'cfg(unix)'.dependencies] +helper = { version = "1", optional = true } +[features] +activate = ["dep:helper"] +weak = ["helper?/enabled"] +"#, + ); + let resolved = resolve( + &dir.path().join("app/notebook.rs"), + r#" +//! ```cargo +//! [target.'cfg(unix)'.dependencies] +//! helper = "1" +//! ``` +"#, + ); + assert!(resolved.features["activate"].as_array().unwrap().is_empty()); + assert_eq!( + resolved.features["weak"].as_array().unwrap(), + &[Value::String("helper/enabled".into())] + ); +} diff --git a/crates/venus-core/src/compile/notebook_manifest/tests/fixtures.rs b/crates/venus-core/src/compile/notebook_manifest/tests/fixtures.rs new file mode 100644 index 0000000..849589d --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/tests/fixtures.rs @@ -0,0 +1,79 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use super::super::{NotebookContext, ResolvedManifest}; +use tempfile::TempDir; + +pub(super) fn write(root: &Path, path: &str, text: &str) { + let path = root.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); +} + +pub(super) fn package(root: &Path, path: &str, name: &str, extra: &str) { + write( + root, + &format!("{path}/Cargo.toml"), + &format!("[package]\nname = {name:?}\nversion = \"0.1.0\"\n{extra}"), + ); + write( + root, + &format!("{path}/src/lib.rs"), + "pub fn value() -> i32 { 42 }", + ); +} + +pub(super) fn workspace() -> (TempDir, PathBuf) { + let dir = tempfile::tempdir().unwrap(); + write( + dir.path(), + "Cargo.toml", + r#" +[workspace] +members = ["member", "internal", "direct", "patched"] +[workspace.dependencies] +internal-alias = { package = "internal", path = "internal", default-features = false, features = ["base"] } +unused = "999.0" +[patch.crates-io] +patched = { path = "patched" } +"#, + ); + package( + dir.path(), + "internal", + "internal", + "\n[features]\nbase = []\nextra = []\n", + ); + package(dir.path(), "direct", "direct", ""); + package(dir.path(), "patched", "patched", ""); + package( + dir.path(), + "member", + "member", + r#" +[dependencies.internal-alias] +workspace = true +features = [ + "extra", + "base", +] +[dependencies.direct] +path = "../direct" +"#, + ); + let notebook = dir.path().join("member/notebooks/report.rs"); + write(dir.path(), "member/notebooks/report.rs", ""); + (dir, notebook) +} + +pub(super) fn resolve(path: &Path, source: &str) -> ResolvedManifest { + ResolvedManifest::resolve(&NotebookContext::for_notebook(path).unwrap(), source).unwrap() +} + +#[cfg(unix)] +pub(super) fn symlinked_workspace() -> (TempDir, PathBuf) { + let (dir, notebook) = workspace(); + let linked = dir.path().join("linked.rs"); + std::os::unix::fs::symlink(notebook, &linked).unwrap(); + (dir, linked) +} diff --git a/crates/venus-core/src/compile/notebook_manifest/tests/mod.rs b/crates/venus-core/src/compile/notebook_manifest/tests/mod.rs new file mode 100644 index 0000000..aa1480e --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/tests/mod.rs @@ -0,0 +1,3 @@ +mod features; +mod fixtures; +mod resolution; diff --git a/crates/venus-core/src/compile/notebook_manifest/tests/resolution.rs b/crates/venus-core/src/compile/notebook_manifest/tests/resolution.rs new file mode 100644 index 0000000..071b9bb --- /dev/null +++ b/crates/venus-core/src/compile/notebook_manifest/tests/resolution.rs @@ -0,0 +1,378 @@ +use super::super::{NotebookContext, ResolvedManifest}; +use super::fixtures::*; +use crate::compile::{CompilerConfig, ManifestConfig}; +use std::fs; +use std::path::Path; +use toml::{Table, Value}; + +#[test] +fn member_inherits_declared_aliases_and_features_from_workspace() { + let (dir, notebook) = workspace(); + let resolved = resolve(¬ebook, ""); + let alias = &resolved.dependencies["internal-alias"]; + assert_eq!(alias["package"].as_str(), Some("internal")); + assert_eq!(alias["default-features"].as_bool(), Some(false)); + assert_eq!( + alias["features"].as_array().unwrap(), + &[Value::String("base".into()), Value::String("extra".into())] + ); + assert_eq!( + Path::new(alias["path"].as_str().unwrap()), + dir.path().join("internal").canonicalize().unwrap() + ); + assert_eq!( + Path::new(resolved.dependencies["direct"]["path"].as_str().unwrap()), + dir.path().join("direct").canonicalize().unwrap() + ); + assert!(!resolved.dependencies.contains_key("unused")); + assert_eq!( + Path::new( + resolved.patches["crates-io"]["patched"]["path"] + .as_str() + .unwrap() + ), + dir.path().join("patched").canonicalize().unwrap() + ); + assert!(resolved.reexports().contains("pub use internal_alias;")); + let manifest: Table = toml::from_str( + &resolved + .render(&ManifestConfig { + standalone_workspace: true, + ..ManifestConfig::default() + }) + .unwrap(), + ) + .unwrap(); + assert!(manifest.contains_key("workspace")); +} + +#[test] +fn explicit_workspace_location_resolves_inherited_paths() { + let (dir, notebook) = workspace(); + let member = dir.path().join("member/Cargo.toml"); + let source = fs::read_to_string(&member).unwrap().replace( + "version = \"0.1.0\"", + "version = \"0.1.0\"\nworkspace = \"..\"", + ); + fs::write(member, source).unwrap(); + assert_eq!( + resolve(¬ebook, "").dependencies["internal-alias"]["package"].as_str(), + Some("internal") + ); +} + +#[test] +fn notebook_declarations_replace_parent_aliases_and_rebase_paths() { + let (dir, notebook) = workspace(); + let resolved = resolve( + ¬ebook, + r#" +//! ```cargo +//! [dependencies.internal-alias] +//! package = "direct" +//! path = "../../direct" +//! features = [] +//! ``` +"#, + ); + assert_eq!(resolved.dependencies.len(), 2); + assert_eq!( + resolved.dependencies["internal-alias"]["package"].as_str(), + Some("direct") + ); + assert_eq!( + Path::new( + resolved.dependencies["internal-alias"]["path"] + .as_str() + .unwrap() + ), + dir.path().join("direct").canonicalize().unwrap() + ); +} + +#[test] +fn virtual_root_notebooks_retain_catalog_dependencies() { + let (dir, _) = workspace(); + write(dir.path(), "root.rs", ""); + let resolved = resolve(&dir.path().join("root.rs"), ""); + assert!(resolved.dependencies.contains_key("unused")); + assert!(resolved.dependencies.contains_key("internal-alias")); +} + +#[test] +fn excluded_package_does_not_inherit_ancestor_patches() { + let dir = tempfile::tempdir().unwrap(); + write( + dir.path(), + "Cargo.toml", + "[workspace]\nmembers = []\nexclude = [\"excluded\"]\n[patch.crates-io]\nremote = { git = \"https://example.invalid/remote\" }\n", + ); + package( + dir.path(), + "excluded", + "excluded", + "[dependencies]\nserde = \"1\"\n", + ); + write(dir.path(), "excluded/notebook.rs", ""); + let resolved = resolve(&dir.path().join("excluded/notebook.rs"), ""); + assert!(resolved.patches.is_empty()); + assert!(resolved.dependencies.contains_key("serde")); +} + +#[test] +fn target_dependencies_keep_cfg_and_named_triple_gates() { + let dir = tempfile::tempdir().unwrap(); + package( + dir.path(), + "app", + "app", + r#" +[target.'cfg(unix)'.dependencies] +native = "1" +[target.x86_64-pc-windows-msvc.dependencies] +windows-api = "2" +"#, + ); + let resolved = resolve(&dir.path().join("app/notebook.rs"), ""); + assert!(resolved.dependencies.is_empty()); + assert_eq!(resolved.targets.len(), 2); + let exports = resolved.reexports(); + assert!(exports.contains("#[cfg(any(unix))]\npub use native;")); + assert!(exports.contains("pub use windows_api;")); + let script = resolved.build_script(); + assert!(script.contains("x86_64-pc-windows-msvc")); + assert!(script.contains("cargo:rustc-check-cfg=cfg(venus_target_")); +} + +#[test] +fn git_registry_selectors_and_multiline_tables_survive_resolution() { + let dir = tempfile::tempdir().unwrap(); + let resolved = resolve( + &dir.path().join("notebook.rs"), + r#" +//! ```cargo +//! [dependencies.renamed] +//! package = "actual" +//! git = "https://example.invalid/actual" +//! rev = "123abc" +//! default-features = false +//! features = [ +//! "a", +//! "b", +//! ] +//! [dependencies.registry-alias] +//! package = "another" +//! version = "1" +//! registry = "private" +//! ``` +"#, + ); + assert_eq!( + resolved.dependencies["renamed"]["rev"].as_str(), + Some("123abc") + ); + assert_eq!( + resolved.dependencies["registry-alias"]["registry"].as_str(), + Some("private") + ); + assert_eq!(resolved.external_dependencies()[0].features.len(), 0); + assert_eq!( + resolved.dependencies["renamed"]["features"] + .as_array() + .unwrap() + .len(), + 2 + ); +} + +#[test] +fn malformed_manifests_and_unresolved_inheritance_return_contextual_errors() { + let dir = tempfile::tempdir().unwrap(); + let notebook = dir.path().join("notebook.rs"); + write(dir.path(), "Cargo.toml", "[dependencies\n"); + let context = NotebookContext::for_notebook(¬ebook).unwrap(); + assert!( + ResolvedManifest::resolve(&context, "") + .unwrap_err() + .to_string() + .contains("Cargo.toml") + ); + fs::remove_file(dir.path().join("Cargo.toml")).unwrap(); + let context = NotebookContext::for_notebook(¬ebook).unwrap(); + let error = ResolvedManifest::resolve( + &context, + "//! ```cargo\n//! [dependencies]\n//! missing.workspace = true\n//! ```", + ) + .unwrap_err() + .to_string(); + assert!(error.contains("missing")); + assert!(error.contains("owning workspace")); + assert!( + ResolvedManifest::resolve( + &context, + "//! ```cargo\n//! [dependencies]\n//! alias = { features = [1] }\n//! ```" + ) + .is_err() + ); +} + +#[test] +fn canonical_manifest_changes_when_parent_features_change() { + let (dir, notebook) = workspace(); + let before = resolve(¬ebook, "").canonical(); + let path = dir.path().join("member/Cargo.toml"); + let source = fs::read_to_string(&path) + .unwrap() + .replace(" \"extra\",\n", ""); + fs::write(path, source).unwrap(); + let after = resolve(¬ebook, "").canonical(); + assert_ne!(before, after); +} + +#[test] +fn target_override_rejects_conflicting_generic_dependency_source() { + let (dir, notebook) = workspace(); + let error = ResolvedManifest::resolve( + &NotebookContext::for_notebook(¬ebook).unwrap(), + r#" +//! ```cargo +//! [target.'cfg(unix)'.dependencies] +//! internal-alias = { package = "direct", path = "../../direct" } +//! ``` +"#, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("internal-alias")); + assert!(error.contains("different packages or sources")); + assert!(dir.path().exists()); +} + +#[test] +fn rendered_manifests_preserve_native_path_strings() { + for path in [ + r"C:\notebooks\local", + r"\\?\C:\notebooks\local", + r"\\server\share\local", + r"\\?\UNC\server\share\local", + r"local\component", + r#"/notebooks/quoted "component"/local"#, + ] { + let dependency = Value::Table(Table::from_iter([( + "path".into(), + Value::String(path.into()), + )])); + let resolved = ResolvedManifest { + dependencies: Table::from_iter([("local".into(), dependency.clone())]), + targets: Table::from_iter([( + "cfg(windows)".into(), + Value::Table(Table::from_iter([( + "dependencies".into(), + Value::Table(Table::from_iter([("local".into(), dependency.clone())])), + )])), + )]), + patches: Table::from_iter([( + "crates-io".into(), + Value::Table(Table::from_iter([("local".into(), dependency)])), + )]), + ..ResolvedManifest::default() + }; + let manifest: Table = + toml::from_str(&resolved.render(&ManifestConfig::default()).unwrap()).unwrap(); + for dependency in [ + &manifest["dependencies"]["local"], + &manifest["target"]["cfg(windows)"]["dependencies"]["local"], + &manifest["patch"]["crates-io"]["local"], + ] { + assert_eq!(dependency["path"].as_str(), Some(path)); + } + } +} + +#[test] +fn generated_paths_match_canonical_sources_and_runtime_aliases() { + let dir = tempfile::tempdir().unwrap(); + let component = if cfg!(unix) { + "native \\ \"quoted\" paths" + } else { + "native ' paths" + }; + let root = dir.path().join(component); + package(&root, "local", "local", ""); + package(&root, "patched", "patched", ""); + package(&root, "runtime", "venus", ""); + package( + &root, + "app", + "app", + r#" +[dependencies] +local = { path = "../local/./" } +venus = { path = "../runtime/./" } +[target.'cfg(windows)'.dependencies] +local = { path = "../local" } +venus = { path = "../runtime" } +[patch.crates-io] +patched = { path = "../patched" } +"#, + ); + write(&root, "app/notebook.rs", ""); + let notebook = root.join("app/notebook.rs"); + let config = CompilerConfig { + venus_crate_path: Some(root.join("runtime")), + ..CompilerConfig::default() + }; + let resolved = resolve(¬ebook, "") + .with_runtime(&config, ¬ebook) + .unwrap(); + let manifest: Table = + toml::from_str(&resolved.render(&ManifestConfig::default()).unwrap()).unwrap(); + for (dependency, directory) in [ + (&manifest["dependencies"]["local"], "local"), + ( + &manifest["target"]["cfg(windows)"]["dependencies"]["local"], + "local", + ), + (&manifest["dependencies"]["venus"], "runtime"), + ( + &manifest["target"]["cfg(windows)"]["dependencies"]["venus"], + "runtime", + ), + (&manifest["patch"]["crates-io"]["patched"], "patched"), + ] { + let expected = root.join(directory).canonicalize().unwrap(); + assert_eq!( + dependency["path"].as_str(), + Some(expected.to_string_lossy().as_ref()) + ); + assert_eq!(Path::new(dependency["path"].as_str().unwrap()), expected); + } +} + +#[cfg(unix)] +#[test] +fn symlinked_notebook_resolves_declarations_from_its_canonical_directory() { + let (dir, notebook) = symlinked_workspace(); + let context = NotebookContext::for_notebook(¬ebook).unwrap(); + assert_eq!( + context.directory, + dir.path().join("member/notebooks").canonicalize().unwrap() + ); + assert_eq!( + context.manifest, + Some(dir.path().join("member/Cargo.toml").canonicalize().unwrap()) + ); + let resolved = resolve( + ¬ebook, + "//! ```cargo\n//! [dependencies]\n//! internal-alias = { package = \"direct\", path = \"../../direct\" }\n//! ```", + ); + let expected = dir.path().join("direct").canonicalize().unwrap(); + assert_eq!( + Path::new( + resolved.dependencies["internal-alias"]["path"] + .as_str() + .unwrap() + ), + expected + ); +} diff --git a/crates/venus-core/src/compile/production.rs b/crates/venus-core/src/compile/production.rs index 7bde33b..32fbf2d 100644 --- a/crates/venus-core/src/compile/production.rs +++ b/crates/venus-core/src/compile/production.rs @@ -11,8 +11,9 @@ use crate::error::{Error, Result}; use crate::graph::{CellInfo, CellParser, GraphEngine}; use super::CompilerConfig; -use super::cargo_generator::{ManifestConfig, ReleaseProfile, generate_cargo_toml}; +use super::cargo_generator::{ManifestConfig, ReleaseProfile}; use super::dependency_parser::DependencyParser; +use super::notebook_manifest::{NotebookContext, ResolvedManifest}; use super::source_processor::NotebookSourceProcessor; /// Builder for standalone production binaries. @@ -33,6 +34,8 @@ pub struct ProductionBuilder { /// Dependency parser parser: DependencyParser, + resolved: ResolvedManifest, + /// Original notebook source source: String, @@ -48,6 +51,7 @@ impl ProductionBuilder { cells: Vec::new(), graph: GraphEngine::new(), parser: DependencyParser::new(), + resolved: ResolvedManifest::default(), source: String::new(), notebook_path: PathBuf::new(), } @@ -69,7 +73,7 @@ impl ProductionBuilder { // Parse cells let mut parser = CellParser::new(); - let parse_result = parser.parse_file(path)?; + let parse_result = parser.parse_str(&self.source, path)?; self.cells = parse_result.code_cells; // Validate unique cell names @@ -84,7 +88,11 @@ impl ProductionBuilder { self.graph.resolve_dependencies()?; // Parse external dependencies - self.parser.parse(&self.source); + let context = NotebookContext::for_notebook(path)?; + let resolved = ResolvedManifest::resolve(&context, &self.source)?; + self.parser + .set_dependencies(resolved.external_dependencies()); + self.resolved = resolved.with_runtime(&self.config, &context.directory)?; Ok(()) } @@ -129,6 +137,7 @@ impl ProductionBuilder { // Generate Cargo.toml let cargo_toml = self.generate_cargo_toml()?; fs::write(build_dir.join("Cargo.toml"), cargo_toml)?; + fs::write(build_dir.join("build.rs"), self.resolved.build_script())?; // Generate main.rs let main_rs = self.generate_main_rs()?; @@ -207,36 +216,6 @@ impl ProductionBuilder { .unwrap_or("notebook") .replace('-', "_"); - // Get the notebook directory for resolving relative paths - let notebook_dir = self - .notebook_path - .parent() - .ok_or_else(|| Error::Compilation { - cell_id: None, - message: format!( - "Could not determine parent directory for notebook: {}", - self.notebook_path.display() - ), - })?; - - // Validate all path dependencies can be resolved - for dep in self.parser.dependencies() { - if let Some(path) = &dep.path - && path.is_relative() - { - let full_path = notebook_dir.join(path); - full_path.canonicalize().map_err(|e| Error::Compilation { - cell_id: None, - message: format!( - "Failed to resolve path dependency '{}' ({}): {}", - dep.name, - full_path.display(), - e - ), - })?; - } - } - let config = ManifestConfig { name: &name, version: "0.1.0", @@ -246,12 +225,24 @@ impl ProductionBuilder { standalone_workspace: true, }; - Ok(generate_cargo_toml( - &config, - self.parser.dependencies(), - true, // Always include serde for consistency - Some(notebook_dir), - )) + let mut resolved = self.resolved.clone(); + // Preserve production defaults only when the notebook has no declaration. + resolved.dependencies.entry("serde").or_insert_with(|| { + toml::Value::Table(toml::Table::from_iter([ + ("version".into(), toml::Value::String("1.0".into())), + ( + "features".into(), + toml::Value::Array(vec![toml::Value::String("derive".into())]), + ), + ])) + }); + resolved.dependencies.entry("bincode").or_insert_with(|| { + toml::Value::Table(toml::Table::from_iter([( + "version".into(), + toml::Value::String("1.3".into()), + )])) + }); + resolved.render(&config) } /// Generate main.rs with all cells and execution logic. diff --git a/crates/venus-core/src/compile/universe.rs b/crates/venus-core/src/compile/universe.rs deleted file mode 100644 index 13a0361..0000000 --- a/crates/venus-core/src/compile/universe.rs +++ /dev/null @@ -1,649 +0,0 @@ -//! Universe builder for Venus notebooks. -//! -//! The "Universe" is a shared library containing all external dependencies -//! that cells can link against. It's compiled once with LLVM and cached. - -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -use crate::error::{Error, Result}; -use crate::graph::DefinitionCell; - -use super::definition_processor::process_definitions; -use super::dependency_parser::{DependencyParser, ExternalDependency}; -use super::toolchain::ToolchainManager; -use super::types::{CompilerConfig, dylib_extension, dylib_prefix}; - -/// Builder for the Universe shared library. -pub struct UniverseBuilder { - /// Compiler configuration - config: CompilerConfig, - - /// Toolchain manager (reserved for future LLVM compilation options). - /// - /// Currently unused but intentionally preserved to avoid breaking API changes - /// when LLVM backend support is added. The universe is currently built using - /// Cargo/rustc, but future versions may support direct LLVM compilation for - /// optimization control and faster compilation times. - /// - /// Keeping this field now prevents: - /// - Breaking API changes to `UniverseBuilder::new()` - /// - Re-threading toolchain manager through the codebase later - /// - Inconsistency with cell compilation (which does use toolchain) - #[allow(dead_code)] - toolchain: ToolchainManager, - - /// Dependency parser (handles parsing and hashing) - parser: DependencyParser, - - /// User-defined type definitions extracted from notebook (promoted to `pub`, - /// with rkyv derives applied). - type_definitions: String, - - /// Notebook `use` statements from definition cells, re-exported as `pub use` - /// so imported names are visible to every compiled cell. - imports: String, - - /// Path to workspace Cargo.toml (for copying dependencies) - workspace_cargo_toml: Option, -} - -impl UniverseBuilder { - /// Create a new universe builder. - pub fn new( - config: CompilerConfig, - toolchain: ToolchainManager, - workspace_cargo_toml: Option, - ) -> Self { - Self { - config, - toolchain, - parser: DependencyParser::new(), - type_definitions: String::new(), - imports: String::new(), - workspace_cargo_toml, - } - } - - /// Parse dependencies and process definition cells from notebook source. - /// - /// Delegates to [`DependencyParser`] for dependency parsing and to - /// [`process_definitions`](super::definition_processor::process_definitions) - /// for turning definition cells into re-exportable imports (`pub use`) and - /// public type definitions (with rkyv derives applied). - pub fn parse_dependencies( - &mut self, - source: &str, - definition_cells: &[DefinitionCell], - ) -> Result<()> { - self.parser.parse(source); - - let contents: Vec = definition_cells - .iter() - .map(|cell| cell.content.clone()) - .collect(); - let processed = process_definitions(&contents); - self.imports = processed.imports; - self.type_definitions = processed.type_definitions; - - Ok(()) - } - - /// Get the dependencies hash (includes imports and type definitions). - pub fn deps_hash(&self) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - self.parser.calculate_hash().hash(&mut hasher); - self.imports.hash(&mut hasher); - self.type_definitions.hash(&mut hasher); - hasher.finish() - } - - /// Get the parsed external dependencies. - pub fn dependencies(&self) -> &[ExternalDependency] { - self.parser.dependencies() - } - - /// Check if the cached universe is valid. - pub fn is_cache_valid(&self) -> bool { - let cache_file = self.cache_hash_file(); - if !cache_file.exists() { - return false; - } - - // Read cached hash - if let Ok(cached_hash) = fs::read_to_string(&cache_file) - && let Ok(hash) = cached_hash.trim().parse::() - { - return hash == self.deps_hash(); - } - - false - } - - /// Get the path to the compiled universe library. - pub fn universe_path(&self) -> PathBuf { - let build_dir = self.config.universe_build_dir(); - let filename = format!("{}venus_universe.{}", dylib_prefix(), dylib_extension()); - build_dir.join(filename) - } - - /// Build the universe library. - pub fn build(&self) -> Result { - // Check cache first - if self.is_cache_valid() && self.universe_path().exists() { - tracing::info!("Using cached universe library"); - return Ok(self.universe_path()); - } - - tracing::info!( - "Building universe library with {} dependencies", - self.dependencies().len() - ); - - let build_dir = self.config.universe_build_dir(); - fs::create_dir_all(&build_dir)?; - - // Generate Cargo.toml - let cargo_toml = self.generate_cargo_toml(); - let cargo_path = build_dir.join("Cargo.toml"); - fs::write(&cargo_path, cargo_toml)?; - - // Generate lib.rs - let lib_rs = self.generate_lib_rs(); - let src_dir = build_dir.join("src"); - fs::create_dir_all(&src_dir)?; - fs::write(src_dir.join("lib.rs"), lib_rs)?; - - // Generate stub notebook.rs (required by lib.rs `pub mod notebook;`) - // The server overwrites this with real cell content for LSP analysis. - // For CLI builds, this stub satisfies the module declaration. - let notebook_rs = "//! Notebook cells module.\n\ - //! This file is populated by the server for LSP analysis.\n\ - //! For CLI builds, this is a stub to satisfy the module declaration.\n"; - fs::write(src_dir.join("notebook.rs"), notebook_rs)?; - - // Build with cargo - let output = Command::new("cargo") - .current_dir(&build_dir) - .args(["build", "--release", "--lib"]) - .output() - .map_err(|e| Error::Compilation { - cell_id: None, - message: format!("Failed to run cargo: {}", e), - })?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(Error::Compilation { - cell_id: None, - message: format!("Universe build failed:\n{}", stderr), - }); - } - - // Copy the built library - let target_lib = build_dir.join("target").join("release").join(format!( - "{}venus_universe.{}", - dylib_prefix(), - dylib_extension() - )); - - let dest = self.universe_path(); - fs::copy(&target_lib, &dest)?; - - // Save cache hash - self.save_cache_hash()?; - - tracing::info!("Universe library built: {}", dest.display()); - Ok(dest) - } - - /// Copy dependencies from workspace Cargo.toml (if it exists). - /// Returns the dependencies section as a string. - fn copy_parent_dependencies(&self) -> String { - if let Some(cargo_toml_path) = &self.workspace_cargo_toml - && let Ok(content) = fs::read_to_string(cargo_toml_path) - { - // Simple parser: extract [workspace.dependencies] or [dependencies] section - if let Some(deps_start) = content.find("[workspace.dependencies]") { - let after_deps = &content[deps_start + "[workspace.dependencies]".len()..]; - - // Find next section (starts with '[') - let deps_end = after_deps.find("\n[").unwrap_or(after_deps.len()); - let deps_section = &after_deps[..deps_end]; - - tracing::info!( - "Copying workspace dependencies from: {}", - cargo_toml_path.display() - ); - return deps_section.trim().to_string(); - } else if let Some(deps_start) = content.find("[dependencies]") { - let after_deps = &content[deps_start + "[dependencies]".len()..]; - - // Find next section (starts with '[') - let deps_end = after_deps.find("\n[").unwrap_or(after_deps.len()); - let deps_section = &after_deps[..deps_end]; - - tracing::info!("Copying dependencies from: {}", cargo_toml_path.display()); - return deps_section.trim().to_string(); - } - } - - String::new() - } - - /// Generate Cargo.toml for the universe crate. - fn generate_cargo_toml(&self) -> String { - let mut toml = String::new(); - - toml.push_str("[package]\n"); - toml.push_str("name = \"venus_universe\"\n"); - toml.push_str("version = \"0.1.0\"\n"); - toml.push_str("edition = \"2021\"\n"); - toml.push('\n'); - toml.push_str("[lib]\n"); - // cdylib for runtime loading, rlib for cell compilation - toml.push_str("crate-type = [\"cdylib\", \"rlib\"]\n"); - toml.push('\n'); - toml.push_str("[dependencies]\n"); - - // Always include rkyv for zero-copy cell serialization - toml.push_str("rkyv = { version = \"0.8\", features = [\"std\", \"bytecheck\"] }\n"); - - // Always include serde_json for widget JSON parsing in cells - toml.push_str("serde_json = \"1.0\"\n"); - - // Always include venus for widget support - if let Some(venus_path) = &self.config.venus_crate_path { - // Use forward slashes for TOML compatibility on Windows - let path_str = venus_path.display().to_string().replace('\\', "/"); - toml.push_str(&format!("venus = {{ path = \"{path_str}\" }}\n")); - } else { - // Use crates.io version when not in development - toml.push_str("venus = \"0.1\"\n"); - } - - for dep in self.dependencies() { - // Skip 'venus' dependency if it's already been auto-added above - // This prevents duplicate key errors in the generated Cargo.toml - if dep.name == "venus" { - continue; - } - - if let Some(path) = &dep.path { - // Use forward slashes for TOML compatibility on Windows - let path_str = path.display().to_string().replace('\\', "/"); - toml.push_str(&format!("{} = {{ path = \"{path_str}\" }}\n", dep.name,)); - } else if let Some(version) = &dep.version { - if dep.features.is_empty() { - toml.push_str(&format!("{} = \"{}\"\n", dep.name, version)); - } else { - toml.push_str(&format!( - "{} = {{ version = \"{}\", features = [{}] }}\n", - dep.name, - version, - dep.features - .iter() - .map(|f| format!("\"{}\"", f)) - .collect::>() - .join(", ") - )); - } - } - } - - // Copy dependencies from parent Cargo.toml (for user's project dependencies) - // Filter out already-added dependencies to avoid duplicates - let parent_deps = self.copy_parent_dependencies(); - if !parent_deps.is_empty() { - toml.push('\n'); - toml.push_str("# Dependencies from parent Cargo.toml\n"); - - // Parse and filter out duplicates (venus, rkyv, serde_json) - for line in parent_deps.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - continue; - } - - // Extract dependency name (before '=') - if let Some(dep_name) = trimmed.split('=').next() { - let dep_name = dep_name.trim(); - // Skip if already added or internal venus crates - if dep_name == "venus" - || dep_name == "venus-macros" - || dep_name == "venus-core" - || dep_name == "venus-sync" - || dep_name == "venus-server" - || dep_name == "rkyv" - || dep_name == "serde_json" - || dep_name == "serde" - { - continue; - } - } - - toml.push_str(line); - toml.push('\n'); - } - } - - // Add empty [workspace] to make this a standalone workspace - // This prevents it from being pulled into parent workspaces - toml.push_str("\n[workspace]\n"); - - toml - } - - /// Generate lib.rs that re-exports all dependencies and includes user types. - fn generate_lib_rs(&self) -> String { - let mut lib = String::new(); - - lib.push_str("//! Venus universe - re-exports all notebook dependencies.\n\n"); - - // Allow common lints in generated code - lib.push_str("#![allow(unused_imports)]\n"); - lib.push_str("#![allow(dead_code)]\n\n"); - - // Always re-export rkyv for zero-copy cell serialization - lib.push_str("pub use rkyv;\n"); - // Re-export derive macros for convenience - lib.push_str("pub use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize};\n"); - lib.push_str("pub use rkyv::rancor::Error as RkyvError;\n\n"); - - // Re-export serde_json for widget JSON parsing in cell wrappers - lib.push_str("pub use serde_json;\n\n"); - - // Re-export venus widget functions and types for interactive notebooks - lib.push_str( - "pub use venus::{input_slider, input_slider_with_step, input_slider_labeled};\n", - ); - lib.push_str("pub use venus::{input_text, input_text_with_default, input_text_labeled};\n"); - lib.push_str("pub use venus::{input_select, input_select_labeled};\n"); - lib.push_str("pub use venus::{input_checkbox, input_checkbox_labeled};\n"); - lib.push_str("pub use venus::widgets::{WidgetContext, WidgetValue, WidgetDef};\n"); - lib.push_str("pub use venus::widgets::{set_widget_context, take_widget_context};\n\n"); - - for dep in self.dependencies() { - // Convert crate name to valid Rust identifier - let ident = dep.name.replace('-', "_"); - // Skip rkyv and venus since we already exported them above - if ident != "rkyv" && ident != "venus" { - lib.push_str(&format!("pub use {};\n", ident)); - } - } - - // Re-export notebook imports so their names are visible to cells (which - // link this crate and glob-import it via `use venus_universe::*;`). - if !self.imports.is_empty() { - lib.push_str("\n// Notebook imports (re-exported for cells)\n"); - lib.push_str(&self.imports); - lib.push('\n'); - } - - // Include user-defined type definitions from the notebook - if !self.type_definitions.is_empty() { - lib.push_str("\n// User-defined types from notebook\n"); - lib.push_str(&self.type_definitions); - } - - // Include notebook module for LSP analysis - // This module is written by the server with current cell content - lib.push_str("\n// Notebook cells (for LSP analysis)\n"); - lib.push_str("pub mod notebook;\n"); - - lib - } - - /// Get the cache hash file path. - fn cache_hash_file(&self) -> PathBuf { - self.config.cache_dir.join("universe_hash") - } - - /// Save the current hash to cache. - fn save_cache_hash(&self) -> Result<()> { - let cache_dir = &self.config.cache_dir; - fs::create_dir_all(cache_dir)?; - - let hash_file = self.cache_hash_file(); - fs::write(&hash_file, self.deps_hash().to_string())?; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::graph::{CellId, DefinitionType, SourceSpan}; - - fn make_builder() -> UniverseBuilder { - let config = CompilerConfig::default(); - let toolchain = ToolchainManager::new().unwrap(); - UniverseBuilder::new(config, toolchain, None) - } - - /// Build a definition cell of the given type with the given content. - fn def_cell(content: &str, definition_type: DefinitionType) -> DefinitionCell { - DefinitionCell { - id: CellId::new(0), - content: content.to_string(), - definition_type, - span: SourceSpan { - start_line: 1, - start_col: 0, - end_line: 1, - end_col: content.len(), - }, - source_file: PathBuf::from("notebook.rs"), - doc_comment: None, - } - } - - /// Assert that a notebook `use` statement referencing `path_fragment` appears - /// in the generated universe and is re-exported (`pub use`), never left as a - /// private `use`. A private `use` in the universe crate is invisible to cells - /// (which link the universe and glob-import it), which is exactly the failure - /// where a notebook import "doesn't work" for a cell that names the type. - fn assert_import_reexported(lib: &str, path_fragment: &str) { - // The universe is rendered from `syn` tokens, which space out `::`, so - // compare with all whitespace removed. - let strip = |s: &str| s.chars().filter(|c| !c.is_whitespace()).collect::(); - let needle = strip(path_fragment); - - let matching: Vec<&str> = lib - .lines() - .map(|l| l.trim()) - .filter(|l| l.contains("use") && strip(l).contains(&needle)) - .collect(); - - assert!( - !matching.is_empty(), - "expected an import referencing `{path_fragment}` in generated universe, found none:\n{lib}" - ); - - for line in matching { - assert!( - strip(line).starts_with("pubuse"), - "notebook import `{line}` must be re-exported as `pub use` so cells can see it, \ - but it was emitted as a private `use` (invisible through `use venus_universe::*`)" - ); - } - } - - #[test] - fn universe_reexports_named_import_from_definition_cell() { - let mut builder = make_builder(); - let cells = vec![def_cell( - "use plotlars::polars::prelude::DataFrame;", - DefinitionType::Import, - )]; - builder.parse_dependencies("", &cells).unwrap(); - - let lib = builder.generate_lib_rs(); - assert_import_reexported(&lib, "plotlars::polars::prelude::DataFrame"); - } - - #[test] - fn universe_reexports_glob_import_from_definition_cell() { - let mut builder = make_builder(); - let cells = vec![def_cell("use venus::prelude::*;", DefinitionType::Import)]; - builder.parse_dependencies("", &cells).unwrap(); - - let lib = builder.generate_lib_rs(); - assert_import_reexported(&lib, "venus::prelude::*"); - } - - #[test] - fn universe_reexports_aliased_import_from_definition_cell() { - let mut builder = make_builder(); - let cells = vec![def_cell( - "use std::collections::HashMap as Map;", - DefinitionType::Import, - )]; - builder.parse_dependencies("", &cells).unwrap(); - - let lib = builder.generate_lib_rs(); - assert_import_reexported(&lib, "std::collections::HashMap as Map"); - } - - #[test] - fn universe_reexports_multiple_imports_in_one_cell() { - let mut builder = make_builder(); - let cells = vec![def_cell( - "use std::collections::HashMap;\nuse std::collections::BTreeMap;", - DefinitionType::Import, - )]; - builder.parse_dependencies("", &cells).unwrap(); - - let lib = builder.generate_lib_rs(); - assert_import_reexported(&lib, "std::collections::HashMap"); - assert_import_reexported(&lib, "std::collections::BTreeMap"); - } - - #[test] - fn universe_reexports_use_from_mixed_definition_cell() { - let mut builder = make_builder(); - // A single definition cell that mixes a `use` with a type definition is - // classified as `Mixed`; its import must still reach cells, and its type - // must still be present. - let cells = vec![def_cell( - "use std::collections::HashMap;\n\npub struct Config {\n pub map: HashMap,\n}", - DefinitionType::Mixed, - )]; - builder.parse_dependencies("", &cells).unwrap(); - - let lib = builder.generate_lib_rs(); - assert_import_reexported(&lib, "std::collections::HashMap"); - assert!( - lib.contains("struct Config"), - "type definition from a mixed cell must still be present in the universe:\n{lib}" - ); - } - - #[test] - fn test_parse_simple_dependency() { - let mut builder = make_builder(); - - let source = r#" -//! # My Notebook -//! -//! ```cargo -//! [dependencies] -//! serde = "1.0" -//! ``` - -#[venus::cell] -pub fn hello() -> i32 { 42 } -"#; - - builder.parse_dependencies(source, &[]).unwrap(); - - assert_eq!(builder.dependencies().len(), 1); - assert_eq!(builder.dependencies()[0].name, "serde"); - assert_eq!(builder.dependencies()[0].version, Some("1.0".to_string())); - } - - #[test] - fn test_parse_complex_dependency() { - let mut builder = make_builder(); - - let source = r#" -//! ```cargo -//! [dependencies] -//! tokio = { version = "1", features = ["full"] } -//! ``` -"#; - - builder.parse_dependencies(source, &[]).unwrap(); - - assert_eq!(builder.dependencies().len(), 1); - assert_eq!(builder.dependencies()[0].name, "tokio"); - assert_eq!(builder.dependencies()[0].version, Some("1".to_string())); - assert_eq!(builder.dependencies()[0].features, vec!["full"]); - } - - #[test] - fn test_parse_multiple_dependencies() { - let mut builder = make_builder(); - - let source = r#" -//! ```cargo -//! [dependencies] -//! serde = "1.0" -//! serde_json = "1.0" -//! tokio = { version = "1", features = ["rt", "macros"] } -//! ``` -"#; - - builder.parse_dependencies(source, &[]).unwrap(); - - assert_eq!(builder.dependencies().len(), 3); - } - - #[test] - fn test_generate_cargo_toml() { - let mut builder = make_builder(); - - // Parse a dependency with features - let source = r#" -//! ```cargo -//! [dependencies] -//! serde = { version = "1.0", features = ["derive"] } -//! ``` -"#; - builder.parse_dependencies(source, &[]).unwrap(); - - let toml = builder.generate_cargo_toml(); - assert!(toml.contains("[package]")); - assert!(toml.contains("venus_universe")); - assert!(toml.contains("serde")); - assert!(toml.contains("derive")); - } - - #[test] - fn test_hash_changes_with_deps() { - let mut builder = make_builder(); - - builder.parse_dependencies("", &[]).unwrap(); - let hash1 = builder.deps_hash(); - - builder - .parse_dependencies( - r#" -//! ```cargo -//! [dependencies] -//! serde = "1.0" -//! ``` -"#, - &[], - ) - .unwrap(); - let hash2 = builder.deps_hash(); - - assert_ne!(hash1, hash2); - } -} diff --git a/crates/venus-core/src/compile/universe/builder.rs b/crates/venus-core/src/compile/universe/builder.rs new file mode 100644 index 0000000..0e677cc --- /dev/null +++ b/crates/venus-core/src/compile/universe/builder.rs @@ -0,0 +1,263 @@ +//! Universe builder for Venus notebooks. +//! +//! The "Universe" is a shared library containing all external dependencies +//! that cells can link against. It's compiled once with LLVM and cached. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::error::{Error, Result}; +use crate::graph::DefinitionCell; + +use super::super::definition_processor::process_definitions; +use super::super::dependency_parser::{DependencyParser, ExternalDependency}; +use super::super::notebook_manifest::{NotebookContext, ResolvedManifest}; +use super::super::toolchain::ToolchainManager; +use super::super::types::{CompilerConfig, dylib_extension, dylib_prefix}; + +/// Builder for the Universe shared library. +pub struct UniverseBuilder { + /// Compiler configuration + pub(super) config: CompilerConfig, + + /// Toolchain manager (reserved for future LLVM compilation options). + /// + /// Currently unused but intentionally preserved to avoid breaking API changes + /// when LLVM backend support is added. The universe is currently built using + /// Cargo/rustc, but future versions may support direct LLVM compilation for + /// optimization control and faster compilation times. + /// + /// Keeping this field now prevents: + /// - Breaking API changes to `UniverseBuilder::new()` + /// - Re-threading toolchain manager through the codebase later + /// - Inconsistency with cell compilation (which does use toolchain) + #[allow(dead_code)] + toolchain: ToolchainManager, + + /// Dependency parser (handles parsing and hashing) + parser: DependencyParser, + + /// User-defined type definitions extracted from notebook (promoted to `pub`, + /// with rkyv derives applied). + pub(super) type_definitions: String, + + /// Notebook `use` statements from definition cells, re-exported as `pub use` + /// so imported names are visible to every compiled cell. + pub(super) imports: String, + + context: NotebookContext, + pub(super) resolved: ResolvedManifest, + cache_salt: Option, + resolution_error: Option, +} + +impl UniverseBuilder { + /// Create a new universe builder. + pub fn new( + config: CompilerConfig, + toolchain: ToolchainManager, + workspace_cargo_toml: Option, + ) -> Self { + let mut builder = Self { + config, + toolchain, + parser: DependencyParser::new(), + type_definitions: String::new(), + imports: String::new(), + context: NotebookContext::from_manifest(workspace_cargo_toml), + resolved: ResolvedManifest::default(), + cache_salt: None, + resolution_error: None, + }; + if let Err(error) = builder.parse_dependencies("", &[]) { + builder.resolution_error = Some(error.to_string()); + } + builder + } + + /// Create a builder with the notebook's Cargo context. + pub fn for_notebook( + config: CompilerConfig, + toolchain: ToolchainManager, + notebook_path: impl AsRef, + ) -> Result { + let context = NotebookContext::for_notebook(notebook_path.as_ref())?; + let mut builder = Self::new(config, toolchain, None); + builder.context = context; + if let Err(error) = builder.parse_dependencies("", &[]) { + builder.resolution_error = Some(error.to_string()); + } + Ok(builder) + } + + /// Resolve notebook dependencies and process definition cells. + /// + /// Definition cells provide public imports and types with rkyv derives. + pub fn parse_dependencies( + &mut self, + source: &str, + definition_cells: &[DefinitionCell], + ) -> Result<()> { + let resolved = ResolvedManifest::resolve(&self.context, source)?; + self.parser + .set_dependencies(resolved.external_dependencies()); + self.resolved = resolved.with_runtime(&self.config, &self.context.directory)?; + self.resolution_error = None; + self.cache_salt = self.resolved.has_local_paths().then(uuid::Uuid::new_v4); + + let contents: Vec = definition_cells + .iter() + .map(|cell| cell.content.clone()) + .collect(); + let processed = process_definitions(&contents); + self.imports = processed.imports; + self.type_definitions = processed.type_definitions; + + Ok(()) + } + + /// Get the dependencies hash (includes imports and type definitions). + pub fn deps_hash(&self) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + self.resolved.canonical().hash(&mut hasher); + self.resolution_error.hash(&mut hasher); + self.cache_salt.hash(&mut hasher); + self.config.venus_crate_path.hash(&mut hasher); + self.config.use_cranelift.hash(&mut hasher); + self.config.debug_info.hash(&mut hasher); + self.config.opt_level.hash(&mut hasher); + self.config.extra_rustc_flags.hash(&mut hasher); + self.imports.hash(&mut hasher); + self.type_definitions.hash(&mut hasher); + hasher.finish() + } + + /// Get the parsed external dependencies. + pub fn dependencies(&self) -> &[ExternalDependency] { + self.parser.dependencies() + } + + /// Check if the cached universe is valid. + pub fn is_cache_valid(&self) -> bool { + if self.resolution_error.is_some() || self.resolved.has_local_paths() { + return false; + } + let cache_file = self.cache_hash_file(); + if !cache_file.exists() { + return false; + } + + // Read cached hash + if let Ok(cached_hash) = fs::read_to_string(&cache_file) + && let Ok(hash) = cached_hash.trim().parse::() + { + return hash == self.deps_hash(); + } + + false + } + + /// Get the path to the compiled universe library. + pub fn universe_path(&self) -> PathBuf { + let build_dir = self.config.universe_build_dir(); + let filename = format!("{}venus_universe.{}", dylib_prefix(), dylib_extension()); + build_dir.join(filename) + } + + /// Build the universe library. + pub fn build(&self) -> Result { + if let Some(message) = &self.resolution_error { + return Err(Error::Compilation { + cell_id: None, + message: message.clone(), + }); + } + // Check cache first + if self.is_cache_valid() && self.universe_path().exists() { + tracing::info!("Using cached universe library"); + return Ok(self.universe_path()); + } + + tracing::info!( + "Building universe library with {} dependencies", + self.dependencies().len() + ); + + let build_dir = self.config.universe_build_dir(); + fs::create_dir_all(&build_dir)?; + + // Generate Cargo.toml + let cargo_toml = self.generate_cargo_toml()?; + let cargo_path = build_dir.join("Cargo.toml"); + fs::write(&cargo_path, cargo_toml)?; + fs::write(build_dir.join("build.rs"), self.resolved.build_script())?; + + // Generate lib.rs + let lib_rs = self.generate_lib_rs(); + let src_dir = build_dir.join("src"); + fs::create_dir_all(&src_dir)?; + fs::write(src_dir.join("lib.rs"), lib_rs)?; + + // Generate stub notebook.rs (required by lib.rs `pub mod notebook;`) + // The server overwrites this with real cell content for LSP analysis. + // For CLI builds, this stub satisfies the module declaration. + let notebook_rs = "//! Notebook cells module.\n\ + //! This file is populated by the server for LSP analysis.\n\ + //! For CLI builds, this is a stub to satisfy the module declaration.\n"; + fs::write(src_dir.join("notebook.rs"), notebook_rs)?; + + // Build with cargo + let output = Command::new("cargo") + .current_dir(&build_dir) + .args(["build", "--release", "--lib"]) + .output() + .map_err(|e| Error::Compilation { + cell_id: None, + message: format!("Failed to run cargo: {}", e), + })?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::Compilation { + cell_id: None, + message: format!("Universe build failed:\n{}", stderr), + }); + } + + // Copy the built library + let target_lib = build_dir.join("target").join("release").join(format!( + "{}venus_universe.{}", + dylib_prefix(), + dylib_extension() + )); + + let dest = self.universe_path(); + fs::copy(&target_lib, &dest)?; + + // Save cache hash + self.save_cache_hash()?; + + tracing::info!("Universe library built: {}", dest.display()); + Ok(dest) + } + + /// Get the cache hash file path. + fn cache_hash_file(&self) -> PathBuf { + self.config.cache_dir.join("universe_hash") + } + + /// Save the current hash to cache. + fn save_cache_hash(&self) -> Result<()> { + let cache_dir = &self.config.cache_dir; + fs::create_dir_all(cache_dir)?; + + let hash_file = self.cache_hash_file(); + fs::write(&hash_file, self.deps_hash().to_string())?; + + Ok(()) + } +} diff --git a/crates/venus-core/src/compile/universe/generation.rs b/crates/venus-core/src/compile/universe/generation.rs new file mode 100644 index 0000000..fed2924 --- /dev/null +++ b/crates/venus-core/src/compile/universe/generation.rs @@ -0,0 +1,68 @@ +use crate::compile::ManifestConfig; +use crate::error::Result; + +use super::UniverseBuilder; + +impl UniverseBuilder { + pub(super) fn generate_cargo_toml(&self) -> Result { + self.resolved.render(&ManifestConfig { + name: "venus_universe", + lib_crate_types: Some(&["cdylib", "rlib"]), + standalone_workspace: true, + ..ManifestConfig::default() + }) + } + + /// Generate lib.rs that re-exports all dependencies and includes user types. + pub(super) fn generate_lib_rs(&self) -> String { + let mut lib = String::new(); + + lib.push_str("//! Venus universe - re-exports all notebook dependencies.\n\n"); + + // Allow common lints in generated code + lib.push_str("#![allow(unused_imports)]\n"); + lib.push_str("#![allow(dead_code)]\n\n"); + + // Always re-export rkyv for zero-copy cell serialization + lib.push_str("pub use rkyv;\n"); + // Re-export derive macros for convenience + lib.push_str("pub use rkyv::{Archive, Serialize as RkyvSerialize, Deserialize as RkyvDeserialize};\n"); + lib.push_str("pub use rkyv::rancor::Error as RkyvError;\n\n"); + + // Re-export serde_json for widget JSON parsing in cell wrappers + lib.push_str("pub use serde_json;\n\n"); + + // Re-export venus widget functions and types for interactive notebooks + lib.push_str( + "pub use venus::{input_slider, input_slider_with_step, input_slider_labeled};\n", + ); + lib.push_str("pub use venus::{input_text, input_text_with_default, input_text_labeled};\n"); + lib.push_str("pub use venus::{input_select, input_select_labeled};\n"); + lib.push_str("pub use venus::{input_checkbox, input_checkbox_labeled};\n"); + lib.push_str("pub use venus::widgets::{WidgetContext, WidgetValue, WidgetDef};\n"); + lib.push_str("pub use venus::widgets::{set_widget_context, take_widget_context};\n\n"); + + lib.push_str(&self.resolved.reexports()); + + // Re-export notebook imports so their names are visible to cells (which + // link this crate and glob-import it via `use venus_universe::*;`). + if !self.imports.is_empty() { + lib.push_str("\n// Notebook imports (re-exported for cells)\n"); + lib.push_str(&self.imports); + lib.push('\n'); + } + + // Include user-defined type definitions from the notebook + if !self.type_definitions.is_empty() { + lib.push_str("\n// User-defined types from notebook\n"); + lib.push_str(&self.type_definitions); + } + + // Include notebook module for LSP analysis + // This module is written by the server with current cell content + lib.push_str("\n// Notebook cells (for LSP analysis)\n"); + lib.push_str("pub mod notebook;\n"); + + lib + } +} diff --git a/crates/venus-core/src/compile/universe/mod.rs b/crates/venus-core/src/compile/universe/mod.rs new file mode 100644 index 0000000..a7e90c2 --- /dev/null +++ b/crates/venus-core/src/compile/universe/mod.rs @@ -0,0 +1,8 @@ +//! Shared dependency library for notebook cells. + +mod builder; +mod generation; +#[cfg(test)] +mod tests; + +pub use builder::UniverseBuilder; diff --git a/crates/venus-core/src/compile/universe/tests.rs b/crates/venus-core/src/compile/universe/tests.rs new file mode 100644 index 0000000..9953c34 --- /dev/null +++ b/crates/venus-core/src/compile/universe/tests.rs @@ -0,0 +1,275 @@ +use super::UniverseBuilder; +use crate::compile::{CompilerConfig, ToolchainManager}; +use crate::graph::{CellId, DefinitionCell, DefinitionType, SourceSpan}; +use std::path::PathBuf; + +fn make_builder() -> UniverseBuilder { + let config = CompilerConfig::default(); + let toolchain = ToolchainManager::new().unwrap(); + UniverseBuilder::new(config, toolchain, None) +} + +/// Build a definition cell of the given type with the given content. +fn def_cell(content: &str, definition_type: DefinitionType) -> DefinitionCell { + DefinitionCell { + id: CellId::new(0), + content: content.to_string(), + definition_type, + span: SourceSpan { + start_line: 1, + start_col: 0, + end_line: 1, + end_col: content.len(), + }, + source_file: PathBuf::from("notebook.rs"), + doc_comment: None, + } +} + +/// Assert that a notebook `use` statement referencing `path_fragment` appears +/// in the generated universe and is re-exported (`pub use`), never left as a +/// private `use`. A private `use` in the universe crate is invisible to cells +/// (which link the universe and glob-import it), which is exactly the failure +/// where a notebook import "doesn't work" for a cell that names the type. +fn assert_import_reexported(lib: &str, path_fragment: &str) { + // The universe is rendered from `syn` tokens, which space out `::`, so + // compare with all whitespace removed. + let strip = |s: &str| s.chars().filter(|c| !c.is_whitespace()).collect::(); + let needle = strip(path_fragment); + + let matching: Vec<&str> = lib + .lines() + .map(|l| l.trim()) + .filter(|l| l.contains("use") && strip(l).contains(&needle)) + .collect(); + + assert!( + !matching.is_empty(), + "expected an import referencing `{path_fragment}` in generated universe, found none:\n{lib}" + ); + + for line in matching { + assert!( + strip(line).starts_with("pubuse"), + "notebook import `{line}` must be re-exported as `pub use` so cells can see it, \ + but it was emitted as a private `use` (invisible through `use venus_universe::*`)" + ); + } +} + +#[test] +fn universe_reexports_named_import_from_definition_cell() { + let mut builder = make_builder(); + let cells = vec![def_cell( + "use plotlars::polars::prelude::DataFrame;", + DefinitionType::Import, + )]; + builder.parse_dependencies("", &cells).unwrap(); + + let lib = builder.generate_lib_rs(); + assert_import_reexported(&lib, "plotlars::polars::prelude::DataFrame"); +} + +#[test] +fn universe_reexports_glob_import_from_definition_cell() { + let mut builder = make_builder(); + let cells = vec![def_cell("use venus::prelude::*;", DefinitionType::Import)]; + builder.parse_dependencies("", &cells).unwrap(); + + let lib = builder.generate_lib_rs(); + assert_import_reexported(&lib, "venus::prelude::*"); +} + +#[test] +fn universe_reexports_aliased_import_from_definition_cell() { + let mut builder = make_builder(); + let cells = vec![def_cell( + "use std::collections::HashMap as Map;", + DefinitionType::Import, + )]; + builder.parse_dependencies("", &cells).unwrap(); + + let lib = builder.generate_lib_rs(); + assert_import_reexported(&lib, "std::collections::HashMap as Map"); +} + +#[test] +fn universe_reexports_multiple_imports_in_one_cell() { + let mut builder = make_builder(); + let cells = vec![def_cell( + "use std::collections::HashMap;\nuse std::collections::BTreeMap;", + DefinitionType::Import, + )]; + builder.parse_dependencies("", &cells).unwrap(); + + let lib = builder.generate_lib_rs(); + assert_import_reexported(&lib, "std::collections::HashMap"); + assert_import_reexported(&lib, "std::collections::BTreeMap"); +} + +#[test] +fn universe_reexports_use_from_mixed_definition_cell() { + let mut builder = make_builder(); + // A single definition cell that mixes a `use` with a type definition is + // classified as `Mixed`; its import must still reach cells, and its type + // must still be present. + let cells = vec![def_cell( + "use std::collections::HashMap;\n\npub struct Config {\n pub map: HashMap,\n}", + DefinitionType::Mixed, + )]; + builder.parse_dependencies("", &cells).unwrap(); + + let lib = builder.generate_lib_rs(); + assert_import_reexported(&lib, "std::collections::HashMap"); + assert!( + lib.contains("struct Config"), + "type definition from a mixed cell must still be present in the universe:\n{lib}" + ); +} + +#[test] +fn test_parse_simple_dependency() { + let mut builder = make_builder(); + + let source = r#" +//! # My Notebook +//! +//! ```cargo +//! [dependencies] +//! serde = "1.0" +//! ``` + +#[venus::cell] +pub fn hello() -> i32 { 42 } +"#; + + builder.parse_dependencies(source, &[]).unwrap(); + + assert_eq!(builder.dependencies().len(), 1); + assert_eq!(builder.dependencies()[0].name, "serde"); + assert_eq!(builder.dependencies()[0].version, Some("1.0".to_string())); +} + +#[test] +fn test_parse_complex_dependency() { + let mut builder = make_builder(); + + let source = r#" +//! ```cargo +//! [dependencies] +//! tokio = { version = "1", features = ["full"] } +//! ``` +"#; + + builder.parse_dependencies(source, &[]).unwrap(); + + assert_eq!(builder.dependencies().len(), 1); + assert_eq!(builder.dependencies()[0].name, "tokio"); + assert_eq!(builder.dependencies()[0].version, Some("1".to_string())); + assert_eq!(builder.dependencies()[0].features, vec!["full"]); +} + +#[test] +fn test_parse_multiple_dependencies() { + let mut builder = make_builder(); + + let source = r#" +//! ```cargo +//! [dependencies] +//! serde = "1.0" +//! serde_json = "1.0" +//! tokio = { version = "1", features = ["rt", "macros"] } +//! ``` +"#; + + builder.parse_dependencies(source, &[]).unwrap(); + + assert_eq!(builder.dependencies().len(), 3); +} + +#[test] +fn test_generate_cargo_toml() { + let mut builder = make_builder(); + + // Parse a dependency with features + let source = r#" +//! ```cargo +//! [dependencies] +//! serde = { version = "1.0", features = ["derive"] } +//! ``` +"#; + builder.parse_dependencies(source, &[]).unwrap(); + + let toml = builder.generate_cargo_toml().unwrap(); + assert!(toml.contains("[package]")); + assert!(toml.contains("venus_universe")); + assert!(toml.contains("serde")); + assert!(toml.contains("derive")); +} + +#[test] +fn test_hash_changes_with_deps() { + let mut builder = make_builder(); + + builder.parse_dependencies("", &[]).unwrap(); + let hash1 = builder.deps_hash(); + + builder + .parse_dependencies( + r#" +//! ```cargo +//! [dependencies] +//! serde = "1.0" +//! ``` +"#, + &[], + ) + .unwrap(); + let hash2 = builder.deps_hash(); + + assert_ne!(hash1, hash2); +} + +#[test] +fn notebook_override_clears_parent_runtime_conflict_after_construction() { + let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("Cargo.toml"), + r#" +[package] +name = "parent" +version = "0.1.0" +[dependencies] +rkyv = "0.7" +"#, + ) + .unwrap(); + let source = r#" +//! ```cargo +//! [dependencies] +//! rkyv = "0.8" +//! ``` +"#; + let notebook = directory.path().join("notebook.rs"); + std::fs::write(¬ebook, source).unwrap(); + let config = CompilerConfig { + venus_crate_path: None, + ..CompilerConfig::default() + }; + let mut builder = + UniverseBuilder::for_notebook(config, ToolchainManager::new().unwrap(), ¬ebook).unwrap(); + assert!(builder.build().unwrap_err().to_string().contains("rkyv")); + builder.parse_dependencies(source, &[]).unwrap(); + let manifest: toml::Table = toml::from_str(&builder.generate_cargo_toml().unwrap()).unwrap(); + let requirement = semver::VersionReq::parse( + manifest["dependencies"]["rkyv"]["version"] + .as_str() + .unwrap(), + ) + .unwrap(); + assert!(requirement.matches(&semver::Version::new(0, 8, 12))); + assert!(!requirement.matches(&semver::Version::new(0, 7, 0))); + assert!(builder.dependencies().iter().any( + |dependency| dependency.name == "rkyv" && dependency.version.as_deref() == Some("0.8") + )); +} diff --git a/crates/venus-core/tests/workspace_dependencies.rs b/crates/venus-core/tests/workspace_dependencies.rs new file mode 100644 index 0000000..5379b0f --- /dev/null +++ b/crates/venus-core/tests/workspace_dependencies.rs @@ -0,0 +1,354 @@ +//! Generated notebook projects retain Cargo workspace dependency semantics. + +use std::fs; +use std::hash::{Hash, Hasher}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use tempfile::TempDir; +use venus_core::compile::{ + CompilerConfig, ProductionBuilder, ToolchainManager, UniverseBuilder, find_notebook_manifest, +}; + +fn write(root: &Path, path: &str, source: &str) { + let path = root.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, source).unwrap(); +} + +fn package(root: &Path, directory: &str, name: &str, manifest: &str, source: &str) { + write( + root, + &format!("{directory}/Cargo.toml"), + &format!("[package]\nname = {name:?}\nversion = \"0.1.0\"\nedition = \"2021\"\n{manifest}"), + ); + write(root, &format!("{directory}/src/lib.rs"), source); +} + +fn runtime(root: &Path) -> PathBuf { + let mut source = String::new(); + for function in [ + "input_slider", + "input_slider_with_step", + "input_slider_labeled", + "input_text", + "input_text_with_default", + "input_text_labeled", + "input_select", + "input_select_labeled", + "input_checkbox", + "input_checkbox_labeled", + ] { + source.push_str(&format!("pub fn {function}() {{}}\n")); + } + source.push_str("pub mod widgets { pub struct WidgetContext; pub struct WidgetValue; pub struct WidgetDef; pub fn set_widget_context() {} pub fn take_widget_context() {} }\n"); + package(root, "runtime", "venus", "", &source); + root.join("runtime") +} + +fn config(root: &Path, runtime: PathBuf) -> CompilerConfig { + CompilerConfig { + build_dir: root.join("artifacts/build"), + cache_dir: root.join("artifacts/cache"), + venus_crate_path: Some(runtime), + use_cranelift: false, + ..CompilerConfig::default() + } +} + +fn host() -> String { + let output = Command::new("rustc").arg("-vV").output().unwrap(); + assert!(output.status.success()); + String::from_utf8(output.stdout) + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("host: ")) + .unwrap() + .to_string() +} + +fn cell_value(config: &CompilerConfig, root: &Path, expression: &str, deps_hash: u64) -> i32 { + let extension = std::env::consts::DLL_EXTENSION; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + expression.hash(&mut hasher); + let expression_hash = hasher.finish(); + let cell_path = root.join(format!( + "cell_{expression_hash:x}_{deps_hash:x}.{extension}" + )); + let source = root.join("cell.rs"); + fs::write(&source, format!("extern crate venus_universe;\nuse venus_universe::*;\n#[no_mangle] pub extern \"C\" fn cell_value() -> i32 {{ {expression} }}")).unwrap(); + let output = Command::new("rustc") + .args(["--edition=2021", "--crate-type=cdylib", "--extern"]) + .arg(format!( + "venus_universe={}", + config + .universe_build_dir() + .join("target/release/libvenus_universe.rlib") + .display() + )) + .arg("-L") + .arg(format!( + "dependency={}", + config + .universe_build_dir() + .join("target/release/deps") + .display() + )) + .arg(&source) + .arg("-o") + .arg(&cell_path) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + unsafe { + let library = libloading::Library::new(cell_path).unwrap(); + let function: libloading::Symbol i32> = + library.get(b"cell_value").unwrap(); + function() + } +} + +fn fixture() -> (TempDir, PathBuf, CompilerConfig) { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + let runtime_path = runtime(root); + package( + root, + "helper", + "internal-helper", + "[features]\ndefault = [\"default-on\"]\ndefault-on = []\nbase = []\nextra = []\n", + "#[cfg(feature = \"default-on\")] compile_error!(\"workspace defaults must stay disabled\"); pub fn value() -> i32 { 40 }", + ); + package( + root, + "optional", + "optional-helper", + "[features]\nenabled = []\n", + "#[cfg(feature = \"enabled\")] pub fn value() -> i32 { 2 }\n#[cfg(not(feature = \"enabled\"))] pub fn value() -> i32 { 200 }", + ); + package( + root, + "active", + "active-helper", + "", + "pub fn value() -> i32 { 1 }", + ); + package( + root, + "blocked", + "blocked-helper", + "", + "compile_error!(\"inactive dependency must stay disabled\");", + ); + package( + root, + "inactive", + "inactive-helper", + "", + "compile_error!(\"inactive target must stay disabled\");", + ); + package( + root, + "patch", + "patched-helper", + "", + "pub fn value() -> i32 { 3 }", + ); + write( + root, + "Cargo.toml", + r#" +[workspace] +members = ["member", "runtime", "helper", "optional", "active", "blocked", "inactive", "patch"] +[workspace.dependencies] +internal-alias = { package = "internal-helper", path = "helper", default-features = false, features = ["base"] } +unused = "999.0" +[patch.crates-io] +patched-helper = { path = "patch" } +"#, + ); + let member_manifest = format!( + r#" +[dependencies] +internal-alias = {{ workspace = true, features = ["extra"] }} +optional-alias = {{ package = "optional-helper", path = "../optional", optional = true }} +blocked = {{ package = "blocked-helper", path = "../blocked", optional = true }} +patched-helper = "0.1" +serde_json = {{ version = "1.0", optional = true }} +[target.{host:?}.dependencies] +active-alias = {{ package = "active-helper", path = "../active" }} +[target.'cfg(target_os = "none")'.dependencies] +inactive = {{ package = "inactive-helper", path = "../inactive" }} +[features] +default = ["outer"] +outer = ["activate", "json"] +activate = ["dep:optional-alias", "optional-alias/enabled"] +json = ["dep:serde_json", "serde_json?/raw_value"] +weak = ["optional-alias?/enabled"] +"#, + host = host() + ); + package(root, "member", "member", &member_manifest, ""); + let notebook = root.join("member/notebooks/report.rs"); + write( + root, + "member/notebooks/report.rs", + "#[venus::cell]\npub fn answer() -> i32 { internal_alias::value() + active_alias::value() + optional_alias::value() + patched_helper::value() }\n", + ); + let config = config(root, runtime_path); + (dir, notebook, config) +} + +#[test] +fn generated_universe_and_standalone_binary_execute_workspace_aliases() { + let (dir, notebook, config) = fixture(); + assert_eq!( + find_notebook_manifest(¬ebook), + Some(dir.path().join("member/Cargo.toml").canonicalize().unwrap()) + ); + let mut universe = + UniverseBuilder::for_notebook(config.clone(), ToolchainManager::new().unwrap(), ¬ebook) + .unwrap(); + let source = fs::read_to_string(¬ebook).unwrap(); + universe.parse_dependencies(&source, &[]).unwrap(); + universe.build().unwrap(); + assert_eq!( + cell_value( + &config, + dir.path(), + "internal_alias::value() + active_alias::value() + optional_alias::value() + patched_helper::value()", + universe.deps_hash() + ), + 46 + ); + let manifest: toml::Table = toml::from_str( + &fs::read_to_string(config.universe_build_dir().join("Cargo.toml")).unwrap(), + ) + .unwrap(); + assert!(manifest.contains_key("workspace")); + assert!( + !manifest["dependencies"] + .as_table() + .unwrap() + .contains_key("unused") + ); + for (alias, directory) in [ + ("internal-alias", "helper"), + ("optional-alias", "optional"), + ("blocked", "blocked"), + ("venus", "runtime"), + ] { + let expected = dir.path().join(directory).canonicalize().unwrap(); + assert_eq!( + manifest["dependencies"][alias]["path"].as_str(), + Some(expected.to_string_lossy().as_ref()) + ); + } + for (alias, directory) in [("active-alias", "active"), ("inactive", "inactive")] { + let dependency = manifest["target"] + .as_table() + .unwrap() + .values() + .find_map(|target| target["dependencies"].get(alias)) + .unwrap(); + let expected = dir.path().join(directory).canonicalize().unwrap(); + assert_eq!( + dependency["path"].as_str(), + Some(expected.to_string_lossy().as_ref()) + ); + } + let expected = dir.path().join("patch").canonicalize().unwrap(); + assert_eq!( + manifest["patch"]["crates-io"]["patched-helper"]["path"].as_str(), + Some(expected.to_string_lossy().as_ref()) + ); + assert!(!universe.is_cache_valid()); + let previous_hash = universe.deps_hash(); + fs::write( + dir.path().join("helper/src/lib.rs"), + "pub fn value() -> i32 { 80 }", + ) + .unwrap(); + universe.parse_dependencies(&source, &[]).unwrap(); + assert_ne!(previous_hash, universe.deps_hash()); + universe.build().unwrap(); + assert_eq!( + cell_value( + &config, + dir.path(), + "internal_alias::value() + active_alias::value() + optional_alias::value() + patched_helper::value()", + universe.deps_hash() + ), + 86 + ); + + let mut production = ProductionBuilder::new(config.clone()); + production.load(¬ebook).unwrap(); + let binary = dir.path().join("report-bin"); + production.build(&binary, false).unwrap(); + let output = Command::new(binary).output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("86")); + + // Disabling default activation must remove optional exports and preserve target exports. + let manifest = dir.path().join("member/Cargo.toml"); + fs::write( + &manifest, + fs::read_to_string(&manifest) + .unwrap() + .replace("default = [\"outer\"]", "default = []"), + ) + .unwrap(); + universe.parse_dependencies(&source, &[]).unwrap(); + universe.build().unwrap(); + assert_eq!( + cell_value( + &config, + dir.path(), + "internal_alias::value() + active_alias::value() + patched_helper::value()", + universe.deps_hash() + ), + 84 + ); + let generated = fs::read_to_string(config.universe_build_dir().join("src/lib.rs")).unwrap(); + assert!(generated.contains("pub use optional_alias;")); + assert!(generated.contains("#[cfg(any(any(")); +} + +#[test] +fn standalone_notebook_and_direct_builder_generate_runtime_dependencies() { + let dir = tempfile::tempdir().unwrap(); + let runtime_path = runtime(dir.path()); + let config = config(dir.path(), runtime_path); + let builder = UniverseBuilder::new(config.clone(), ToolchainManager::new().unwrap(), None); + let dependency_hash = builder.deps_hash(); + builder.build().unwrap(); + assert_eq!(builder.deps_hash(), dependency_hash); + assert_eq!( + fs::read_to_string(config.cache_dir.join("universe_hash")).unwrap(), + dependency_hash.to_string() + ); + let manifest: toml::Table = toml::from_str( + &fs::read_to_string(config.universe_build_dir().join("Cargo.toml")).unwrap(), + ) + .unwrap(); + assert!(manifest["dependencies"].get("rkyv").is_some()); + assert!(manifest["dependencies"].get("serde_json").is_some()); + let notebook = dir.path().join("standalone.rs"); + fs::write(¬ebook, "#[venus::cell]\npub fn answer() -> i32 { 7 }\n").unwrap(); + let mut production = ProductionBuilder::new(config); + production.load(¬ebook).unwrap(); + let binary = dir.path().join("standalone-bin"); + production.build(&binary, false).unwrap(); + let output = Command::new(binary).output().unwrap(); + assert!(output.status.success()); + assert!(String::from_utf8_lossy(&output.stdout).contains("7")); +} diff --git a/crates/venus-server/src/session.rs b/crates/venus-server/src/session.rs deleted file mode 100644 index e3ef314..0000000 --- a/crates/venus-server/src/session.rs +++ /dev/null @@ -1,2071 +0,0 @@ -//! Notebook session management. -//! -//! Manages the state of an active notebook session including -//! compilation, execution, and output caching. - -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Duration, Instant}; - -use tokio::sync::{RwLock, broadcast}; -use venus_core::compile::{ - CellCompiler, CompilationResult, CompilerConfig, ToolchainManager, UniverseBuilder, -}; -use venus_core::execute::{ExecutorKillHandle, ProcessExecutor}; -use venus_core::graph::{ - CellId, CellInfo, CellParser, CellType, DefinitionCell, GraphEngine, MarkdownCell, - MoveDirection, SourceEditor, -}; -use venus_core::paths::NotebookDirs; -use venus_core::widgets::{WidgetDef, WidgetValue}; - -use crate::error::{ServerError, ServerResult}; -use crate::protocol::{CellOutput, CellState, CellStatus, ServerMessage}; -use crate::undo::{UndoManager, UndoableOperation}; -use venus_core::state::BoxedOutput; - -/// Find workspace root by walking up from notebook path to find Cargo.toml. -/// Returns (workspace_root, cargo_toml_path). -fn find_workspace_root(notebook_path: &Path) -> (Option, Option) { - let mut current = notebook_path.parent(); - - while let Some(dir) = current { - let cargo_toml = dir.join("Cargo.toml"); - if cargo_toml.exists() { - return (Some(dir.to_path_buf()), Some(cargo_toml)); - } - current = dir.parent(); - } - - (None, None) -} - -/// Shared interrupt flag that can be checked without locks. -pub type InterruptFlag = Arc; - -/// Capacity for the broadcast channel. -/// 256 messages should be sufficient for normal notebook operation. -/// If clients fall behind, older messages will be dropped. -const MESSAGE_CHANNEL_CAPACITY: usize = 256; - -/// A notebook session. -pub struct NotebookSession { - /// Path to the notebook file. - path: PathBuf, - - /// Path to workspace Cargo.toml (if found). - workspace_cargo_toml: Option, - - /// Parsed code cells. - cells: Vec, - - /// Parsed markdown cells. - markdown_cells: Vec, - - /// Parsed definition cells (imports, types, helpers). - definition_cells: Vec, - - /// Dependency graph. - graph: GraphEngine, - - /// Cell states for clients (both code and markdown). - cell_states: HashMap, - - /// Toolchain manager. - toolchain: ToolchainManager, - - /// Compiler configuration. - config: CompilerConfig, - - /// Universe path (compiled dependencies). - universe_path: Option, - - /// Dependencies hash for cache invalidation. - deps_hash: u64, - - /// Broadcast channel for server messages. - tx: broadcast::Sender, - - /// Whether an execution is in progress. - executing: bool, - - /// Cached cell outputs for dependency passing. - /// Maps cell ID to its serialized output. - cell_outputs: HashMap>, - - /// Process-based executor for isolated cell execution. - /// Uses worker processes that can be killed for true interruption. - executor: ProcessExecutor, - - /// Optional execution timeout for execute_all. - /// After this duration, the executor kills the current worker. - execution_timeout: Option, - - /// Shared flag indicating if current execution was interrupted by user. - /// When true, errors should be reported as "interrupted" not as failures. - /// This is shared with AppState so interrupt handler can set it. - interrupted: InterruptFlag, - - /// Widget values per cell. - /// Maps cell ID -> widget ID -> current value. - widget_values: HashMap>, - - /// Widget definitions per cell (from last execution). - /// Used to send widget state to newly connected clients. - widget_defs: HashMap>, - - /// Execution history per cell. - /// Stores both serialized output (for dependent cells) and display output. - cell_output_history: HashMap>, - - /// Current history index per cell. - cell_history_index: HashMap, - - /// Undo/redo manager for cell operations. - undo_manager: UndoManager, - - /// Pending edits from the editor (not yet saved to disk). - /// These are saved to disk when the cell is executed. - pending_edits: HashMap, -} - -/// Maximum number of history entries per cell. -const MAX_HISTORY_PER_CELL: usize = 10; - -/// A single history entry for a cell's execution. -#[derive(Clone)] -pub struct OutputHistoryEntry { - /// Serialized output for passing to dependent cells. - pub serialized: Arc, - /// Display output for the frontend. - pub display: CellOutput, - /// Timestamp when this execution completed. - pub timestamp: u64, -} - -/// Thread-safe session handle. -pub type SessionHandle = Arc>; - -impl NotebookSession { - /// Create a new notebook session. - /// - /// Uses process isolation for cell execution, allowing true interruption - /// by killing worker processes. - /// - /// The `interrupted` flag is shared with AppState so the interrupt handler - /// can set it without needing the session lock. - pub fn new( - path: impl AsRef, - interrupted: InterruptFlag, - ) -> ServerResult<(Self, broadcast::Receiver)> { - let path = path.as_ref().canonicalize().map_err(|e| ServerError::Io { - path: path.as_ref().to_path_buf(), - message: e.to_string(), - })?; - - // Find workspace Cargo.toml (if it exists) - let (_workspace_root, workspace_cargo_toml) = find_workspace_root(&path); - - // Set up directories using shared abstraction - let dirs = NotebookDirs::from_notebook_path(&path)?; - - let toolchain = ToolchainManager::new()?; - let config = CompilerConfig::for_notebook(&dirs); - - let (tx, rx) = broadcast::channel(MESSAGE_CHANNEL_CAPACITY); - - // Create process executor with warm worker pool - let executor = ProcessExecutor::new(&dirs.state_dir)?; - - let mut session = Self { - path, - workspace_cargo_toml, - cells: Vec::new(), - markdown_cells: Vec::new(), - definition_cells: Vec::new(), - graph: GraphEngine::new(), - cell_states: HashMap::new(), - toolchain, - config, - universe_path: None, - deps_hash: 0, - tx, - executing: false, - cell_outputs: HashMap::new(), - executor, - execution_timeout: None, - interrupted, - widget_values: HashMap::new(), - widget_defs: HashMap::new(), - cell_output_history: HashMap::new(), - cell_history_index: HashMap::new(), - undo_manager: UndoManager::new(), - pending_edits: HashMap::new(), - }; - - session.reload()?; - - Ok((session, rx)) - } - - /// Get the notebook path. - pub fn path(&self) -> &Path { - &self.path - } - - /// Subscribe to server messages. - pub fn subscribe(&self) -> broadcast::Receiver { - self.tx.subscribe() - } - - /// Get a cell by ID. - fn get_cell(&self, cell_id: CellId) -> Option<&CellInfo> { - self.cells.iter().find(|c| c.id == cell_id) - } - - /// Set the status of a code cell. - fn set_cell_status(&mut self, cell_id: CellId, status: CellStatus) { - if let Some(CellState::Code { - status: cell_status, - .. - }) = self.cell_states.get_mut(&cell_id) - { - *cell_status = status; - } - } - - /// Broadcast a server message, ignoring send failures. - pub fn broadcast(&self, msg: ServerMessage) { - let _ = self.tx.send(msg); - } - - /// Reload the notebook from disk. - pub fn reload(&mut self) -> ServerResult<()> { - let source = std::fs::read_to_string(&self.path)?; - - // Parse cells (code, markdown, and definitions) - let mut parser = CellParser::new(); - let parse_result = parser.parse_file(&self.path)?; - self.cells = parse_result.code_cells; - self.markdown_cells = parse_result.markdown_cells; - self.definition_cells = parse_result.definition_cells; - - // Build graph and update code cells with real IDs (parser returns placeholder IDs) - self.graph = GraphEngine::new(); - for cell in &mut self.cells { - let real_id = self.graph.add_cell(cell.clone()); - cell.id = real_id; - } - self.graph.resolve_dependencies()?; - - // Assign unique IDs to markdown cells (they don't participate in the dependency graph) - let mut next_id = - if let Some(max_code_id) = self.cells.iter().map(|c| c.id.as_usize()).max() { - max_code_id + 1 - } else { - 0 - }; - for md_cell in &mut self.markdown_cells { - md_cell.id = CellId::new(next_id); - next_id += 1; - } - - // Assign unique IDs to definition cells - for def_cell in &mut self.definition_cells { - def_cell.id = CellId::new(next_id); - next_id += 1; - } - - // Write virtual notebook.rs file for LSP analysis BEFORE building universe - // This ensures the file exists when universe is compiled (lib.rs includes `pub mod notebook;`) - if let Err(e) = self.write_virtual_notebook_file() { - tracing::warn!("Failed to write virtual notebook file: {}", e); - } - - // Build universe (always needed for bincode/serde runtime) - let mut universe_builder = UniverseBuilder::new( - self.config.clone(), - self.toolchain.clone(), - self.workspace_cargo_toml.clone(), - ); - universe_builder.parse_dependencies(&source, &self.definition_cells)?; - - self.universe_path = Some(universe_builder.build()?); - self.deps_hash = universe_builder.deps_hash(); - - // Update cell states - self.update_cell_states(); - - // NOTE: We do NOT broadcast state here because reload() is called by the file watcher - // when the notebook file changes (e.g., editor auto-save). Broadcasting on every file - // change causes the UI to refresh continuously. Instead, each cell operation (insert, - // edit, delete, etc.) explicitly broadcasts state after calling reload(). - - Ok(()) - } - - /// Strip the first heading from a doc comment (since it's used as display name). - /// - /// If the doc comment starts with `# Heading`, removes that line and returns - /// the rest of the content. Otherwise returns the original content. - fn strip_display_name_from_description(doc_comment: &Option) -> Option { - doc_comment.as_ref().and_then(|doc| { - let lines: Vec<&str> = doc.lines().collect(); - - // Find the first heading line - if let Some(first_line) = lines.first() { - let trimmed = first_line.trim(); - if trimmed.starts_with('#') { - // Skip the heading line and return the rest - let remaining: Vec<&str> = lines.iter().skip(1).copied().collect(); - - // Trim leading empty lines - let trimmed_lines: Vec<&str> = remaining - .iter() - .skip_while(|line| line.trim().is_empty()) - .copied() - .collect(); - - if trimmed_lines.is_empty() { - return None; - } - - return Some(trimmed_lines.join("\n")); - } - } - - // No heading found, return original - Some(doc.clone()) - }) - } - - /// Update cell states from parsed cells. - fn update_cell_states(&mut self) { - let mut new_states = HashMap::new(); - - // Add code cells - for cell in &self.cells { - let existing = self.cell_states.get(&cell.id); - - // Extract status, output, dirty from existing state if it's a code cell - let (status, output, dirty) = if let Some(CellState::Code { - status, - output, - dirty, - .. - }) = existing - { - (*status, output.clone(), *dirty) - } else { - // New cells start pristine: no output, not dirty - (CellStatus::default(), None, false) - }; - - let state = CellState::Code { - id: cell.id, - name: cell.name.clone(), - display_name: cell.display_name.clone(), - source: cell.source_code.clone(), - description: Self::strip_display_name_from_description(&cell.doc_comment), - return_type: cell.return_type.clone(), - dependencies: cell - .dependencies - .iter() - .map(|d| d.param_name.clone()) - .collect(), - status, - output, - dirty, - }; - new_states.insert(cell.id, state); - } - - // Add markdown cells - for md_cell in &self.markdown_cells { - let state = CellState::Markdown { - id: md_cell.id, - content: md_cell.content.clone(), - }; - new_states.insert(md_cell.id, state); - } - - // Add definition cells - for def_cell in &self.definition_cells { - let state = CellState::Definition { - id: def_cell.id, - content: def_cell.content.clone(), - definition_type: def_cell.definition_type, - doc_comment: def_cell.doc_comment.clone(), - }; - new_states.insert(def_cell.id, state); - } - - self.cell_states = new_states; - } - - /// Write virtual notebook.rs file for LSP analysis. - /// This file contains all cell content in source order so rust-analyzer can analyze it. - /// Collect all cells (code and definition) in source order. - /// Returns a vector of (cell_id, start_line, cell_type) tuples sorted by line number. - /// Includes all cell types: code, markdown, and definition cells. - fn collect_cells_in_source_order(&self) -> Vec<(CellId, usize, CellType)> { - let mut all_cells: Vec<(CellId, usize, CellType)> = Vec::new(); - - for cell in &self.cells { - all_cells.push((cell.id, cell.span.start_line, CellType::Code)); - } - - for md_cell in &self.markdown_cells { - all_cells.push((md_cell.id, md_cell.span.start_line, CellType::Markdown)); - } - - for def_cell in &self.definition_cells { - all_cells.push((def_cell.id, def_cell.span.start_line, CellType::Definition)); - } - - all_cells.sort_by_key(|(_, line, _)| *line); - all_cells - } - - fn write_virtual_notebook_file(&self) -> std::io::Result<()> { - use std::fs; - - let dirs = NotebookDirs::from_notebook_path(&self.path) - .map_err(|e| std::io::Error::other(e.to_string()))?; - let universe_src = dirs.build_dir.join("universe").join("src"); - fs::create_dir_all(&universe_src)?; - - let mut lines = Vec::new(); - let all_cells = self.collect_cells_in_source_order(); - - // Build combined source - for (cell_id, _, cell_type) in all_cells { - match cell_type { - CellType::Code => { - if let Some(cell) = self.cells.iter().find(|c| c.id == cell_id) { - lines.push(cell.source_code.clone()); - lines.push(String::new()); // Empty line between cells - } - } - CellType::Definition => { - if let Some(def_cell) = self.definition_cells.iter().find(|c| c.id == cell_id) { - lines.push(def_cell.content.clone()); - lines.push(String::new()); // Empty line between cells - } - } - _ => {} // Ignore markdown cells - } - } - - let content = lines.join("\n"); - fs::write(universe_src.join("notebook.rs"), content)?; - - Ok(()) - } - - /// Get the full notebook state. - /// Returns a snapshot of the current notebook state for UI rendering. - /// Note: The virtual notebook.rs file for LSP is written during reload(), not here. - pub fn get_state(&self) -> ServerMessage { - // Source order: all cells (code + markdown + definition) in the order they appear in the .rs file - let all_cells = self.collect_cells_in_source_order(); - let source_order: Vec = all_cells.into_iter().map(|(id, _, _)| id).collect(); - - // Execution order: topologically sorted for dependency resolution (code cells only) - let execution_order = match self.graph.topological_order() { - Ok(order) => order, - Err(e) => { - tracing::error!("Failed to compute execution order: {}", e); - Vec::new() - } - }; - - // Find workspace root by walking up from notebook path to find Cargo.toml - let (workspace_root, cargo_toml_path) = find_workspace_root(&self.path); - - ServerMessage::NotebookState { - path: self.path.display().to_string(), - cells: self.cell_states.values().cloned().collect(), - source_order, - execution_order, - workspace_root: workspace_root.map(|p| p.display().to_string()), - cargo_toml_path: cargo_toml_path.map(|p| p.display().to_string()), - } - } - - /// Store a pending edit from the editor (not yet saved to disk). - /// - /// The edit will be saved to disk when the cell is executed. - pub fn store_pending_edit(&mut self, cell_id: CellId, source: String) { - self.pending_edits.insert(cell_id, source); - } - - /// Execute a specific cell. - /// - /// Uses process isolation - the cell runs in a worker process that can - /// be killed immediately for interruption. - pub async fn execute_cell(&mut self, cell_id: CellId) -> ServerResult<()> { - // Get cell name before potential reload (IDs change after reload!) - let cell_name = self - .get_cell(cell_id) - .map(|c| c.name.clone()) - .ok_or(ServerError::CellNotFound(cell_id))?; - - // Save pending edit to disk before executing - if let Some(new_source) = self.pending_edits.remove(&cell_id) { - self.edit_cell(cell_id, new_source)?; - } - if self.executing { - return Err(ServerError::ExecutionInProgress); - } - - // After reload(), cell IDs change! Find cell by name instead - let cell = self - .cells - .iter() - .find(|c| c.name == cell_name) - .ok_or(ServerError::CellNotFound(cell_id))? - .clone(); - - let cell_id = cell.id; // Use the NEW ID after reload - - self.executing = true; - - // Reset interrupted flag at the start of each execution - self.interrupted.store(false, Ordering::SeqCst); - - // Check if all dependencies have outputs available - let missing_deps: Vec<&str> = cell - .dependencies - .iter() - .filter(|dep| { - let producer = self.cells.iter().find(|c| c.name == dep.param_name); - match producer { - Some(c) => !self.cell_outputs.contains_key(&c.id), - None => true, - } - }) - .map(|d| d.param_name.as_str()) - .collect(); - - if !missing_deps.is_empty() { - self.set_cell_status(cell_id, CellStatus::Error); - self.broadcast(ServerMessage::CellError { - cell_id, - error: format!( - "Missing dependencies: {}. Run dependent cells first.", - missing_deps.join(", ") - ), - location: None, - }); - self.executing = false; - return Ok(()); - } - - // Compile - self.set_cell_status(cell_id, CellStatus::Compiling); - - let mut compiler = CellCompiler::new(self.config.clone(), self.toolchain.clone()); - if let Some(ref up) = self.universe_path { - compiler = compiler.with_universe(up.clone()); - } - - let result = compiler.compile(&cell, self.deps_hash); - - match result { - CompilationResult::Success(compiled) | CompilationResult::Cached(compiled) => { - // Execute - self.set_cell_status(cell_id, CellStatus::Running); - self.broadcast(ServerMessage::CellStarted { cell_id }); - - let start = Instant::now(); - - // Register the compiled cell with the executor - self.executor - .register_cell(compiled, cell.dependencies.len()); - - // Gather dependency outputs in the order the cell expects them - let inputs: Vec> = cell - .dependencies - .iter() - .filter_map(|dep| { - self.cells - .iter() - .find(|c| c.name == dep.param_name) - .and_then(|c| self.cell_outputs.get(&c.id).cloned()) - }) - .collect(); - - // Get ALL widget values from all cells (widgets can be in any cell) - let widget_values = self.get_all_widget_values(); - let widget_values_json = if widget_values.is_empty() { - Vec::new() - } else { - serde_json::to_vec(&widget_values).unwrap_or_default() - }; - - // Execute the cell in an isolated worker process with widget values - let exec_result = - self.executor - .execute_cell_with_widgets(cell_id, &inputs, widget_values_json); - - let duration = start.elapsed(); - - match exec_result { - Ok((output, widgets_json)) => { - // Check if output changed (for smart dirty marking) - let old_hash = self - .cell_outputs - .get(&cell_id) - .map(|old| Self::output_hash(old)); - let new_hash = Self::output_hash(&output); - let output_changed = old_hash.is_none_or(|h| h != new_hash); - - // Store output for dependent cells - let output_arc = Arc::new(output); - self.cell_outputs.insert(cell_id, output_arc.clone()); - - // Also store in executor state for consistency - self.executor - .state_mut() - .store_output(cell_id, (*output_arc).clone()); - - // Parse and store widget definitions - let widgets: Vec = if widgets_json.is_empty() { - Vec::new() - } else { - serde_json::from_slice(&widgets_json).unwrap_or_default() - }; - self.store_widget_defs(cell_id, widgets.clone()); - - let cell_output = CellOutput { - text: output_arc.display_text().map(|s| s.to_string()), - html: None, - image: None, - json: None, - widgets, - }; - - // Add to history - self.add_to_history(cell_id, output_arc.clone(), cell_output.clone()); - - if let Some(state) = self.cell_states.get_mut(&cell_id) { - state.set_status(CellStatus::Success); - state.set_output(Some(cell_output.clone())); - state.set_dirty(false); - } - - // Mark dependents dirty if output changed - if output_changed { - let dirty_cells = self.mark_dependents_dirty_and_get(cell_id); - for dirty_id in dirty_cells { - self.broadcast(ServerMessage::CellDirty { cell_id: dirty_id }); - } - } - - self.broadcast(ServerMessage::CellCompleted { - cell_id, - duration_ms: duration.as_millis() as u64, - output: Some(cell_output), - }); - } - Err(e) => { - // Check if this was an abort or user-initiated interrupt - let was_interrupted = self.interrupted.swap(false, Ordering::SeqCst); - if matches!(e, venus_core::Error::Aborted) || was_interrupted { - // Send friendly "interrupted" message instead of error - self.set_cell_status(cell_id, CellStatus::Idle); - self.broadcast(ServerMessage::ExecutionAborted { - cell_id: Some(cell_id), - }); - } else { - self.set_cell_status(cell_id, CellStatus::Error); - self.broadcast(ServerMessage::CellError { - cell_id, - error: e.to_string(), - location: None, - }); - } - } - } - } - CompilationResult::Failed { errors, .. } => { - self.set_cell_status(cell_id, CellStatus::Error); - - let compile_errors = errors - .iter() - .map(|e| crate::protocol::CompileErrorInfo { - message: e.message.clone(), - code: e.code.clone(), - location: e.spans.first().map(|s| crate::protocol::SourceLocation { - line: s.location.line as u32, - column: s.location.column as u32, - end_line: s.end_location.as_ref().map(|l| l.line as u32), - end_column: s.end_location.as_ref().map(|l| l.column as u32), - }), - rendered: e.rendered.clone(), - }) - .collect(); - - self.broadcast(ServerMessage::CompileError { - cell_id, - errors: compile_errors, - }); - } - } - - self.executing = false; - Ok(()) - } - - /// Execute all cells in order. - /// - /// If `execution_timeout` is set, kills the worker process after that duration. - /// Unlike cooperative cancellation, this immediately terminates the cell. - pub async fn execute_all(&mut self) -> ServerResult<()> { - let order = self.graph.topological_order()?; - let start = Instant::now(); - let timeout = self.execution_timeout; - - for cell_id in order { - // Check timeout before each cell - if timeout.is_some_and(|max_duration| start.elapsed() > max_duration) { - self.executor.abort(); - self.broadcast(ServerMessage::ExecutionAborted { - cell_id: Some(cell_id), - }); - return Err(ServerError::ExecutionTimeout); - } - - self.execute_cell(cell_id).await?; - } - Ok(()) - } - - /// Mark a cell as dirty (needs re-execution). - /// - /// Only marks cells as dirty if they have existing output (data). - /// Cells without output remain pristine (no border). - pub fn mark_dirty(&mut self, cell_id: CellId) { - // Mark the edited cell as dirty only if it has output - if self.cell_outputs.contains_key(&cell_id) - && let Some(state) = self.cell_states.get_mut(&cell_id) - { - state.set_dirty(true); - } - - // Also mark dependents as dirty (only those with output) - let dependents = self.graph.invalidated_cells(cell_id); - for dep_id in dependents { - if self.cell_outputs.contains_key(&dep_id) - && let Some(state) = self.cell_states.get_mut(&dep_id) - { - state.set_dirty(true); - } - } - } - - /// Check if execution is in progress. - pub fn is_executing(&self) -> bool { - self.executing - } - - /// Abort the current execution immediately. - /// - /// Unlike cooperative cancellation, this **kills the worker process**, - /// providing true interruption even for long-running computations. - /// Returns `true` if there was an execution in progress to abort. - pub fn abort(&mut self) -> bool { - if self.executing { - // Kill the worker process - this is immediate - self.executor.abort(); - self.broadcast(ServerMessage::ExecutionAborted { cell_id: None }); - self.executing = false; - true - } else { - false - } - } - - /// Set the execution timeout for execute_all. - /// - /// When set, execute_all will kill the worker process after this duration, - /// providing immediate interruption of even long-running cells. - pub fn set_execution_timeout(&mut self, timeout: Option) { - self.execution_timeout = timeout; - } - - /// Get the current execution timeout. - pub fn execution_timeout(&self) -> Option { - self.execution_timeout - } - - /// Set the interrupted flag. - /// - /// When true, execution errors will be reported as "interrupted" - /// rather than as failures, showing a friendly message to users. - pub fn set_interrupted(&mut self, value: bool) { - self.interrupted.store(value, Ordering::SeqCst); - } - - /// Get a kill handle for the executor. - /// - /// This handle can be used from another task to kill the current execution - /// without needing to acquire the session lock. - pub fn get_kill_handle(&self) -> Option { - self.executor.get_kill_handle() - } - - /// Restart the kernel: kill WorkerPool, spin up new one, clear memory state, preserve source. - /// - /// This clears all execution state including: - /// - Cell outputs and output history - /// - Widget values - /// - Cached serialized outputs - /// - Cell execution status (all cells reset to Idle) - /// - /// Source code and cell definitions are preserved. - pub fn restart_kernel(&mut self) -> ServerResult<()> { - // Abort any running execution first - if self.executing { - self.abort(); - } - - // Reload notebook from disk (picks up any file changes) - self.reload()?; - - // Shutdown old executor and worker pool - self.executor.shutdown(); - - // Reconstruct state directory path - let dirs = NotebookDirs::from_notebook_path(&self.path)?; - - // Create new ProcessExecutor with warm worker pool - self.executor = ProcessExecutor::new(&dirs.state_dir)?; - - // Clear all execution state - self.cell_outputs.clear(); - self.widget_values.clear(); - self.widget_defs.clear(); - self.cell_output_history.clear(); - self.cell_history_index.clear(); - - // Reset all cell states to Idle and clear outputs - for state in self.cell_states.values_mut() { - state.set_status(CellStatus::Idle); - state.clear_output(); - state.set_dirty(false); - } - - // Broadcast kernel restarted message - self.broadcast(ServerMessage::KernelRestarted { error: None }); - - // Send updated state to all clients - let state_msg = self.get_state(); - self.broadcast(state_msg); - - Ok(()) - } - - /// Clear all cell outputs without restarting the kernel. - /// - /// This clears the display outputs but preserves: - /// - Worker pool and execution state - /// - Widget values - /// - Cell source code - /// - /// All cells are reset to pristine state (no output, not dirty). - pub fn clear_outputs(&mut self) { - // Clear outputs from cell states - back to pristine (not dirty) - for state in self.cell_states.values_mut() { - state.clear_output(); - state.set_dirty(false); // Pristine - no data, no dirty - state.set_status(CellStatus::Idle); - } - - // Clear cached outputs - self.cell_outputs.clear(); - let _ = self.executor.state_mut().clear(); - - // Clear output history - self.cell_output_history.clear(); - self.cell_history_index.clear(); - - // Broadcast outputs cleared message - self.broadcast(ServerMessage::OutputsCleared { error: None }); - - // Send updated state to all clients - let state_msg = self.get_state(); - self.broadcast(state_msg); - } - - /// Get IDs of all dirty cells in topological order. - pub fn get_dirty_cell_ids(&self) -> Vec { - let order = match self.graph.topological_order() { - Ok(order) => order, - Err(e) => { - tracing::error!("Failed to compute order for dirty cells: {}", e); - Vec::new() - } - }; - order - .into_iter() - .filter(|id| { - self.cell_states - .get(id) - .is_some_and(|state| state.is_dirty()) - }) - .collect() - } - - /// Update a widget value for a cell. - /// - /// This stores the new value but does NOT trigger re-execution. - /// The user must explicitly run the cell to see the effect. - pub fn update_widget_value(&mut self, cell_id: CellId, widget_id: String, value: WidgetValue) { - self.widget_values - .entry(cell_id) - .or_default() - .insert(widget_id, value); - } - - /// Get widget values for a cell. - pub fn get_widget_values(&self, cell_id: CellId) -> HashMap { - self.widget_values - .get(&cell_id) - .cloned() - .unwrap_or_default() - } - - /// Get ALL widget values from all cells, flattened into a single map. - /// Widget IDs should be unique across the notebook. - pub fn get_all_widget_values(&self) -> HashMap { - let mut all_values = HashMap::new(); - for cell_widgets in self.widget_values.values() { - for (widget_id, value) in cell_widgets { - all_values.insert(widget_id.clone(), value.clone()); - } - } - all_values - } - - /// Get widget definitions for a cell. - pub fn get_widget_defs(&self, cell_id: CellId) -> Vec { - self.widget_defs.get(&cell_id).cloned().unwrap_or_default() - } - - /// Store widget definitions from cell execution. - fn store_widget_defs(&mut self, cell_id: CellId, widgets: Vec) { - if widgets.is_empty() { - self.widget_defs.remove(&cell_id); - } else { - self.widget_defs.insert(cell_id, widgets); - } - } - - /// Add an execution result to history. - fn add_to_history( - &mut self, - cell_id: CellId, - serialized: Arc, - display: CellOutput, - ) { - use std::time::{SystemTime, UNIX_EPOCH}; - - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - - let entry = OutputHistoryEntry { - serialized, - display, - timestamp, - }; - - let history = self.cell_output_history.entry(cell_id).or_default(); - history.push(entry); - - // Trim if too long - while history.len() > MAX_HISTORY_PER_CELL { - history.remove(0); - } - - // Set current index to the latest entry - self.cell_history_index.insert(cell_id, history.len() - 1); - } - - /// Select a history entry for a cell, making it the current output. - /// Returns the display output if successful. - pub fn select_history_entry(&mut self, cell_id: CellId, index: usize) -> Option { - // Clone what we need before doing any mutations (to avoid borrow conflicts) - let (serialized, display) = { - let history = self.cell_output_history.get(&cell_id)?; - let entry = history.get(index)?; - (entry.serialized.clone(), entry.display.clone()) - }; - - // Update the current output for dependent cells - self.cell_outputs.insert(cell_id, serialized.clone()); - self.executor - .state_mut() - .store_output(cell_id, (*serialized).clone()); - - // Update the cell state - if let Some(state) = self.cell_states.get_mut(&cell_id) { - state.set_output(Some(display.clone())); - } - - // Update history index - self.cell_history_index.insert(cell_id, index); - - // Mark dependent cells as dirty - let _ = self.mark_dependents_dirty_and_get(cell_id); - - Some(display) - } - - /// Mark all cells that depend on the given cell as dirty. - /// - /// Only marks cells that have existing output (data). Cells without - /// output remain pristine (no border) since they haven't been executed yet. - /// Returns the list of cells that were marked dirty. - fn mark_dependents_dirty_and_get(&mut self, cell_id: CellId) -> Vec { - // Use the graph's invalidated_cells which returns all dependents - let dependents = self.graph.invalidated_cells(cell_id); - let mut dirty_cells = Vec::new(); - - // Skip the first one (the changed cell itself) and mark the rest as dirty - // BUT only if they have output (data) - pristine cells stay pristine - for dep_id in dependents.into_iter().skip(1) { - // Only mark dirty if cell has output (has been executed before) - if self.cell_outputs.contains_key(&dep_id) - && let Some(state) = self.cell_states.get_mut(&dep_id) - { - state.set_dirty(true); - dirty_cells.push(dep_id); - } - } - dirty_cells - } - - /// Compute a hash of output bytes for change detection. - fn output_hash(output: &BoxedOutput) -> u64 { - let mut hasher = rustc_hash::FxHasher::default(); - output.bytes().hash(&mut hasher); - hasher.finish() - } - - /// Get history count for a cell. - pub fn get_history_count(&self, cell_id: CellId) -> usize { - self.cell_output_history - .get(&cell_id) - .map(|h| h.len()) - .unwrap_or(0) - } - - /// Get current history index for a cell. - pub fn get_history_index(&self, cell_id: CellId) -> usize { - self.cell_history_index.get(&cell_id).copied().unwrap_or(0) - } - - /// Get reference to cell states. - pub fn cell_states(&self) -> &HashMap { - &self.cell_states - } - - /// Insert a new cell after the specified cell. - /// - /// Modifies the source file and triggers a reload. - /// Returns the name of the newly created cell. - pub fn insert_cell(&mut self, after_cell_id: Option) -> ServerResult { - // Convert CellId to cell name if provided - let after_name = after_cell_id.and_then(|id| { - self.cells - .iter() - .find(|c| c.id == id) - .map(|c| c.name.clone()) - }); - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - let new_name = editor.insert_cell(after_name.as_deref())?; - editor.save()?; - - // Record for undo (with position for redo) - self.undo_manager.record(UndoableOperation::InsertCell { - cell_name: new_name.clone(), - after_cell_name: after_name, - }); - - // File watcher will trigger reload, but we can also reload now - // to ensure immediate consistency - self.reload()?; - - Ok(new_name) - } - - /// Delete a cell from the notebook. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn delete_cell(&mut self, cell_id: CellId) -> ServerResult<()> { - // Find the cell name - let cell_name = self - .cells - .iter() - .find(|c| c.id == cell_id) - .map(|c| c.name.clone()) - .ok_or(ServerError::CellNotFound(cell_id))?; - - // Check if any other cells depend on this cell - let dependents: Vec = self - .cells - .iter() - .filter(|c| c.id != cell_id) // Don't check self - .filter(|c| c.dependencies.iter().any(|dep| dep.param_name == cell_name)) - .map(|c| c.name.clone()) - .collect(); - - if !dependents.is_empty() { - return Err(ServerError::InvalidOperation(format!( - "Cannot delete cell '{}' because it is used by: {}", - cell_name, - dependents.join(", ") - ))); - } - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - - // Capture source and position before deletion (for undo) - let source = editor.get_cell_source(&cell_name)?; - let after_cell_name = editor.get_previous_cell_name(&cell_name)?; - - editor.delete_cell(&cell_name)?; - editor.save()?; - - // Record for undo - self.undo_manager.record(UndoableOperation::DeleteCell { - cell_name: cell_name.clone(), - source, - after_cell_name, - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Duplicate a cell in the notebook. - /// - /// Creates a copy of the cell with a unique name. - /// Returns the name of the new cell. - pub fn duplicate_cell(&mut self, cell_id: CellId) -> ServerResult { - // Find the cell name - let cell_name = self - .cells - .iter() - .find(|c| c.id == cell_id) - .map(|c| c.name.clone()) - .ok_or(ServerError::CellNotFound(cell_id))?; - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - let new_name = editor.duplicate_cell(&cell_name)?; - editor.save()?; - - // Record for undo - self.undo_manager.record(UndoableOperation::DuplicateCell { - original_cell_name: cell_name, - new_cell_name: new_name.clone(), - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(new_name) - } - - /// Move a cell up or down in the notebook. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn move_cell(&mut self, cell_id: CellId, direction: MoveDirection) -> ServerResult<()> { - // Find the cell name - let cell_name = self - .cells - .iter() - .find(|c| c.id == cell_id) - .map(|c| c.name.clone()) - .ok_or(ServerError::CellNotFound(cell_id))?; - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - editor.move_cell(&cell_name, direction)?; - editor.save()?; - - // Record for undo - self.undo_manager.record(UndoableOperation::MoveCell { - cell_name, - direction, - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Edit a code cell's source. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn edit_cell(&mut self, cell_id: CellId, new_source: String) -> ServerResult<()> { - // Find the cell - let cell = self - .cells - .iter() - .find(|c| c.id == cell_id) - .ok_or(ServerError::CellNotFound(cell_id))?; - - let cell_name = cell.name.clone(); - let old_source = cell.source_code.clone(); - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - - // Reconstruct complete cell (doc comments + #[venus::cell] + function) and get FRESH line numbers - let (reconstructed, start_line, end_line) = - editor.reconstruct_and_get_span(&cell_name, &new_source)?; - - tracing::info!( - "Editing cell '{}' lines {}-{}, reconstructed length: {}", - cell_name, - start_line, - end_line, - reconstructed.len() - ); - editor.edit_raw_code(start_line, end_line, &reconstructed)?; - editor.save()?; - - // Record for undo - self.undo_manager.record(UndoableOperation::EditCell { - cell_id, - start_line, - end_line, - old_source, - new_source: new_source.clone(), - }); - - // Reload to update in-memory state - // Save outputs by name BEFORE reload (IDs will change) - let outputs_by_name: HashMap> = self - .cells - .iter() - .filter_map(|c| { - self.cell_outputs - .get(&c.id) - .map(|o| (c.name.clone(), o.clone())) - }) - .collect(); - - self.reload()?; - - // Restore outputs with NEW IDs (except for the edited cell) - self.cell_outputs.clear(); - for cell in &self.cells { - if cell.name != cell_name - && let Some(output) = outputs_by_name.get(&cell.name) - { - self.cell_outputs.insert(cell.id, output.clone()); - } - } - - Ok(()) - } - - /// Rename a cell's display name. - /// - /// Updates the cell's doc comment with the new display name and reloads the notebook. - pub fn rename_cell(&mut self, cell_id: CellId, new_display_name: String) -> ServerResult<()> { - // Find the cell name and current display name - let (cell_name, old_display_name) = self - .cells - .iter() - .find(|c| c.id == cell_id) - .map(|c| (c.name.clone(), c.display_name.clone())) - .ok_or(ServerError::CellNotFound(cell_id))?; - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - editor.rename_cell(&cell_name, &new_display_name)?; - editor.save()?; - - // Record for undo - self.undo_manager.record(UndoableOperation::RenameCell { - cell_name, - old_display_name, - new_display_name, - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Insert a new markdown cell. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn insert_markdown_cell( - &mut self, - content: String, - after_cell_id: Option, - ) -> ServerResult<()> { - // Convert cell ID to line number if provided - let after_line = after_cell_id.and_then(|id| { - // Try to find in code cells - self.cells - .iter() - .find(|c| c.id == id) - .map(|c| c.span.end_line) - .or_else(|| { - // Try to find in markdown cells - self.markdown_cells - .iter() - .find(|m| m.id == id) - .map(|m| m.span.end_line) - }) - }); - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - editor.insert_markdown_cell(&content, after_line)?; - - // Get the line range of the newly inserted cell (approximate) - let start_line = after_line.map(|l| l + 1).unwrap_or(0); - let line_count = content.lines().count(); - let end_line = start_line + line_count; - - editor.save()?; - - // Record for undo - self.undo_manager - .record(UndoableOperation::InsertMarkdownCell { - start_line, - end_line, - content: content.clone(), - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Edit a markdown cell's content. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn edit_markdown_cell(&mut self, cell_id: CellId, new_content: String) -> ServerResult<()> { - // Find the markdown cell - let md_cell = self - .markdown_cells - .iter() - .find(|m| m.id == cell_id) - .ok_or(ServerError::CellNotFound(cell_id))?; - - let start_line = md_cell.span.start_line; - let end_line = md_cell.span.end_line; - let old_content = md_cell.content.clone(); - let is_module_doc = md_cell.is_module_doc; - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - editor.edit_markdown_cell(start_line, end_line, &new_content, is_module_doc)?; - editor.save()?; - - // Record for undo - self.undo_manager - .record(UndoableOperation::EditMarkdownCell { - start_line, - end_line, - old_content, - new_content, - is_module_doc, - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Delete a markdown cell. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn delete_markdown_cell(&mut self, cell_id: CellId) -> ServerResult<()> { - // Find the markdown cell - let md_cell = self - .markdown_cells - .iter() - .find(|m| m.id == cell_id) - .ok_or(ServerError::CellNotFound(cell_id))?; - - let start_line = md_cell.span.start_line; - let end_line = md_cell.span.end_line; - let content = md_cell.content.clone(); - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - editor.delete_markdown_cell(start_line, end_line)?; - editor.save()?; - - // Record for undo - self.undo_manager - .record(UndoableOperation::DeleteMarkdownCell { - start_line, - content, - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Move a markdown cell up or down. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn move_markdown_cell( - &mut self, - cell_id: CellId, - direction: MoveDirection, - ) -> ServerResult<()> { - // Find the markdown cell - let md_cell = self - .markdown_cells - .iter() - .find(|m| m.id == cell_id) - .ok_or(ServerError::CellNotFound(cell_id))?; - - let start_line = md_cell.span.start_line; - let end_line = md_cell.span.end_line; - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - editor.move_markdown_cell(start_line, end_line, direction)?; - editor.save()?; - - // Record for undo - self.undo_manager - .record(UndoableOperation::MoveMarkdownCell { - start_line, - end_line, - direction, - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Infer the definition type from content for validation. - /// - /// Provides early error detection when users specify an incorrect definition type. - /// This is a best-effort heuristic based on content analysis. - /// - /// Returns `None` if the type cannot be reliably inferred. - fn infer_definition_type(content: &str) -> Option { - use venus_core::graph::DefinitionType; - - let trimmed = content.trim(); - - // Check for import statements (use declarations) - if trimmed.starts_with("use ") || trimmed.starts_with("pub use ") { - return Some(DefinitionType::Import); - } - - // Check for struct definitions - if trimmed.contains("struct ") { - return Some(DefinitionType::Struct); - } - - // Check for enum definitions - if trimmed.contains("enum ") { - return Some(DefinitionType::Enum); - } - - // Check for type alias (but not inside a function) - if trimmed.contains("type ") && !trimmed.contains("fn ") { - return Some(DefinitionType::TypeAlias); - } - - // Check for function definitions (without #[venus::cell]) - if trimmed.contains("fn ") && !trimmed.contains("#[venus::cell]") { - return Some(DefinitionType::HelperFunction); - } - - None - } - - /// Validate that the declared definition type matches the content. - /// - /// Provides a warning if there's a mismatch, but doesn't fail the operation - /// since the actual parsing will catch any real errors during universe build. - /// - /// Returns `Ok(())` if valid or cannot be validated, `Err` only for clear mismatches. - fn validate_definition_type( - content: &str, - declared_type: venus_core::graph::DefinitionType, - ) -> ServerResult<()> { - if let Some(inferred_type) = Self::infer_definition_type(content) - && std::mem::discriminant(&inferred_type) != std::mem::discriminant(&declared_type) - { - tracing::warn!( - "Definition type mismatch: declared {:?} but content suggests {:?}", - declared_type, - inferred_type - ); - // For now, just warn - don't fail the operation - // The universe build will catch actual syntax errors - } - Ok(()) - } - - /// Insert a new definition cell. - /// - /// Modifies the .rs source file and reloads the notebook. - /// Returns the ID of the newly inserted definition cell. - /// - /// Validates that the definition type matches the content before insertion. - pub fn insert_definition_cell( - &mut self, - content: String, - definition_type: venus_core::graph::DefinitionType, - after_cell_id: Option, - ) -> ServerResult { - // Validate definition type matches content - Self::validate_definition_type(&content, definition_type)?; - // Convert cell ID to line number if provided - let after_line = after_cell_id.and_then(|id| { - // Try to find in code cells - self.cells - .iter() - .find(|c| c.id == id) - .map(|c| c.span.end_line) - .or_else(|| { - // Try to find in markdown cells - self.markdown_cells - .iter() - .find(|m| m.id == id) - .map(|m| m.span.end_line) - }) - .or_else(|| { - // Try to find in definition cells - self.definition_cells - .iter() - .find(|d| d.id == id) - .map(|d| d.span.end_line) - }) - }); - - // Use insert_raw_code which writes raw Rust code without // prefix - let mut editor = SourceEditor::load(&self.path)?; - editor.insert_raw_code(&content, after_line)?; - - let start_line = after_line.map(|l| l + 1).unwrap_or(0); - let line_count = content.lines().count(); - let end_line = start_line + line_count; - - editor.save()?; - - // Record for undo - self.undo_manager - .record(UndoableOperation::InsertDefinitionCell { - start_line, - end_line, - content: content.clone(), - definition_type, - }); - - // Reload to update in-memory state - self.reload()?; - - // Find the newly inserted definition cell (it should be at the expected line) - let new_cell_id = self - .definition_cells - .iter() - .find(|d| d.span.start_line >= start_line && d.span.start_line <= end_line) - .map(|d| d.id) - .ok_or_else(|| { - ServerError::InvalidOperation("Failed to find inserted definition cell".to_string()) - })?; - - Ok(new_cell_id) - } - - /// Edit a definition cell's content. - /// - /// Modifies the .rs source file and reloads the notebook. - /// Returns a list of cells that are now dirty due to the definition change. - pub fn edit_definition_cell( - &mut self, - cell_id: CellId, - new_content: String, - ) -> ServerResult> { - // Find the definition cell - let def_cell = self - .definition_cells - .iter() - .find(|d| d.id == cell_id) - .ok_or(ServerError::CellNotFound(cell_id))?; - - let start_line = def_cell.span.start_line; - let end_line = def_cell.span.end_line; - let old_content = def_cell.content.clone(); - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - // Use edit_raw_code which edits raw Rust code without // prefix - editor.edit_raw_code(start_line, end_line, &new_content)?; - editor.save()?; - - // Record for undo - self.undo_manager - .record(UndoableOperation::EditDefinitionCell { - cell_id, - start_line, - end_line, - old_content, - new_content: new_content.clone(), - }); - - // Reload to update in-memory state (rebuilds universe with new definitions) - self.reload()?; - - // Mark ALL executable cells as dirty (only if they have output - pristine cells stay pristine) - let dirty_cells: Vec = self - .cells - .iter() - .filter(|c| self.cell_outputs.contains_key(&c.id)) // Only cells with output - .map(|c| c.id) - .collect(); - for &cell_id in &dirty_cells { - if let Some(state) = self.cell_states.get_mut(&cell_id) { - state.set_dirty(true); - } - } - - Ok(dirty_cells) - } - - /// Delete a definition cell. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn delete_definition_cell(&mut self, cell_id: CellId) -> ServerResult<()> { - // Find the definition cell - let def_cell = self - .definition_cells - .iter() - .find(|d| d.id == cell_id) - .ok_or(ServerError::CellNotFound(cell_id))?; - - let start_line = def_cell.span.start_line; - let end_line = def_cell.span.end_line; - let content = def_cell.content.clone(); - let definition_type = def_cell.definition_type; - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - editor.delete_markdown_cell(start_line, end_line)?; - editor.save()?; - - // Record for undo - self.undo_manager - .record(UndoableOperation::DeleteDefinitionCell { - start_line, - end_line, - content, - definition_type, - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Move a definition cell up or down. - /// - /// Modifies the .rs source file and reloads the notebook. - pub fn move_definition_cell( - &mut self, - cell_id: CellId, - direction: MoveDirection, - ) -> ServerResult<()> { - // Find the definition cell - let def_cell = self - .definition_cells - .iter() - .find(|d| d.id == cell_id) - .ok_or(ServerError::CellNotFound(cell_id))?; - - let start_line = def_cell.span.start_line; - let end_line = def_cell.span.end_line; - - // Load and edit the source file - let mut editor = SourceEditor::load(&self.path)?; - editor.move_markdown_cell(start_line, end_line, direction)?; - editor.save()?; - - // Record for undo - self.undo_manager - .record(UndoableOperation::MoveDefinitionCell { - start_line, - end_line, - direction, - }); - - // Reload to update in-memory state - self.reload()?; - - Ok(()) - } - - /// Undo the last cell management operation. - /// - /// Returns a description of what was undone, or an error if undo failed. - pub fn undo(&mut self) -> ServerResult { - let operation = self - .undo_manager - .pop_undo() - .ok_or_else(|| ServerError::InvalidOperation("Nothing to undo".to_string()))?; - - let description = operation.undo_description(); - - // Execute the reverse operation - let mut editor = SourceEditor::load(&self.path)?; - - match &operation { - UndoableOperation::InsertCell { cell_name, .. } => { - // Undo insert = delete - editor.delete_cell(cell_name)?; - } - UndoableOperation::DeleteCell { - source, - after_cell_name, - .. - } => { - // Undo delete = restore - editor.restore_cell(source, after_cell_name.as_deref())?; - } - UndoableOperation::DuplicateCell { new_cell_name, .. } => { - // Undo duplicate = delete the new cell - editor.delete_cell(new_cell_name)?; - } - UndoableOperation::MoveCell { - cell_name, - direction, - } => { - // Undo move = move in opposite direction - let reverse_direction = match direction { - MoveDirection::Up => MoveDirection::Down, - MoveDirection::Down => MoveDirection::Up, - }; - editor.move_cell(cell_name, reverse_direction)?; - } - UndoableOperation::RenameCell { - cell_name, - old_display_name, - .. - } => { - // Undo rename = restore old display name - editor.rename_cell(cell_name, old_display_name)?; - } - UndoableOperation::EditCell { - start_line, - end_line, - old_source, - .. - } => { - // Undo edit = restore old source - editor.edit_raw_code(*start_line, *end_line, old_source)?; - } - UndoableOperation::InsertMarkdownCell { - start_line, - end_line, - .. - } => { - // Undo insert markdown = delete it - editor.delete_markdown_cell(*start_line, *end_line)?; - } - UndoableOperation::EditMarkdownCell { - start_line, - end_line, - old_content, - is_module_doc, - .. - } => { - // Undo edit markdown = restore old content - editor.edit_markdown_cell(*start_line, *end_line, old_content, *is_module_doc)?; - } - UndoableOperation::DeleteMarkdownCell { - start_line, - content, - } => { - // Undo delete markdown = restore it - let after_line = if *start_line > 0 { - Some(start_line - 1) - } else { - None - }; - editor.insert_markdown_cell(content, after_line)?; - } - UndoableOperation::MoveMarkdownCell { - start_line, - end_line, - direction, - } => { - // Undo move markdown = move in opposite direction - let reverse_direction = match direction { - MoveDirection::Up => MoveDirection::Down, - MoveDirection::Down => MoveDirection::Up, - }; - editor.move_markdown_cell(*start_line, *end_line, reverse_direction)?; - } - UndoableOperation::InsertDefinitionCell { - start_line, - end_line, - .. - } => { - // Undo insert definition = delete it - editor.delete_markdown_cell(*start_line, *end_line)?; - } - UndoableOperation::EditDefinitionCell { - start_line, - end_line, - old_content, - .. - } => { - // Undo edit definition = restore old content - editor.edit_markdown_cell(*start_line, *end_line, old_content, false)?; - } - UndoableOperation::DeleteDefinitionCell { - start_line, - content, - .. - } => { - // Undo delete definition = restore it - let after_line = if *start_line > 0 { - Some(start_line - 1) - } else { - None - }; - editor.insert_markdown_cell(content, after_line)?; - } - UndoableOperation::MoveDefinitionCell { - start_line, - end_line, - direction, - } => { - // Undo move definition = move in opposite direction - let reverse_direction = match direction { - MoveDirection::Up => MoveDirection::Down, - MoveDirection::Down => MoveDirection::Up, - }; - editor.move_markdown_cell(*start_line, *end_line, reverse_direction)?; - } - } - - editor.save()?; - - // Record for redo - self.undo_manager.record_redo(operation); - - // Reload to update in-memory state - self.reload()?; - - Ok(description) - } - - /// Redo the last undone operation. - /// - /// Returns a description of what was redone, or an error if redo failed. - pub fn redo(&mut self) -> ServerResult { - let operation = self - .undo_manager - .pop_redo() - .ok_or_else(|| ServerError::InvalidOperation("Nothing to redo".to_string()))?; - - let description = operation.description(); - - // Execute the original operation - let mut editor = SourceEditor::load(&self.path)?; - - match &operation { - UndoableOperation::InsertCell { - after_cell_name, .. - } => { - // Re-insert at the original position - let _ = editor.insert_cell(after_cell_name.as_deref())?; - } - UndoableOperation::DeleteCell { cell_name, .. } => { - // Redo delete = delete again - editor.delete_cell(cell_name)?; - } - UndoableOperation::DuplicateCell { - original_cell_name, .. - } => { - // Redo duplicate = duplicate again (new name will be generated) - let _ = editor.duplicate_cell(original_cell_name)?; - } - UndoableOperation::MoveCell { - cell_name, - direction, - } => { - // Redo move = move in same direction - editor.move_cell(cell_name, *direction)?; - } - UndoableOperation::RenameCell { - cell_name, - new_display_name, - .. - } => { - // Redo rename = apply new display name again - editor.rename_cell(cell_name, new_display_name)?; - } - UndoableOperation::EditCell { - start_line, - end_line, - new_source, - .. - } => { - // Redo edit = apply new source again - editor.edit_raw_code(*start_line, *end_line, new_source)?; - } - UndoableOperation::InsertMarkdownCell { - start_line, - content, - .. - } => { - // Redo insert markdown = insert again at original position - let after_line = if *start_line > 0 { - Some(start_line - 1) - } else { - None - }; - editor.insert_markdown_cell(content, after_line)?; - } - UndoableOperation::EditMarkdownCell { - start_line, - end_line, - new_content, - is_module_doc, - .. - } => { - // Redo edit markdown = apply new content again - editor.edit_markdown_cell(*start_line, *end_line, new_content, *is_module_doc)?; - } - UndoableOperation::DeleteMarkdownCell { - start_line, - content, - } => { - // Redo delete markdown = delete again - // We need to find the end line by counting content lines - let line_count = content.lines().count(); - let end_line = start_line + line_count; - editor.delete_markdown_cell(*start_line, end_line)?; - } - UndoableOperation::MoveMarkdownCell { - start_line, - end_line, - direction, - } => { - // Redo move markdown = move in same direction - editor.move_markdown_cell(*start_line, *end_line, *direction)?; - } - UndoableOperation::InsertDefinitionCell { - start_line, - content, - .. - } => { - // Redo insert definition = insert again at original position - let after_line = if *start_line > 0 { - Some(start_line - 1) - } else { - None - }; - editor.insert_markdown_cell(content, after_line)?; - } - UndoableOperation::EditDefinitionCell { - start_line, - end_line, - new_content, - .. - } => { - // Redo edit definition = apply new content again - editor.edit_markdown_cell(*start_line, *end_line, new_content, false)?; - } - UndoableOperation::DeleteDefinitionCell { - start_line, - content, - .. - } => { - // Redo delete definition = delete again - let line_count = content.lines().count(); - let end_line = start_line + line_count; - editor.delete_markdown_cell(*start_line, end_line)?; - } - UndoableOperation::MoveDefinitionCell { - start_line, - end_line, - direction, - } => { - // Redo move definition = move in same direction - editor.move_markdown_cell(*start_line, *end_line, *direction)?; - } - } - - editor.save()?; - - // Record for undo (so we can undo the redo) - self.undo_manager.record(operation); - - // Reload to update in-memory state - self.reload()?; - - Ok(description) - } - - /// Get the current undo/redo state. - pub fn get_undo_redo_state(&self) -> ServerMessage { - ServerMessage::UndoRedoState { - can_undo: self.undo_manager.can_undo(), - can_redo: self.undo_manager.can_redo(), - undo_description: self.undo_manager.undo_description(), - redo_description: self.undo_manager.redo_description(), - } - } - - /// Clear undo/redo history. - /// - /// Called when the file is externally modified. - pub fn clear_undo_history(&mut self) { - self.undo_manager.clear(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_session_creation() { - // This would require a real notebook file, so we just test the types compile - let (tx, _rx) = broadcast::channel::(16); - drop(tx); - } -} diff --git a/crates/venus-server/src/session/code_editing.rs b/crates/venus-server/src/session/code_editing.rs new file mode 100644 index 0000000..384495f --- /dev/null +++ b/crates/venus-server/src/session/code_editing.rs @@ -0,0 +1,248 @@ +use super::NotebookSession; +use crate::error::{ServerError, ServerResult}; +use crate::undo::UndoableOperation; +use std::collections::HashMap; +use std::sync::Arc; +use venus_core::graph::{CellId, MoveDirection, SourceEditor}; +use venus_core::state::BoxedOutput; + +impl NotebookSession { + /// Insert a new cell after the specified cell. + /// + /// Modifies the source file and triggers a reload. + /// Returns the name of the newly created cell. + pub fn insert_cell(&mut self, after_cell_id: Option) -> ServerResult { + // Convert CellId to cell name if provided + let after_name = after_cell_id.and_then(|id| { + self.cells + .iter() + .find(|c| c.id == id) + .map(|c| c.name.clone()) + }); + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + let new_name = editor.insert_cell(after_name.as_deref())?; + editor.save()?; + + // Record for undo (with position for redo) + self.undo_manager.record(UndoableOperation::InsertCell { + cell_name: new_name.clone(), + after_cell_name: after_name, + }); + + // File watcher will trigger reload, but we can also reload now + // to ensure immediate consistency + self.reload()?; + + Ok(new_name) + } + + /// Delete a cell from the notebook. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn delete_cell(&mut self, cell_id: CellId) -> ServerResult<()> { + // Find the cell name + let cell_name = self + .cells + .iter() + .find(|c| c.id == cell_id) + .map(|c| c.name.clone()) + .ok_or(ServerError::CellNotFound(cell_id))?; + + // Check if any other cells depend on this cell + let dependents: Vec = self + .cells + .iter() + .filter(|c| c.id != cell_id) // Don't check self + .filter(|c| c.dependencies.iter().any(|dep| dep.param_name == cell_name)) + .map(|c| c.name.clone()) + .collect(); + + if !dependents.is_empty() { + return Err(ServerError::InvalidOperation(format!( + "Cannot delete cell '{}' because it is used by: {}", + cell_name, + dependents.join(", ") + ))); + } + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + + // Capture source and position before deletion (for undo) + let source = editor.get_cell_source(&cell_name)?; + let after_cell_name = editor.get_previous_cell_name(&cell_name)?; + + editor.delete_cell(&cell_name)?; + editor.save()?; + + // Record for undo + self.undo_manager.record(UndoableOperation::DeleteCell { + cell_name: cell_name.clone(), + source, + after_cell_name, + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } + + /// Duplicate a cell in the notebook. + /// + /// Creates a copy of the cell with a unique name. + /// Returns the name of the new cell. + pub fn duplicate_cell(&mut self, cell_id: CellId) -> ServerResult { + // Find the cell name + let cell_name = self + .cells + .iter() + .find(|c| c.id == cell_id) + .map(|c| c.name.clone()) + .ok_or(ServerError::CellNotFound(cell_id))?; + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + let new_name = editor.duplicate_cell(&cell_name)?; + editor.save()?; + + // Record for undo + self.undo_manager.record(UndoableOperation::DuplicateCell { + original_cell_name: cell_name, + new_cell_name: new_name.clone(), + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(new_name) + } + + /// Move a cell up or down in the notebook. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn move_cell(&mut self, cell_id: CellId, direction: MoveDirection) -> ServerResult<()> { + // Find the cell name + let cell_name = self + .cells + .iter() + .find(|c| c.id == cell_id) + .map(|c| c.name.clone()) + .ok_or(ServerError::CellNotFound(cell_id))?; + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + editor.move_cell(&cell_name, direction)?; + editor.save()?; + + // Record for undo + self.undo_manager.record(UndoableOperation::MoveCell { + cell_name, + direction, + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } + + /// Edit a code cell's source. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn edit_cell(&mut self, cell_id: CellId, new_source: String) -> ServerResult<()> { + // Find the cell + let cell = self + .cells + .iter() + .find(|c| c.id == cell_id) + .ok_or(ServerError::CellNotFound(cell_id))?; + + let cell_name = cell.name.clone(); + let old_source = cell.source_code.clone(); + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + + // Reconstruct complete cell (doc comments + #[venus::cell] + function) and get FRESH line numbers + let (reconstructed, start_line, end_line) = + editor.reconstruct_and_get_span(&cell_name, &new_source)?; + + tracing::info!( + "Editing cell '{}' lines {}-{}, reconstructed length: {}", + cell_name, + start_line, + end_line, + reconstructed.len() + ); + editor.edit_raw_code(start_line, end_line, &reconstructed)?; + editor.save()?; + + // Record for undo + self.undo_manager.record(UndoableOperation::EditCell { + cell_id, + start_line, + end_line, + old_source, + new_source: new_source.clone(), + }); + + // Reload to update in-memory state + // Save outputs by name BEFORE reload (IDs will change) + let outputs_by_name: HashMap> = self + .cells + .iter() + .filter_map(|c| { + self.cell_outputs + .get(&c.id) + .map(|o| (c.name.clone(), o.clone())) + }) + .collect(); + + self.reload()?; + + // Restore outputs with NEW IDs (except for the edited cell) + self.cell_outputs.clear(); + for cell in &self.cells { + if cell.name != cell_name + && let Some(output) = outputs_by_name.get(&cell.name) + { + self.cell_outputs.insert(cell.id, output.clone()); + } + } + + Ok(()) + } + + /// Rename a cell's display name. + /// + /// Updates the cell's doc comment with the new display name and reloads the notebook. + pub fn rename_cell(&mut self, cell_id: CellId, new_display_name: String) -> ServerResult<()> { + // Find the cell name and current display name + let (cell_name, old_display_name) = self + .cells + .iter() + .find(|c| c.id == cell_id) + .map(|c| (c.name.clone(), c.display_name.clone())) + .ok_or(ServerError::CellNotFound(cell_id))?; + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + editor.rename_cell(&cell_name, &new_display_name)?; + editor.save()?; + + // Record for undo + self.undo_manager.record(UndoableOperation::RenameCell { + cell_name, + old_display_name, + new_display_name, + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } +} diff --git a/crates/venus-server/src/session/definition_editing.rs b/crates/venus-server/src/session/definition_editing.rs new file mode 100644 index 0000000..51470e4 --- /dev/null +++ b/crates/venus-server/src/session/definition_editing.rs @@ -0,0 +1,269 @@ +use super::NotebookSession; +use crate::error::{ServerError, ServerResult}; +use crate::undo::UndoableOperation; +use venus_core::graph::{CellId, MoveDirection, SourceEditor}; + +impl NotebookSession { + /// Infer the definition type from content for validation. + /// + /// Provides early error detection when users specify an incorrect definition type. + /// This is a best-effort heuristic based on content analysis. + /// + /// Returns `None` if the type cannot be reliably inferred. + fn infer_definition_type(content: &str) -> Option { + use venus_core::graph::DefinitionType; + + let trimmed = content.trim(); + + // Check for import statements (use declarations) + if trimmed.starts_with("use ") || trimmed.starts_with("pub use ") { + return Some(DefinitionType::Import); + } + + // Check for struct definitions + if trimmed.contains("struct ") { + return Some(DefinitionType::Struct); + } + + // Check for enum definitions + if trimmed.contains("enum ") { + return Some(DefinitionType::Enum); + } + + // Check for type alias (but not inside a function) + if trimmed.contains("type ") && !trimmed.contains("fn ") { + return Some(DefinitionType::TypeAlias); + } + + // Check for function definitions (without #[venus::cell]) + if trimmed.contains("fn ") && !trimmed.contains("#[venus::cell]") { + return Some(DefinitionType::HelperFunction); + } + + None + } + + /// Validate that the declared definition type matches the content. + /// + /// Provides a warning if there's a mismatch, but doesn't fail the operation + /// since the actual parsing will catch any real errors during universe build. + /// + /// Returns `Ok(())` if valid or cannot be validated, `Err` only for clear mismatches. + fn validate_definition_type( + content: &str, + declared_type: venus_core::graph::DefinitionType, + ) -> ServerResult<()> { + if let Some(inferred_type) = Self::infer_definition_type(content) + && std::mem::discriminant(&inferred_type) != std::mem::discriminant(&declared_type) + { + tracing::warn!( + "Definition type mismatch: declared {:?} but content suggests {:?}", + declared_type, + inferred_type + ); + // For now, just warn - don't fail the operation + // The universe build will catch actual syntax errors + } + Ok(()) + } + + /// Insert a new definition cell. + /// + /// Modifies the .rs source file and reloads the notebook. + /// Returns the ID of the newly inserted definition cell. + /// + /// Validates that the definition type matches the content before insertion. + pub fn insert_definition_cell( + &mut self, + content: String, + definition_type: venus_core::graph::DefinitionType, + after_cell_id: Option, + ) -> ServerResult { + // Validate definition type matches content + Self::validate_definition_type(&content, definition_type)?; + // Convert cell ID to line number if provided + let after_line = after_cell_id.and_then(|id| { + // Try to find in code cells + self.cells + .iter() + .find(|c| c.id == id) + .map(|c| c.span.end_line) + .or_else(|| { + // Try to find in markdown cells + self.markdown_cells + .iter() + .find(|m| m.id == id) + .map(|m| m.span.end_line) + }) + .or_else(|| { + // Try to find in definition cells + self.definition_cells + .iter() + .find(|d| d.id == id) + .map(|d| d.span.end_line) + }) + }); + + // Use insert_raw_code which writes raw Rust code without // prefix + let mut editor = SourceEditor::load(&self.path)?; + editor.insert_raw_code(&content, after_line)?; + + let start_line = after_line.map(|l| l + 1).unwrap_or(0); + let line_count = content.lines().count(); + let end_line = start_line + line_count; + + editor.save()?; + + // Record for undo + self.undo_manager + .record(UndoableOperation::InsertDefinitionCell { + start_line, + end_line, + content: content.clone(), + definition_type, + }); + + // Reload to update in-memory state + self.reload()?; + + // Find the newly inserted definition cell (it should be at the expected line) + let new_cell_id = self + .definition_cells + .iter() + .find(|d| d.span.start_line >= start_line && d.span.start_line <= end_line) + .map(|d| d.id) + .ok_or_else(|| { + ServerError::InvalidOperation("Failed to find inserted definition cell".to_string()) + })?; + + Ok(new_cell_id) + } + + /// Edit a definition cell's content. + /// + /// Modifies the .rs source file and reloads the notebook. + /// Returns a list of cells that are now dirty due to the definition change. + pub fn edit_definition_cell( + &mut self, + cell_id: CellId, + new_content: String, + ) -> ServerResult> { + // Find the definition cell + let def_cell = self + .definition_cells + .iter() + .find(|d| d.id == cell_id) + .ok_or(ServerError::CellNotFound(cell_id))?; + + let start_line = def_cell.span.start_line; + let end_line = def_cell.span.end_line; + let old_content = def_cell.content.clone(); + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + // Use edit_raw_code which edits raw Rust code without // prefix + editor.edit_raw_code(start_line, end_line, &new_content)?; + editor.save()?; + + // Record for undo + self.undo_manager + .record(UndoableOperation::EditDefinitionCell { + cell_id, + start_line, + end_line, + old_content, + new_content: new_content.clone(), + }); + + // Reload to update in-memory state (rebuilds universe with new definitions) + self.reload()?; + + // Mark ALL executable cells as dirty (only if they have output - pristine cells stay pristine) + let dirty_cells: Vec = self + .cells + .iter() + .filter(|c| self.cell_outputs.contains_key(&c.id)) // Only cells with output + .map(|c| c.id) + .collect(); + for &cell_id in &dirty_cells { + if let Some(state) = self.cell_states.get_mut(&cell_id) { + state.set_dirty(true); + } + } + + Ok(dirty_cells) + } + + /// Delete a definition cell. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn delete_definition_cell(&mut self, cell_id: CellId) -> ServerResult<()> { + // Find the definition cell + let def_cell = self + .definition_cells + .iter() + .find(|d| d.id == cell_id) + .ok_or(ServerError::CellNotFound(cell_id))?; + + let start_line = def_cell.span.start_line; + let end_line = def_cell.span.end_line; + let content = def_cell.content.clone(); + let definition_type = def_cell.definition_type; + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + editor.delete_markdown_cell(start_line, end_line)?; + editor.save()?; + + // Record for undo + self.undo_manager + .record(UndoableOperation::DeleteDefinitionCell { + start_line, + end_line, + content, + definition_type, + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } + + /// Move a definition cell up or down. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn move_definition_cell( + &mut self, + cell_id: CellId, + direction: MoveDirection, + ) -> ServerResult<()> { + // Find the definition cell + let def_cell = self + .definition_cells + .iter() + .find(|d| d.id == cell_id) + .ok_or(ServerError::CellNotFound(cell_id))?; + + let start_line = def_cell.span.start_line; + let end_line = def_cell.span.end_line; + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + editor.move_markdown_cell(start_line, end_line, direction)?; + editor.save()?; + + // Record for undo + self.undo_manager + .record(UndoableOperation::MoveDefinitionCell { + start_line, + end_line, + direction, + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } +} diff --git a/crates/venus-server/src/session/execution.rs b/crates/venus-server/src/session/execution.rs new file mode 100644 index 0000000..d864b16 --- /dev/null +++ b/crates/venus-server/src/session/execution.rs @@ -0,0 +1,357 @@ +use super::NotebookSession; +use crate::error::{ServerError, ServerResult}; +use crate::protocol::{CellOutput, CellStatus, ServerMessage}; +use std::sync::{Arc, atomic::Ordering}; +use std::time::{Duration, Instant}; +use venus_core::compile::{CellCompiler, CompilationResult}; +use venus_core::execute::{ExecutorKillHandle, ProcessExecutor}; +use venus_core::graph::CellId; +use venus_core::paths::NotebookDirs; +use venus_core::state::BoxedOutput; +use venus_core::widgets::WidgetDef; + +impl NotebookSession { + /// Execute a specific cell. + /// + /// Uses process isolation - the cell runs in a worker process that can + /// be killed immediately for interruption. + pub async fn execute_cell(&mut self, cell_id: CellId) -> ServerResult<()> { + // Get cell name before potential reload (IDs change after reload!) + let cell_name = self + .get_cell(cell_id) + .map(|c| c.name.clone()) + .ok_or(ServerError::CellNotFound(cell_id))?; + + // Save pending edit to disk before executing + if let Some(new_source) = self.pending_edits.remove(&cell_id) { + self.edit_cell(cell_id, new_source)?; + } + if self.executing { + return Err(ServerError::ExecutionInProgress); + } + + // After reload(), cell IDs change! Find cell by name instead + let cell = self + .cells + .iter() + .find(|c| c.name == cell_name) + .ok_or(ServerError::CellNotFound(cell_id))? + .clone(); + + let cell_id = cell.id; // Use the NEW ID after reload + + self.executing = true; + + // Reset interrupted flag at the start of each execution + self.interrupted.store(false, Ordering::SeqCst); + + // Check if all dependencies have outputs available + let missing_deps: Vec<&str> = cell + .dependencies + .iter() + .filter(|dep| { + let producer = self.cells.iter().find(|c| c.name == dep.param_name); + match producer { + Some(c) => !self.cell_outputs.contains_key(&c.id), + None => true, + } + }) + .map(|d| d.param_name.as_str()) + .collect(); + + if !missing_deps.is_empty() { + self.set_cell_status(cell_id, CellStatus::Error); + self.broadcast(ServerMessage::CellError { + cell_id, + error: format!( + "Missing dependencies: {}. Run dependent cells first.", + missing_deps.join(", ") + ), + location: None, + }); + self.executing = false; + return Ok(()); + } + + // Compile + self.set_cell_status(cell_id, CellStatus::Compiling); + + let mut compiler = CellCompiler::new(self.config.clone(), self.toolchain.clone()); + if let Some(ref up) = self.universe_path { + compiler = compiler.with_universe(up.clone()); + } + + let result = compiler.compile(&cell, self.deps_hash); + + match result { + CompilationResult::Success(compiled) | CompilationResult::Cached(compiled) => { + // Execute + self.set_cell_status(cell_id, CellStatus::Running); + self.broadcast(ServerMessage::CellStarted { cell_id }); + + let start = Instant::now(); + + // Register the compiled cell with the executor + self.executor + .register_cell(compiled, cell.dependencies.len()); + + // Gather dependency outputs in the order the cell expects them + let inputs: Vec> = cell + .dependencies + .iter() + .filter_map(|dep| { + self.cells + .iter() + .find(|c| c.name == dep.param_name) + .and_then(|c| self.cell_outputs.get(&c.id).cloned()) + }) + .collect(); + + // Get ALL widget values from all cells (widgets can be in any cell) + let widget_values = self.get_all_widget_values(); + let widget_values_json = if widget_values.is_empty() { + Vec::new() + } else { + serde_json::to_vec(&widget_values).unwrap_or_default() + }; + + // Execute the cell in an isolated worker process with widget values + let exec_result = + self.executor + .execute_cell_with_widgets(cell_id, &inputs, widget_values_json); + + let duration = start.elapsed(); + + match exec_result { + Ok((output, widgets_json)) => { + // Check if output changed (for smart dirty marking) + let old_hash = self + .cell_outputs + .get(&cell_id) + .map(|old| Self::output_hash(old)); + let new_hash = Self::output_hash(&output); + let output_changed = old_hash.is_none_or(|h| h != new_hash); + + // Store output for dependent cells + let output_arc = Arc::new(output); + self.cell_outputs.insert(cell_id, output_arc.clone()); + + // Also store in executor state for consistency + self.executor + .state_mut() + .store_output(cell_id, (*output_arc).clone()); + + // Parse and store widget definitions + let widgets: Vec = if widgets_json.is_empty() { + Vec::new() + } else { + serde_json::from_slice(&widgets_json).unwrap_or_default() + }; + self.store_widget_defs(cell_id, widgets.clone()); + + let cell_output = CellOutput { + text: output_arc.display_text().map(|s| s.to_string()), + html: None, + image: None, + json: None, + widgets, + }; + + // Add to history + self.add_to_history(cell_id, output_arc.clone(), cell_output.clone()); + + if let Some(state) = self.cell_states.get_mut(&cell_id) { + state.set_status(CellStatus::Success); + state.set_output(Some(cell_output.clone())); + state.set_dirty(false); + } + + // Mark dependents dirty if output changed + if output_changed { + let dirty_cells = self.mark_dependents_dirty_and_get(cell_id); + for dirty_id in dirty_cells { + self.broadcast(ServerMessage::CellDirty { cell_id: dirty_id }); + } + } + + self.broadcast(ServerMessage::CellCompleted { + cell_id, + duration_ms: duration.as_millis() as u64, + output: Some(cell_output), + }); + } + Err(e) => { + // Check if this was an abort or user-initiated interrupt + let was_interrupted = self.interrupted.swap(false, Ordering::SeqCst); + if matches!(e, venus_core::Error::Aborted) || was_interrupted { + // Send friendly "interrupted" message instead of error + self.set_cell_status(cell_id, CellStatus::Idle); + self.broadcast(ServerMessage::ExecutionAborted { + cell_id: Some(cell_id), + }); + } else { + self.set_cell_status(cell_id, CellStatus::Error); + self.broadcast(ServerMessage::CellError { + cell_id, + error: e.to_string(), + location: None, + }); + } + } + } + } + CompilationResult::Failed { errors, .. } => { + self.set_cell_status(cell_id, CellStatus::Error); + + let compile_errors = errors + .iter() + .map(|e| crate::protocol::CompileErrorInfo { + message: e.message.clone(), + code: e.code.clone(), + location: e.spans.first().map(|s| crate::protocol::SourceLocation { + line: s.location.line as u32, + column: s.location.column as u32, + end_line: s.end_location.as_ref().map(|l| l.line as u32), + end_column: s.end_location.as_ref().map(|l| l.column as u32), + }), + rendered: e.rendered.clone(), + }) + .collect(); + + self.broadcast(ServerMessage::CompileError { + cell_id, + errors: compile_errors, + }); + } + } + + self.executing = false; + Ok(()) + } + + /// Execute all cells in order. + /// + /// If `execution_timeout` is set, kills the worker process after that duration. + /// Unlike cooperative cancellation, this immediately terminates the cell. + pub async fn execute_all(&mut self) -> ServerResult<()> { + let order = self.graph.topological_order()?; + let start = Instant::now(); + let timeout = self.execution_timeout; + + for cell_id in order { + // Check timeout before each cell + if timeout.is_some_and(|max_duration| start.elapsed() > max_duration) { + self.executor.abort(); + self.broadcast(ServerMessage::ExecutionAborted { + cell_id: Some(cell_id), + }); + return Err(ServerError::ExecutionTimeout); + } + + self.execute_cell(cell_id).await?; + } + Ok(()) + } + + /// Check if execution is in progress. + pub fn is_executing(&self) -> bool { + self.executing + } + + /// Abort the current execution immediately. + /// + /// Unlike cooperative cancellation, this **kills the worker process**, + /// providing true interruption even for long-running computations. + /// Returns `true` if there was an execution in progress to abort. + pub fn abort(&mut self) -> bool { + if self.executing { + // Kill the worker process - this is immediate + self.executor.abort(); + self.broadcast(ServerMessage::ExecutionAborted { cell_id: None }); + self.executing = false; + true + } else { + false + } + } + + /// Set the execution timeout for execute_all. + /// + /// When set, execute_all will kill the worker process after this duration, + /// providing immediate interruption of even long-running cells. + pub fn set_execution_timeout(&mut self, timeout: Option) { + self.execution_timeout = timeout; + } + + /// Get the current execution timeout. + pub fn execution_timeout(&self) -> Option { + self.execution_timeout + } + + /// Set the interrupted flag. + /// + /// When true, execution errors will be reported as "interrupted" + /// rather than as failures, showing a friendly message to users. + pub fn set_interrupted(&mut self, value: bool) { + self.interrupted.store(value, Ordering::SeqCst); + } + + /// Get a kill handle for the executor. + /// + /// This handle can be used from another task to kill the current execution + /// without needing to acquire the session lock. + pub fn get_kill_handle(&self) -> Option { + self.executor.get_kill_handle() + } + + /// Restart the kernel: kill WorkerPool, spin up new one, clear memory state, preserve source. + /// + /// This clears all execution state including: + /// - Cell outputs and output history + /// - Widget values + /// - Cached serialized outputs + /// - Cell execution status (all cells reset to Idle) + /// + /// Source code and cell definitions are preserved. + pub fn restart_kernel(&mut self) -> ServerResult<()> { + // Abort any running execution first + if self.executing { + self.abort(); + } + + // Reload notebook from disk (picks up any file changes) + self.reload()?; + + // Shutdown old executor and worker pool + self.executor.shutdown(); + + // Reconstruct state directory path + let dirs = NotebookDirs::from_notebook_path(&self.path)?; + + // Create new ProcessExecutor with warm worker pool + self.executor = ProcessExecutor::new(&dirs.state_dir)?; + + // Clear all execution state + self.cell_outputs.clear(); + self.widget_values.clear(); + self.widget_defs.clear(); + self.cell_output_history.clear(); + self.cell_history_index.clear(); + + // Reset all cell states to Idle and clear outputs + for state in self.cell_states.values_mut() { + state.set_status(CellStatus::Idle); + state.clear_output(); + state.set_dirty(false); + } + + // Broadcast kernel restarted message + self.broadcast(ServerMessage::KernelRestarted { error: None }); + + // Send updated state to all clients + let state_msg = self.get_state(); + self.broadcast(state_msg); + + Ok(()) + } +} diff --git a/crates/venus-server/src/session/history.rs b/crates/venus-server/src/session/history.rs new file mode 100644 index 0000000..d3dc002 --- /dev/null +++ b/crates/venus-server/src/session/history.rs @@ -0,0 +1,147 @@ +use super::NotebookSession; +use super::model::{MAX_HISTORY_PER_CELL, OutputHistoryEntry}; +use crate::protocol::{CellOutput, CellStatus, ServerMessage}; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; +use venus_core::graph::CellId; +use venus_core::state::BoxedOutput; + +impl NotebookSession { + /// Clear all cell outputs without restarting the kernel. + /// + /// This clears the display outputs but preserves: + /// - Worker pool and execution state + /// - Widget values + /// - Cell source code + /// + /// All cells are reset to pristine state (no output, not dirty). + pub fn clear_outputs(&mut self) { + // Clear outputs from cell states - back to pristine (not dirty) + for state in self.cell_states.values_mut() { + state.clear_output(); + state.set_dirty(false); // Pristine - no data, no dirty + state.set_status(CellStatus::Idle); + } + + // Clear cached outputs + self.cell_outputs.clear(); + let _ = self.executor.state_mut().clear(); + + // Clear output history + self.cell_output_history.clear(); + self.cell_history_index.clear(); + + // Broadcast outputs cleared message + self.broadcast(ServerMessage::OutputsCleared { error: None }); + + // Send updated state to all clients + let state_msg = self.get_state(); + self.broadcast(state_msg); + } + + /// Add an execution result to history. + pub(super) fn add_to_history( + &mut self, + cell_id: CellId, + serialized: Arc, + display: CellOutput, + ) { + use std::time::{SystemTime, UNIX_EPOCH}; + + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + + let entry = OutputHistoryEntry { + serialized, + display, + timestamp, + }; + + let history = self.cell_output_history.entry(cell_id).or_default(); + history.push(entry); + + // Trim if too long + while history.len() > MAX_HISTORY_PER_CELL { + history.remove(0); + } + + // Set current index to the latest entry + self.cell_history_index.insert(cell_id, history.len() - 1); + } + + /// Select a history entry for a cell, making it the current output. + /// Returns the display output if successful. + pub fn select_history_entry(&mut self, cell_id: CellId, index: usize) -> Option { + // Clone what we need before doing any mutations (to avoid borrow conflicts) + let (serialized, display) = { + let history = self.cell_output_history.get(&cell_id)?; + let entry = history.get(index)?; + (entry.serialized.clone(), entry.display.clone()) + }; + + // Update the current output for dependent cells + self.cell_outputs.insert(cell_id, serialized.clone()); + self.executor + .state_mut() + .store_output(cell_id, (*serialized).clone()); + + // Update the cell state + if let Some(state) = self.cell_states.get_mut(&cell_id) { + state.set_output(Some(display.clone())); + } + + // Update history index + self.cell_history_index.insert(cell_id, index); + + // Mark dependent cells as dirty + let _ = self.mark_dependents_dirty_and_get(cell_id); + + Some(display) + } + + /// Mark all cells that depend on the given cell as dirty. + /// + /// Only marks cells that have existing output (data). Cells without + /// output remain pristine (no border) since they haven't been executed yet. + /// Returns the list of cells that were marked dirty. + pub(super) fn mark_dependents_dirty_and_get(&mut self, cell_id: CellId) -> Vec { + // Use the graph's invalidated_cells which returns all dependents + let dependents = self.graph.invalidated_cells(cell_id); + let mut dirty_cells = Vec::new(); + + // Skip the first one (the changed cell itself) and mark the rest as dirty + // BUT only if they have output (data) - pristine cells stay pristine + for dep_id in dependents.into_iter().skip(1) { + // Only mark dirty if cell has output (has been executed before) + if self.cell_outputs.contains_key(&dep_id) + && let Some(state) = self.cell_states.get_mut(&dep_id) + { + state.set_dirty(true); + dirty_cells.push(dep_id); + } + } + dirty_cells + } + + /// Compute a hash of output bytes for change detection. + pub(super) fn output_hash(output: &BoxedOutput) -> u64 { + let mut hasher = rustc_hash::FxHasher::default(); + output.bytes().hash(&mut hasher); + hasher.finish() + } + + /// Get history count for a cell. + pub fn get_history_count(&self, cell_id: CellId) -> usize { + self.cell_output_history + .get(&cell_id) + .map(|h| h.len()) + .unwrap_or(0) + } + + /// Get current history index for a cell. + pub fn get_history_index(&self, cell_id: CellId) -> usize { + self.cell_history_index.get(&cell_id).copied().unwrap_or(0) + } +} diff --git a/crates/venus-server/src/session/lifecycle.rs b/crates/venus-server/src/session/lifecycle.rs new file mode 100644 index 0000000..82dfd9f --- /dev/null +++ b/crates/venus-server/src/session/lifecycle.rs @@ -0,0 +1,169 @@ +use super::NotebookSession; +use super::model::{InterruptFlag, MESSAGE_CHANNEL_CAPACITY}; +use crate::error::{ServerError, ServerResult}; +use crate::protocol::ServerMessage; +use crate::undo::UndoManager; +use std::collections::HashMap; +use std::path::Path; +use tokio::sync::broadcast; +use venus_core::compile::{CompilerConfig, ToolchainManager, UniverseBuilder}; +use venus_core::execute::ProcessExecutor; +use venus_core::graph::{CellId, CellParser, CellType, GraphEngine}; +use venus_core::paths::NotebookDirs; + +impl NotebookSession { + /// Create a new notebook session. + /// + /// Uses process isolation for cell execution, allowing true interruption + /// by killing worker processes. + /// + /// The `interrupted` flag is shared with AppState so the interrupt handler + /// can set it without needing the session lock. + pub fn new( + path: impl AsRef, + interrupted: InterruptFlag, + ) -> ServerResult<(Self, broadcast::Receiver)> { + let path = path.as_ref().canonicalize().map_err(|e| ServerError::Io { + path: path.as_ref().to_path_buf(), + message: e.to_string(), + })?; + + // Set up directories using shared abstraction + let dirs = NotebookDirs::from_notebook_path(&path)?; + + let toolchain = ToolchainManager::new()?; + let config = CompilerConfig::for_notebook(&dirs); + + let (tx, rx) = broadcast::channel(MESSAGE_CHANNEL_CAPACITY); + + // Create process executor with warm worker pool + let executor = ProcessExecutor::new(&dirs.state_dir)?; + + let mut session = Self { + path, + cells: Vec::new(), + markdown_cells: Vec::new(), + definition_cells: Vec::new(), + graph: GraphEngine::new(), + cell_states: HashMap::new(), + toolchain, + config, + universe_path: None, + deps_hash: 0, + tx, + executing: false, + cell_outputs: HashMap::new(), + executor, + execution_timeout: None, + interrupted, + widget_values: HashMap::new(), + widget_defs: HashMap::new(), + cell_output_history: HashMap::new(), + cell_history_index: HashMap::new(), + undo_manager: UndoManager::new(), + pending_edits: HashMap::new(), + }; + + session.reload()?; + + Ok((session, rx)) + } + + /// Reload the notebook from disk. + pub fn reload(&mut self) -> ServerResult<()> { + let source = std::fs::read_to_string(&self.path)?; + + // Parse cells (code, markdown, and definitions) + let mut parser = CellParser::new(); + let parse_result = parser.parse_str(&source, &self.path)?; + self.cells = parse_result.code_cells; + self.markdown_cells = parse_result.markdown_cells; + self.definition_cells = parse_result.definition_cells; + + // Build graph and update code cells with real IDs (parser returns placeholder IDs) + self.graph = GraphEngine::new(); + for cell in &mut self.cells { + let real_id = self.graph.add_cell(cell.clone()); + cell.id = real_id; + } + self.graph.resolve_dependencies()?; + + // Assign unique IDs to markdown cells (they don't participate in the dependency graph) + let mut next_id = + if let Some(max_code_id) = self.cells.iter().map(|c| c.id.as_usize()).max() { + max_code_id + 1 + } else { + 0 + }; + for md_cell in &mut self.markdown_cells { + md_cell.id = CellId::new(next_id); + next_id += 1; + } + + // Assign unique IDs to definition cells + for def_cell in &mut self.definition_cells { + def_cell.id = CellId::new(next_id); + next_id += 1; + } + + // Write virtual notebook.rs file for LSP analysis BEFORE building universe + // This ensures the file exists when universe is compiled (lib.rs includes `pub mod notebook;`) + if let Err(e) = self.write_virtual_notebook_file() { + tracing::warn!("Failed to write virtual notebook file: {}", e); + } + + // Build universe (always needed for bincode/serde runtime) + let mut universe_builder = + UniverseBuilder::for_notebook(self.config.clone(), self.toolchain.clone(), &self.path)?; + universe_builder.parse_dependencies(&source, &self.definition_cells)?; + + self.universe_path = Some(universe_builder.build()?); + self.deps_hash = universe_builder.deps_hash(); + + // Update cell states + self.update_cell_states(); + + // NOTE: We do NOT broadcast state here because reload() is called by the file watcher + // when the notebook file changes (e.g., editor auto-save). Broadcasting on every file + // change causes the UI to refresh continuously. Instead, each cell operation (insert, + // edit, delete, etc.) explicitly broadcasts state after calling reload(). + + Ok(()) + } + + fn write_virtual_notebook_file(&self) -> std::io::Result<()> { + use std::fs; + + let dirs = NotebookDirs::from_notebook_path(&self.path) + .map_err(|e| std::io::Error::other(e.to_string()))?; + let universe_src = dirs.build_dir.join("universe").join("src"); + fs::create_dir_all(&universe_src)?; + + let mut lines = Vec::new(); + let all_cells = self.collect_cells_in_source_order(); + + // Build combined source + for (cell_id, _, cell_type) in all_cells { + match cell_type { + CellType::Code => { + if let Some(cell) = self.cells.iter().find(|c| c.id == cell_id) { + lines.push(cell.source_code.clone()); + lines.push(String::new()); // Empty line between cells + } + } + CellType::Definition => { + if let Some(def_cell) = self.definition_cells.iter().find(|c| c.id == cell_id) { + lines.push(def_cell.content.clone()); + lines.push(String::new()); // Empty line between cells + } + } + _ => {} // Ignore markdown cells + } + } + + let content = lines.join("\n"); + fs::write(universe_src.join("notebook.rs"), content)?; + + Ok(()) + } +} diff --git a/crates/venus-server/src/session/markdown_editing.rs b/crates/venus-server/src/session/markdown_editing.rs new file mode 100644 index 0000000..96ac12e --- /dev/null +++ b/crates/venus-server/src/session/markdown_editing.rs @@ -0,0 +1,162 @@ +use super::NotebookSession; +use crate::error::{ServerError, ServerResult}; +use crate::undo::UndoableOperation; +use venus_core::graph::{CellId, MoveDirection, SourceEditor}; + +impl NotebookSession { + /// Insert a new markdown cell. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn insert_markdown_cell( + &mut self, + content: String, + after_cell_id: Option, + ) -> ServerResult<()> { + // Convert cell ID to line number if provided + let after_line = after_cell_id.and_then(|id| { + // Try to find in code cells + self.cells + .iter() + .find(|c| c.id == id) + .map(|c| c.span.end_line) + .or_else(|| { + // Try to find in markdown cells + self.markdown_cells + .iter() + .find(|m| m.id == id) + .map(|m| m.span.end_line) + }) + }); + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + editor.insert_markdown_cell(&content, after_line)?; + + // Get the line range of the newly inserted cell (approximate) + let start_line = after_line.map(|l| l + 1).unwrap_or(0); + let line_count = content.lines().count(); + let end_line = start_line + line_count; + + editor.save()?; + + // Record for undo + self.undo_manager + .record(UndoableOperation::InsertMarkdownCell { + start_line, + end_line, + content: content.clone(), + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } + + /// Edit a markdown cell's content. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn edit_markdown_cell(&mut self, cell_id: CellId, new_content: String) -> ServerResult<()> { + // Find the markdown cell + let md_cell = self + .markdown_cells + .iter() + .find(|m| m.id == cell_id) + .ok_or(ServerError::CellNotFound(cell_id))?; + + let start_line = md_cell.span.start_line; + let end_line = md_cell.span.end_line; + let old_content = md_cell.content.clone(); + let is_module_doc = md_cell.is_module_doc; + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + editor.edit_markdown_cell(start_line, end_line, &new_content, is_module_doc)?; + editor.save()?; + + // Record for undo + self.undo_manager + .record(UndoableOperation::EditMarkdownCell { + start_line, + end_line, + old_content, + new_content, + is_module_doc, + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } + + /// Delete a markdown cell. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn delete_markdown_cell(&mut self, cell_id: CellId) -> ServerResult<()> { + // Find the markdown cell + let md_cell = self + .markdown_cells + .iter() + .find(|m| m.id == cell_id) + .ok_or(ServerError::CellNotFound(cell_id))?; + + let start_line = md_cell.span.start_line; + let end_line = md_cell.span.end_line; + let content = md_cell.content.clone(); + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + editor.delete_markdown_cell(start_line, end_line)?; + editor.save()?; + + // Record for undo + self.undo_manager + .record(UndoableOperation::DeleteMarkdownCell { + start_line, + content, + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } + + /// Move a markdown cell up or down. + /// + /// Modifies the .rs source file and reloads the notebook. + pub fn move_markdown_cell( + &mut self, + cell_id: CellId, + direction: MoveDirection, + ) -> ServerResult<()> { + // Find the markdown cell + let md_cell = self + .markdown_cells + .iter() + .find(|m| m.id == cell_id) + .ok_or(ServerError::CellNotFound(cell_id))?; + + let start_line = md_cell.span.start_line; + let end_line = md_cell.span.end_line; + + // Load and edit the source file + let mut editor = SourceEditor::load(&self.path)?; + editor.move_markdown_cell(start_line, end_line, direction)?; + editor.save()?; + + // Record for undo + self.undo_manager + .record(UndoableOperation::MoveMarkdownCell { + start_line, + end_line, + direction, + }); + + // Reload to update in-memory state + self.reload()?; + + Ok(()) + } +} diff --git a/crates/venus-server/src/session/mod.rs b/crates/venus-server/src/session/mod.rs new file mode 100644 index 0000000..f606e65 --- /dev/null +++ b/crates/venus-server/src/session/mod.rs @@ -0,0 +1,17 @@ +//! Notebook session management. + +mod code_editing; +mod definition_editing; +mod execution; +mod history; +mod lifecycle; +mod markdown_editing; +mod model; +mod state; +mod undo_redo; +mod widgets; + +#[cfg(test)] +mod tests; + +pub use model::{InterruptFlag, NotebookSession, OutputHistoryEntry, SessionHandle}; diff --git a/crates/venus-server/src/session/model.rs b/crates/venus-server/src/session/model.rs new file mode 100644 index 0000000..67cd001 --- /dev/null +++ b/crates/venus-server/src/session/model.rs @@ -0,0 +1,115 @@ +use crate::protocol::{CellOutput, CellState, ServerMessage}; +use crate::undo::UndoManager; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, atomic::AtomicBool}; +use std::time::Duration; +use tokio::sync::{RwLock, broadcast}; +use venus_core::compile::{CompilerConfig, ToolchainManager}; +use venus_core::execute::ProcessExecutor; +use venus_core::graph::{CellId, CellInfo, DefinitionCell, GraphEngine, MarkdownCell}; +use venus_core::state::BoxedOutput; +use venus_core::widgets::{WidgetDef, WidgetValue}; + +/// Shared interrupt flag that can be checked without locks. +pub type InterruptFlag = Arc; + +/// Capacity for the broadcast channel. +/// 256 messages should be sufficient for normal notebook operation. +/// If clients fall behind, older messages will be dropped. +pub(super) const MESSAGE_CHANNEL_CAPACITY: usize = 256; + +/// A notebook session. +pub struct NotebookSession { + /// Path to the notebook file. + pub(super) path: PathBuf, + + /// Parsed code cells. + pub(super) cells: Vec, + + /// Parsed markdown cells. + pub(super) markdown_cells: Vec, + + /// Parsed definition cells (imports, types, helpers). + pub(super) definition_cells: Vec, + + /// Dependency graph. + pub(super) graph: GraphEngine, + + /// Cell states for clients (both code and markdown). + pub(super) cell_states: HashMap, + + /// Toolchain manager. + pub(super) toolchain: ToolchainManager, + + /// Compiler configuration. + pub(super) config: CompilerConfig, + + /// Universe path (compiled dependencies). + pub(super) universe_path: Option, + + /// Dependencies hash for cache invalidation. + pub(super) deps_hash: u64, + + /// Broadcast channel for server messages. + pub(super) tx: broadcast::Sender, + + /// Whether an execution is in progress. + pub(super) executing: bool, + + /// Cached cell outputs for dependency passing. + /// Maps cell ID to its serialized output. + pub(super) cell_outputs: HashMap>, + + /// Process-based executor for isolated cell execution. + /// Uses worker processes that can be killed for true interruption. + pub(super) executor: ProcessExecutor, + + /// Optional execution timeout for execute_all. + /// After this duration, the executor kills the current worker. + pub(super) execution_timeout: Option, + + /// Shared flag indicating if current execution was interrupted by user. + /// When true, errors should be reported as "interrupted" not as failures. + /// This is shared with AppState so interrupt handler can set it. + pub(super) interrupted: InterruptFlag, + + /// Widget values per cell. + /// Maps cell ID -> widget ID -> current value. + pub(super) widget_values: HashMap>, + + /// Widget definitions per cell (from last execution). + /// Used to send widget state to newly connected clients. + pub(super) widget_defs: HashMap>, + + /// Execution history per cell. + /// Stores both serialized output (for dependent cells) and display output. + pub(super) cell_output_history: HashMap>, + + /// Current history index per cell. + pub(super) cell_history_index: HashMap, + + /// Undo/redo manager for cell operations. + pub(super) undo_manager: UndoManager, + + /// Pending edits from the editor (not yet saved to disk). + /// These are saved to disk when the cell is executed. + pub(super) pending_edits: HashMap, +} + +/// Maximum number of history entries per cell. +pub(super) const MAX_HISTORY_PER_CELL: usize = 10; + +/// A single history entry for a cell's execution. +#[derive(Clone)] +pub struct OutputHistoryEntry { + /// Serialized output for passing to dependent cells. + pub serialized: Arc, + /// Display output for the frontend. + pub display: CellOutput, + /// Timestamp when this execution completed. + pub timestamp: u64, +} + +/// Thread-safe session handle. +pub type SessionHandle = Arc>; diff --git a/crates/venus-server/src/session/state.rs b/crates/venus-server/src/session/state.rs new file mode 100644 index 0000000..e03d7d3 --- /dev/null +++ b/crates/venus-server/src/session/state.rs @@ -0,0 +1,247 @@ +use super::NotebookSession; +use crate::protocol::{CellState, CellStatus, ServerMessage}; +use std::collections::HashMap; +use std::path::Path; +use tokio::sync::broadcast; +use venus_core::compile::find_notebook_manifest; +use venus_core::graph::{CellId, CellInfo, CellType}; + +impl NotebookSession { + /// Get the notebook path. + pub fn path(&self) -> &Path { + &self.path + } + + /// Subscribe to server messages. + pub fn subscribe(&self) -> broadcast::Receiver { + self.tx.subscribe() + } + + /// Get a cell by ID. + pub(super) fn get_cell(&self, cell_id: CellId) -> Option<&CellInfo> { + self.cells.iter().find(|c| c.id == cell_id) + } + + /// Set the status of a code cell. + pub(super) fn set_cell_status(&mut self, cell_id: CellId, status: CellStatus) { + if let Some(CellState::Code { + status: cell_status, + .. + }) = self.cell_states.get_mut(&cell_id) + { + *cell_status = status; + } + } + + /// Broadcast a server message, ignoring send failures. + pub fn broadcast(&self, msg: ServerMessage) { + let _ = self.tx.send(msg); + } + + /// Strip the first heading from a doc comment (since it's used as display name). + /// + /// If the doc comment starts with `# Heading`, removes that line and returns + /// the rest of the content. Otherwise returns the original content. + fn strip_display_name_from_description(doc_comment: &Option) -> Option { + doc_comment.as_ref().and_then(|doc| { + let lines: Vec<&str> = doc.lines().collect(); + + // Find the first heading line + if let Some(first_line) = lines.first() { + let trimmed = first_line.trim(); + if trimmed.starts_with('#') { + // Skip the heading line and return the rest + let remaining: Vec<&str> = lines.iter().skip(1).copied().collect(); + + // Trim leading empty lines + let trimmed_lines: Vec<&str> = remaining + .iter() + .skip_while(|line| line.trim().is_empty()) + .copied() + .collect(); + + if trimmed_lines.is_empty() { + return None; + } + + return Some(trimmed_lines.join("\n")); + } + } + + // No heading found, return original + Some(doc.clone()) + }) + } + + /// Update cell states from parsed cells. + pub(super) fn update_cell_states(&mut self) { + let mut new_states = HashMap::new(); + + // Add code cells + for cell in &self.cells { + let existing = self.cell_states.get(&cell.id); + + // Extract status, output, dirty from existing state if it's a code cell + let (status, output, dirty) = if let Some(CellState::Code { + status, + output, + dirty, + .. + }) = existing + { + (*status, output.clone(), *dirty) + } else { + // New cells start pristine: no output, not dirty + (CellStatus::default(), None, false) + }; + + let state = CellState::Code { + id: cell.id, + name: cell.name.clone(), + display_name: cell.display_name.clone(), + source: cell.source_code.clone(), + description: Self::strip_display_name_from_description(&cell.doc_comment), + return_type: cell.return_type.clone(), + dependencies: cell + .dependencies + .iter() + .map(|d| d.param_name.clone()) + .collect(), + status, + output, + dirty, + }; + new_states.insert(cell.id, state); + } + + // Add markdown cells + for md_cell in &self.markdown_cells { + let state = CellState::Markdown { + id: md_cell.id, + content: md_cell.content.clone(), + }; + new_states.insert(md_cell.id, state); + } + + // Add definition cells + for def_cell in &self.definition_cells { + let state = CellState::Definition { + id: def_cell.id, + content: def_cell.content.clone(), + definition_type: def_cell.definition_type, + doc_comment: def_cell.doc_comment.clone(), + }; + new_states.insert(def_cell.id, state); + } + + self.cell_states = new_states; + } + + /// Write virtual notebook.rs file for LSP analysis. + /// This file contains all cell content in source order so rust-analyzer can analyze it. + /// Collect all cells (code and definition) in source order. + /// Returns a vector of (cell_id, start_line, cell_type) tuples sorted by line number. + /// Includes all cell types: code, markdown, and definition cells. + pub(super) fn collect_cells_in_source_order(&self) -> Vec<(CellId, usize, CellType)> { + let mut all_cells: Vec<(CellId, usize, CellType)> = Vec::new(); + + for cell in &self.cells { + all_cells.push((cell.id, cell.span.start_line, CellType::Code)); + } + + for md_cell in &self.markdown_cells { + all_cells.push((md_cell.id, md_cell.span.start_line, CellType::Markdown)); + } + + for def_cell in &self.definition_cells { + all_cells.push((def_cell.id, def_cell.span.start_line, CellType::Definition)); + } + + all_cells.sort_by_key(|(_, line, _)| *line); + all_cells + } + + /// Get the full notebook state. + /// Returns a snapshot of the current notebook state for UI rendering. + /// Note: The virtual notebook.rs file for LSP is written during reload(), not here. + pub fn get_state(&self) -> ServerMessage { + // Source order: all cells (code + markdown + definition) in the order they appear in the .rs file + let all_cells = self.collect_cells_in_source_order(); + let source_order: Vec = all_cells.into_iter().map(|(id, _, _)| id).collect(); + + // Execution order: topologically sorted for dependency resolution (code cells only) + let execution_order = match self.graph.topological_order() { + Ok(order) => order, + Err(e) => { + tracing::error!("Failed to compute execution order: {}", e); + Vec::new() + } + }; + + let cargo_toml_path = find_notebook_manifest(&self.path); + let workspace_root = cargo_toml_path.as_deref().and_then(Path::parent); + + ServerMessage::NotebookState { + path: self.path.display().to_string(), + cells: self.cell_states.values().cloned().collect(), + source_order, + execution_order, + workspace_root: workspace_root.map(|p| p.display().to_string()), + cargo_toml_path: cargo_toml_path.map(|p| p.display().to_string()), + } + } + + /// Store a pending edit from the editor (not yet saved to disk). + /// + /// The edit will be saved to disk when the cell is executed. + pub fn store_pending_edit(&mut self, cell_id: CellId, source: String) { + self.pending_edits.insert(cell_id, source); + } + + /// Mark a cell as dirty (needs re-execution). + /// + /// Only marks cells as dirty if they have existing output (data). + /// Cells without output remain pristine (no border). + pub fn mark_dirty(&mut self, cell_id: CellId) { + // Mark the edited cell as dirty only if it has output + if self.cell_outputs.contains_key(&cell_id) + && let Some(state) = self.cell_states.get_mut(&cell_id) + { + state.set_dirty(true); + } + + // Also mark dependents as dirty (only those with output) + let dependents = self.graph.invalidated_cells(cell_id); + for dep_id in dependents { + if self.cell_outputs.contains_key(&dep_id) + && let Some(state) = self.cell_states.get_mut(&dep_id) + { + state.set_dirty(true); + } + } + } + + /// Get IDs of all dirty cells in topological order. + pub fn get_dirty_cell_ids(&self) -> Vec { + let order = match self.graph.topological_order() { + Ok(order) => order, + Err(e) => { + tracing::error!("Failed to compute order for dirty cells: {}", e); + Vec::new() + } + }; + order + .into_iter() + .filter(|id| { + self.cell_states + .get(id) + .is_some_and(|state| state.is_dirty()) + }) + .collect() + } + + /// Get reference to cell states. + pub fn cell_states(&self) -> &HashMap { + &self.cell_states + } +} diff --git a/crates/venus-server/src/session/tests.rs b/crates/venus-server/src/session/tests.rs new file mode 100644 index 0000000..9234d01 --- /dev/null +++ b/crates/venus-server/src/session/tests.rs @@ -0,0 +1,169 @@ +use super::NotebookSession; +use crate::undo::UndoManager; +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, atomic::AtomicBool}; +use tempfile::TempDir; +use venus_core::compile::{CompilerConfig, ToolchainManager}; +use venus_core::execute::ProcessExecutor; +use venus_core::graph::GraphEngine; +use venus_core::paths::NotebookDirs; + +use crate::protocol::ServerMessage; +use tokio::sync::broadcast; + +#[test] +fn test_session_creation() { + // This would require a real notebook file, so we just test the types compile + let (tx, _rx) = broadcast::channel::(16); + drop(tx); +} + +#[test] +fn notebook_state_refreshes_nearest_manifest_paths() -> Result<(), Box> { + let directory = TempDir::new()?; + let root = directory.path().canonicalize()?; + let member = root.join("member"); + std::fs::create_dir_all(member.join("notebooks"))?; + let notebook = member.join("notebooks/runtime.rs"); + std::fs::write(¬ebook, "")?; + let root_manifest = root.join("Cargo.toml"); + std::fs::write(&root_manifest, "[workspace]\n")?; + let session = unloaded_session(¬ebook)?; + let check_paths = |manifest: &Path, directory: &Path| { + let ServerMessage::NotebookState { + cargo_toml_path, + workspace_root, + .. + } = session.get_state() + else { + panic!("Expected NotebookState"); + }; + assert_eq!(cargo_toml_path, Some(manifest.display().to_string())); + assert_eq!(workspace_root, Some(directory.display().to_string())); + }; + + check_paths(&root_manifest, &root); + let member_manifest = member.join("Cargo.toml"); + std::fs::write(&member_manifest, "[package]\nname = \"member\"\n")?; + check_paths(&member_manifest, &member); + std::fs::remove_file(member_manifest)?; + check_paths(&root_manifest, &root); + Ok(()) +} + +fn unloaded_session(notebook: &Path) -> Result> { + let dirs = NotebookDirs::from_notebook_path(notebook)?; + let (tx, _rx) = broadcast::channel(16); + Ok(NotebookSession { + path: notebook.to_path_buf(), + cells: Vec::new(), + markdown_cells: Vec::new(), + definition_cells: Vec::new(), + graph: GraphEngine::new(), + cell_states: HashMap::new(), + toolchain: ToolchainManager::new()?, + config: CompilerConfig::for_notebook(&dirs), + universe_path: None, + deps_hash: 0, + tx, + executing: false, + cell_outputs: HashMap::new(), + executor: ProcessExecutor::new(&dirs.state_dir)?, + execution_timeout: None, + interrupted: Arc::new(AtomicBool::new(false)), + widget_values: HashMap::new(), + widget_defs: HashMap::new(), + cell_output_history: HashMap::new(), + cell_history_index: HashMap::new(), + undo_manager: UndoManager::new(), + pending_edits: HashMap::new(), + }) +} + +#[test] +fn reload_refreshes_generated_dependency_context() -> Result<(), Box> { + let directory = TempDir::new()?; + let root = directory.path().canonicalize()?; + let member = root.join("member"); + std::fs::create_dir_all(member.join("notebooks"))?; + std::fs::create_dir_all(member.join("src"))?; + std::fs::create_dir_all(root.join("internal-helper/src"))?; + std::fs::write( + root.join("Cargo.toml"), + r#"[workspace] +members = ["internal-helper"] +exclude = ["member"] +resolver = "2" + +[workspace.dependencies] +root_alias = { package = "internal-helper", path = "internal-helper" } +"#, + )?; + std::fs::write( + root.join("internal-helper/Cargo.toml"), + r#"[package] +name = "internal-helper" +version = "0.1.0" +edition = "2024" +"#, + )?; + std::fs::write( + root.join("internal-helper/src/lib.rs"), + "pub fn value() -> i32 { 42 }\n", + )?; + std::fs::write(member.join("src/lib.rs"), "")?; + let notebook = member.join("notebooks/runtime.rs"); + std::fs::write(¬ebook, "#[venus::cell]\npub fn value() -> i32 { 42 }\n")?; + let mut session = unloaded_session(¬ebook)?; + let dirs = NotebookDirs::from_notebook_path(¬ebook)?; + let universe_manifest = dirs.build_dir.join("universe/Cargo.toml"); + + session.reload()?; + check_dependency_alias(&universe_manifest, "root_alias", "member_alias")?; + let member_manifest = member.join("Cargo.toml"); + std::fs::write( + &member_manifest, + r#"[package] +name = "notebook-member" +version = "0.1.0" +edition = "2024" + +[workspace] + +[dependencies] +member_alias = { package = "internal-helper", path = "../internal-helper" } +"#, + )?; + session.reload()?; + check_dependency_alias(&universe_manifest, "member_alias", "root_alias")?; + std::fs::remove_file(member_manifest)?; + session.reload()?; + check_dependency_alias(&universe_manifest, "root_alias", "member_alias") +} + +fn check_dependency_alias( + manifest: &Path, + present: &str, + absent: &str, +) -> Result<(), Box> { + let source = std::fs::read_to_string(manifest)?; + let declared = |alias: &str| { + let dependency_table = format!("[dependencies.{alias}]"); + source.lines().any(|line| { + line.trim() == dependency_table + || line + .split_once('=') + .is_some_and(|(name, _)| name.trim() == alias) + }) + }; + assert!( + declared(present), + "Missing {present} in generated manifest:\n{source}" + ); + assert!( + !declared(absent), + "Unexpected {absent} in generated manifest:\n{source}" + ); + Ok(()) +} diff --git a/crates/venus-server/src/session/undo_redo.rs b/crates/venus-server/src/session/undo_redo.rs new file mode 100644 index 0000000..be7126b --- /dev/null +++ b/crates/venus-server/src/session/undo_redo.rs @@ -0,0 +1,329 @@ +use super::NotebookSession; +use crate::error::{ServerError, ServerResult}; +use crate::protocol::ServerMessage; +use crate::undo::UndoableOperation; +use venus_core::graph::{MoveDirection, SourceEditor}; + +impl NotebookSession { + /// Undo the last cell management operation. + /// + /// Returns a description of what was undone, or an error if undo failed. + pub fn undo(&mut self) -> ServerResult { + let operation = self + .undo_manager + .pop_undo() + .ok_or_else(|| ServerError::InvalidOperation("Nothing to undo".to_string()))?; + + let description = operation.undo_description(); + + // Execute the reverse operation + let mut editor = SourceEditor::load(&self.path)?; + + match &operation { + UndoableOperation::InsertCell { cell_name, .. } => { + // Undo insert = delete + editor.delete_cell(cell_name)?; + } + UndoableOperation::DeleteCell { + source, + after_cell_name, + .. + } => { + // Undo delete = restore + editor.restore_cell(source, after_cell_name.as_deref())?; + } + UndoableOperation::DuplicateCell { new_cell_name, .. } => { + // Undo duplicate = delete the new cell + editor.delete_cell(new_cell_name)?; + } + UndoableOperation::MoveCell { + cell_name, + direction, + } => { + // Undo move = move in opposite direction + let reverse_direction = match direction { + MoveDirection::Up => MoveDirection::Down, + MoveDirection::Down => MoveDirection::Up, + }; + editor.move_cell(cell_name, reverse_direction)?; + } + UndoableOperation::RenameCell { + cell_name, + old_display_name, + .. + } => { + // Undo rename = restore old display name + editor.rename_cell(cell_name, old_display_name)?; + } + UndoableOperation::EditCell { + start_line, + end_line, + old_source, + .. + } => { + // Undo edit = restore old source + editor.edit_raw_code(*start_line, *end_line, old_source)?; + } + UndoableOperation::InsertMarkdownCell { + start_line, + end_line, + .. + } => { + // Undo insert markdown = delete it + editor.delete_markdown_cell(*start_line, *end_line)?; + } + UndoableOperation::EditMarkdownCell { + start_line, + end_line, + old_content, + is_module_doc, + .. + } => { + // Undo edit markdown = restore old content + editor.edit_markdown_cell(*start_line, *end_line, old_content, *is_module_doc)?; + } + UndoableOperation::DeleteMarkdownCell { + start_line, + content, + } => { + // Undo delete markdown = restore it + let after_line = if *start_line > 0 { + Some(start_line - 1) + } else { + None + }; + editor.insert_markdown_cell(content, after_line)?; + } + UndoableOperation::MoveMarkdownCell { + start_line, + end_line, + direction, + } => { + // Undo move markdown = move in opposite direction + let reverse_direction = match direction { + MoveDirection::Up => MoveDirection::Down, + MoveDirection::Down => MoveDirection::Up, + }; + editor.move_markdown_cell(*start_line, *end_line, reverse_direction)?; + } + UndoableOperation::InsertDefinitionCell { + start_line, + end_line, + .. + } => { + // Undo insert definition = delete it + editor.delete_markdown_cell(*start_line, *end_line)?; + } + UndoableOperation::EditDefinitionCell { + start_line, + end_line, + old_content, + .. + } => { + // Undo edit definition = restore old content + editor.edit_markdown_cell(*start_line, *end_line, old_content, false)?; + } + UndoableOperation::DeleteDefinitionCell { + start_line, + content, + .. + } => { + // Undo delete definition = restore it + let after_line = if *start_line > 0 { + Some(start_line - 1) + } else { + None + }; + editor.insert_markdown_cell(content, after_line)?; + } + UndoableOperation::MoveDefinitionCell { + start_line, + end_line, + direction, + } => { + // Undo move definition = move in opposite direction + let reverse_direction = match direction { + MoveDirection::Up => MoveDirection::Down, + MoveDirection::Down => MoveDirection::Up, + }; + editor.move_markdown_cell(*start_line, *end_line, reverse_direction)?; + } + } + + editor.save()?; + + // Record for redo + self.undo_manager.record_redo(operation); + + // Reload to update in-memory state + self.reload()?; + + Ok(description) + } + + /// Redo the last undone operation. + /// + /// Returns a description of what was redone, or an error if redo failed. + pub fn redo(&mut self) -> ServerResult { + let operation = self + .undo_manager + .pop_redo() + .ok_or_else(|| ServerError::InvalidOperation("Nothing to redo".to_string()))?; + + let description = operation.description(); + + // Execute the original operation + let mut editor = SourceEditor::load(&self.path)?; + + match &operation { + UndoableOperation::InsertCell { + after_cell_name, .. + } => { + // Re-insert at the original position + let _ = editor.insert_cell(after_cell_name.as_deref())?; + } + UndoableOperation::DeleteCell { cell_name, .. } => { + // Redo delete = delete again + editor.delete_cell(cell_name)?; + } + UndoableOperation::DuplicateCell { + original_cell_name, .. + } => { + // Redo duplicate = duplicate again (new name will be generated) + let _ = editor.duplicate_cell(original_cell_name)?; + } + UndoableOperation::MoveCell { + cell_name, + direction, + } => { + // Redo move = move in same direction + editor.move_cell(cell_name, *direction)?; + } + UndoableOperation::RenameCell { + cell_name, + new_display_name, + .. + } => { + // Redo rename = apply new display name again + editor.rename_cell(cell_name, new_display_name)?; + } + UndoableOperation::EditCell { + start_line, + end_line, + new_source, + .. + } => { + // Redo edit = apply new source again + editor.edit_raw_code(*start_line, *end_line, new_source)?; + } + UndoableOperation::InsertMarkdownCell { + start_line, + content, + .. + } => { + // Redo insert markdown = insert again at original position + let after_line = if *start_line > 0 { + Some(start_line - 1) + } else { + None + }; + editor.insert_markdown_cell(content, after_line)?; + } + UndoableOperation::EditMarkdownCell { + start_line, + end_line, + new_content, + is_module_doc, + .. + } => { + // Redo edit markdown = apply new content again + editor.edit_markdown_cell(*start_line, *end_line, new_content, *is_module_doc)?; + } + UndoableOperation::DeleteMarkdownCell { + start_line, + content, + } => { + // Redo delete markdown = delete again + // We need to find the end line by counting content lines + let line_count = content.lines().count(); + let end_line = start_line + line_count; + editor.delete_markdown_cell(*start_line, end_line)?; + } + UndoableOperation::MoveMarkdownCell { + start_line, + end_line, + direction, + } => { + // Redo move markdown = move in same direction + editor.move_markdown_cell(*start_line, *end_line, *direction)?; + } + UndoableOperation::InsertDefinitionCell { + start_line, + content, + .. + } => { + // Redo insert definition = insert again at original position + let after_line = if *start_line > 0 { + Some(start_line - 1) + } else { + None + }; + editor.insert_markdown_cell(content, after_line)?; + } + UndoableOperation::EditDefinitionCell { + start_line, + end_line, + new_content, + .. + } => { + // Redo edit definition = apply new content again + editor.edit_markdown_cell(*start_line, *end_line, new_content, false)?; + } + UndoableOperation::DeleteDefinitionCell { + start_line, + content, + .. + } => { + // Redo delete definition = delete again + let line_count = content.lines().count(); + let end_line = start_line + line_count; + editor.delete_markdown_cell(*start_line, end_line)?; + } + UndoableOperation::MoveDefinitionCell { + start_line, + end_line, + direction, + } => { + // Redo move definition = move in same direction + editor.move_markdown_cell(*start_line, *end_line, *direction)?; + } + } + + editor.save()?; + + // Record for undo (so we can undo the redo) + self.undo_manager.record(operation); + + // Reload to update in-memory state + self.reload()?; + + Ok(description) + } + + /// Get the current undo/redo state. + pub fn get_undo_redo_state(&self) -> ServerMessage { + ServerMessage::UndoRedoState { + can_undo: self.undo_manager.can_undo(), + can_redo: self.undo_manager.can_redo(), + undo_description: self.undo_manager.undo_description(), + redo_description: self.undo_manager.redo_description(), + } + } + + /// Clear undo/redo history. + /// + /// Called when the file is externally modified. + pub fn clear_undo_history(&mut self) { + self.undo_manager.clear(); + } +} diff --git a/crates/venus-server/src/session/widgets.rs b/crates/venus-server/src/session/widgets.rs new file mode 100644 index 0000000..753c48d --- /dev/null +++ b/crates/venus-server/src/session/widgets.rs @@ -0,0 +1,51 @@ +use super::NotebookSession; +use std::collections::HashMap; +use venus_core::graph::CellId; +use venus_core::widgets::{WidgetDef, WidgetValue}; + +impl NotebookSession { + /// Update a widget value for a cell. + /// + /// This stores the new value but does NOT trigger re-execution. + /// The user must explicitly run the cell to see the effect. + pub fn update_widget_value(&mut self, cell_id: CellId, widget_id: String, value: WidgetValue) { + self.widget_values + .entry(cell_id) + .or_default() + .insert(widget_id, value); + } + + /// Get widget values for a cell. + pub fn get_widget_values(&self, cell_id: CellId) -> HashMap { + self.widget_values + .get(&cell_id) + .cloned() + .unwrap_or_default() + } + + /// Get ALL widget values from all cells, flattened into a single map. + /// Widget IDs should be unique across the notebook. + pub fn get_all_widget_values(&self) -> HashMap { + let mut all_values = HashMap::new(); + for cell_widgets in self.widget_values.values() { + for (widget_id, value) in cell_widgets { + all_values.insert(widget_id.clone(), value.clone()); + } + } + all_values + } + + /// Get widget definitions for a cell. + pub fn get_widget_defs(&self, cell_id: CellId) -> Vec { + self.widget_defs.get(&cell_id).cloned().unwrap_or_default() + } + + /// Store widget definitions from cell execution. + pub(super) fn store_widget_defs(&mut self, cell_id: CellId, widgets: Vec) { + if widgets.is_empty() { + self.widget_defs.remove(&cell_id); + } else { + self.widget_defs.insert(cell_id, widgets); + } + } +} diff --git a/crates/venus/Cargo.toml b/crates/venus/Cargo.toml index 6f139a4..b1299bb 100644 --- a/crates/venus/Cargo.toml +++ b/crates/venus/Cargo.toml @@ -25,6 +25,11 @@ name = "venus-worker" path = "worker/src/main.rs" required-features = ["cli"] +[[test]] +name = "workspace" +path = "cli/tests/workspace.rs" +required-features = ["cli"] + [dependencies] # Core library dependencies (always available) venus-macros.workspace = true diff --git a/crates/venus/cli/src/executor.rs b/crates/venus/cli/src/executor.rs deleted file mode 100644 index 9f1ea1f..0000000 --- a/crates/venus/cli/src/executor.rs +++ /dev/null @@ -1,508 +0,0 @@ -//! Shared notebook execution pipeline for Venus CLI. -//! -//! This module provides a unified execution pipeline used by `run`, `watch`, and `export` commands. -//! It handles the common workflow of parsing, compiling, and executing notebook cells. - -use std::collections::HashMap; -use std::fs; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use venus_core::Error; -use venus_core::compile::{ - CellCompiler, CompilationResult, CompiledCell, CompilerConfig, ToolchainManager, - UniverseBuilder, -}; -use venus_core::execute::{ExecutionCallback, LinearExecutor}; -use venus_core::graph::{CellId, CellInfo, CellParser, GraphEngine}; -use venus_core::paths::NotebookDirs; -use venus_core::state::{BoxedOutput, StateManager}; - -use crate::colors; - -/// Progress callback that prints execution status to the terminal. -pub struct ProgressCallback { - /// Whether to show verbose output. - verbose: bool, -} - -impl ProgressCallback { - /// Create a new progress callback. - pub fn new() -> Self { - Self { verbose: false } - } - - /// Create a verbose progress callback. - #[allow(dead_code)] - pub fn verbose() -> Self { - Self { verbose: true } - } -} - -impl Default for ProgressCallback { - fn default() -> Self { - Self::new() - } -} - -impl ExecutionCallback for ProgressCallback { - fn on_cell_started(&self, _cell_id: CellId, name: &str) { - print!( - "{} ▶ Running{} {}{}... ", - colors::CYAN, - colors::RESET, - colors::BOLD, - name - ); - colors::flush_stdout(); - } - - fn on_cell_completed(&self, _cell_id: CellId, _name: &str) { - println!("{}✓{}", colors::GREEN, colors::RESET); - } - - fn on_cell_error(&self, _cell_id: CellId, _name: &str, error: &Error) { - println!("{}✗{}", colors::RED, colors::RESET); - eprintln!("{} Error:{} {}", colors::RED, colors::RESET, error); - } - - fn on_level_started(&self, level: usize, cell_count: usize) { - if self.verbose && cell_count > 1 { - println!( - "{}Level {}:{} {} cells (parallel)", - colors::DIM, - level, - colors::RESET, - cell_count - ); - } - } - - fn on_level_completed(&self, _level: usize) {} -} - -/// Compiled cell with metadata for execution. -#[derive(Clone)] -pub struct CompiledCellInfo { - /// The compiled cell. - pub compiled: CompiledCell, - /// Number of dependencies. - pub dep_count: usize, - /// Compilation time in milliseconds. - #[allow(dead_code)] - pub compile_time_ms: u64, - /// Whether the cell was cached. - #[allow(dead_code)] - pub cached: bool, -} - -/// Result of cell compilation. -pub struct CompilationInfo { - /// Successfully compiled cells. - pub cells: HashMap, - /// Compilation errors by cell name. - pub errors: Vec<(String, Vec)>, -} - -/// Result of cell execution. -pub struct ExecutionInfo { - /// Cells that were executed (in order). - pub executed_cells: Vec, - /// Execution time. - pub execution_time: Duration, - /// Cell outputs (cell_id -> output). - pub outputs: HashMap>, -} - -/// Notebook executor that manages the full execution pipeline. -pub struct NotebookExecutor { - /// Absolute path to the notebook file. - pub notebook_path: PathBuf, - /// Notebook source code. - #[allow(dead_code)] - pub source: String, - /// Notebook directories. - pub dirs: NotebookDirs, - /// Toolchain manager. - pub toolchain: ToolchainManager, - /// Parsed cells. - pub cells: Vec, - /// Cell name to ID mapping. - pub cell_ids: HashMap, - /// Topological execution order. - pub order: Vec, - /// Dependency map (cell_id -> dependencies). - pub deps: HashMap>, - /// Compiler configuration. - pub config: CompilerConfig, - /// Universe builder (for dependency hash). - pub universe_builder: UniverseBuilder, - /// Path to compiled universe. - pub universe_path: PathBuf, - /// Whether using release mode. - #[allow(dead_code)] - pub release: bool, -} - -impl NotebookExecutor { - /// Create a new notebook executor. - /// - /// This performs the setup phase: parsing, dependency resolution, and universe building. - pub fn new(notebook_path: &str, release: bool) -> anyhow::Result { - let path = Path::new(notebook_path); - if !path.exists() { - anyhow::bail!("Notebook not found: {}", notebook_path); - } - - let source = fs::read_to_string(path)?; - let abs_path = path.canonicalize()?; - let dirs = NotebookDirs::from_notebook_path(&abs_path)?; - - // Initialize toolchain - Self::print_step("Checking toolchain"); - let toolchain = ToolchainManager::new()?; - Self::print_success(None); - - // Parse cells - Self::print_step("Parsing cells"); - let mut parser = CellParser::new(); - let parse_result = parser.parse_file(&abs_path)?; - let cells = parse_result.code_cells; - let definition_cells = parse_result.definition_cells; - Self::print_success(Some(&format!("{} code cells", cells.len()))); - - // Build dependency graph - Self::print_step("Building dependency graph"); - let mut graph = GraphEngine::new(); - let mut cell_ids: HashMap = HashMap::new(); - for cell in &cells { - let real_id = graph.add_cell(cell.clone()); - cell_ids.insert(cell.name.clone(), real_id); - } - graph.resolve_dependencies()?; - let order = graph.topological_order()?; - Self::print_success(None); - - // Build dependency map - let deps: HashMap> = cells - .iter() - .map(|cell| { - let real_id = cell_ids[&cell.name]; - let dep_ids: Vec = cell - .dependencies - .iter() - .filter_map(|dep| cell_ids.get(&dep.param_name).copied()) - .collect(); - (real_id, dep_ids) - }) - .collect(); - - // Build universe - Self::print_step("Building universe"); - let config = if release { - CompilerConfig::for_notebook_release(&dirs) - } else { - CompilerConfig::for_notebook(&dirs) - }; - - let mut universe_builder = UniverseBuilder::new(config.clone(), toolchain.clone(), None); - universe_builder.parse_dependencies(&source, &definition_cells)?; - let universe_path = universe_builder.build()?; - - if universe_builder.dependencies().is_empty() { - Self::print_success(Some("runtime only")); - } else { - Self::print_success(None); - } - - Ok(Self { - notebook_path: abs_path, - source, - dirs, - toolchain, - cells, - cell_ids, - order, - deps, - config, - universe_builder, - universe_path, - release, - }) - } - - /// Get the notebook name (file stem). - pub fn notebook_name(&self) -> String { - self.notebook_path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .to_string() - } - - /// Compile all cells. - pub fn compile(&self) -> anyhow::Result { - println!("\n{}Compiling cells...{}", colors::BOLD, colors::RESET); - - let compiler = CellCompiler::new(self.config.clone(), self.toolchain.clone()) - .with_universe(self.universe_path.clone()); - - let mut compiled_cells = HashMap::new(); - let mut compile_errors = Vec::new(); - - for cell in &self.cells { - print!(" {} {} ... ", colors::DIM, cell.name); - colors::flush_stdout(); - - let real_id = self.cell_ids[&cell.name]; - let deps_hash = self.universe_builder.deps_hash(); - let result = compiler.compile(cell, deps_hash); - - match result { - CompilationResult::Success(mut compiled) => { - let compile_time_ms = compiled.compile_time_ms; - compiled.cell_id = real_id; - println!( - "{}✓{} ({}ms)", - colors::GREEN, - colors::RESET, - compile_time_ms - ); - compiled_cells.insert( - real_id, - CompiledCellInfo { - compiled, - dep_count: cell.dependencies.len(), - compile_time_ms, - cached: false, - }, - ); - } - CompilationResult::Cached(mut compiled) => { - compiled.cell_id = real_id; - println!( - "{}✓{} {}(cached){}", - colors::GREEN, - colors::RESET, - colors::DIM, - colors::RESET - ); - compiled_cells.insert( - real_id, - CompiledCellInfo { - compiled, - dep_count: cell.dependencies.len(), - compile_time_ms: 0, - cached: true, - }, - ); - } - CompilationResult::Failed { cell_id: _, errors } => { - println!("{}✗{}", colors::RED, colors::RESET); - for error in &errors { - if let Some(rendered) = &error.rendered { - eprintln!("{}", rendered); - } else { - eprintln!("{}", error.format_terminal()); - } - } - compile_errors.push((cell.name.clone(), errors)); - } - } - } - - Ok(CompilationInfo { - cells: compiled_cells, - errors: compile_errors, - }) - } - - /// Execute cells with the given compilation info. - /// - /// If `cell_filter` is provided, only execute that cell and its dependencies. - pub fn execute( - &self, - compilation: &CompilationInfo, - cell_filter: Option<&str>, - ) -> anyhow::Result { - if !compilation.errors.is_empty() { - println!( - "\n{}Compilation failed for {} cell(s){}", - colors::RED, - compilation.errors.len(), - colors::RESET - ); - anyhow::bail!("Compilation failed"); - } - - println!("\n{}Executing cells...{}", colors::BOLD, colors::RESET); - - let state = StateManager::new(&self.dirs.state_dir)?; - let mut executor = LinearExecutor::with_state(state); - executor.set_callback(ProgressCallback::new()); - - // Load all compiled cells - for info in compilation.cells.values() { - executor.load_cell(info.compiled.clone(), info.dep_count)?; - } - - // Filter execution order if specific cell requested - let execution_order = self.filter_execution_order(cell_filter)?; - - // Execute - let exec_start = Instant::now(); - executor.execute_in_order(&execution_order, &self.deps)?; - let execution_time = exec_start.elapsed(); - - // Collect outputs - let mut outputs = HashMap::new(); - for &cell_id in &execution_order { - if let Some(output) = executor.state().get_output(cell_id) { - outputs.insert(cell_id, output); - } - } - - Ok(ExecutionInfo { - executed_cells: execution_order, - execution_time, - outputs, - }) - } - - /// Execute cells without a callback (for export mode). - pub fn execute_silent( - &self, - compilation: &CompilationInfo, - cell_filter: Option<&str>, - ) -> anyhow::Result { - if !compilation.errors.is_empty() { - anyhow::bail!("Compilation failed"); - } - - let state = StateManager::new(&self.dirs.state_dir)?; - let mut executor = LinearExecutor::with_state(state); - - // Load all compiled cells - for info in compilation.cells.values() { - executor.load_cell(info.compiled.clone(), info.dep_count)?; - } - - // Filter execution order if specific cell requested - let execution_order = self.filter_execution_order(cell_filter)?; - - // Execute - let exec_start = Instant::now(); - executor.execute_in_order(&execution_order, &self.deps)?; - let execution_time = exec_start.elapsed(); - - // Collect outputs - let mut outputs = HashMap::new(); - for &cell_id in &execution_order { - if let Some(output) = executor.state().get_output(cell_id) { - outputs.insert(cell_id, output); - } - } - - Ok(ExecutionInfo { - executed_cells: execution_order, - execution_time, - outputs, - }) - } - - /// Filter execution order based on cell filter. - fn filter_execution_order(&self, cell_filter: Option<&str>) -> anyhow::Result> { - if let Some(cell_name) = cell_filter { - let target_id = self - .cell_ids - .get(cell_name) - .ok_or_else(|| anyhow::anyhow!("Cell '{}' not found", cell_name))?; - - Ok(self - .order - .iter() - .copied() - .filter(|&id| { - id == *target_id || is_transitive_dependency(id, *target_id, &self.deps) - }) - .collect()) - } else { - Ok(self.order.clone()) - } - } - - /// Get cell info by ID. - pub fn cell_by_id(&self, cell_id: CellId) -> Option<&CellInfo> { - self.cells - .iter() - .find(|c| self.cell_ids[&c.name] == cell_id) - } - - /// Print a setup step. - fn print_step(name: &str) { - print!("{} ◆ {}{} ... ", colors::BLUE, name, colors::RESET); - colors::flush_stdout(); - } - - /// Print success for a step. - fn print_success(extra: Option<&str>) { - match extra { - Some(s) => println!("{}✓{} ({})", colors::GREEN, colors::RESET, s), - None => println!("{}✓{}", colors::GREEN, colors::RESET), - } - } - - /// Print the header for a run. - pub fn print_header(&self, action: &str) { - println!( - "\n{}Venus{} - {} {}{}{}", - colors::BOLD, - colors::RESET, - action, - colors::CYAN, - self.notebook_name(), - colors::RESET - ); - println!("{}", "─".repeat(50)); - } -} - -/// Check if `dep_id` is a transitive dependency of `target_id`. -pub fn is_transitive_dependency( - dep_id: CellId, - target_id: CellId, - deps: &HashMap>, -) -> bool { - let mut visited = std::collections::HashSet::new(); - let mut stack = vec![target_id]; - - while let Some(current) = stack.pop() { - if !visited.insert(current) { - continue; - } - - if let Some(current_deps) = deps.get(¤t) { - if current_deps.contains(&dep_id) { - return true; - } - stack.extend(current_deps.iter().copied()); - } - } - - false -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_progress_callback_creation() { - let callback = ProgressCallback::new(); - assert!(!callback.verbose); - - let verbose = ProgressCallback::verbose(); - assert!(verbose.verbose); - } -} diff --git a/crates/venus/cli/src/executor/compilation.rs b/crates/venus/cli/src/executor/compilation.rs new file mode 100644 index 0000000..4ea79cb --- /dev/null +++ b/crates/venus/cli/src/executor/compilation.rs @@ -0,0 +1,83 @@ +use super::{CompilationInfo, CompiledCellInfo, NotebookExecutor}; +use crate::colors; +use std::collections::HashMap; +use venus_core::compile::{CellCompiler, CompilationResult}; + +impl NotebookExecutor { + /// Compile all cells. + pub fn compile(&self) -> anyhow::Result { + println!("\n{}Compiling cells...{}", colors::BOLD, colors::RESET); + + let compiler = CellCompiler::new(self.config.clone(), self.toolchain.clone()) + .with_universe(self.universe_path.clone()); + + let mut compiled_cells = HashMap::new(); + let mut compile_errors = Vec::new(); + let deps_hash = self.universe_builder.deps_hash(); + + for cell in &self.cells { + print!(" {} {} ... ", colors::DIM, cell.name); + colors::flush_stdout(); + + let real_id = self.cell_ids[&cell.name]; + let result = compiler.compile(cell, deps_hash); + + match result { + CompilationResult::Success(mut compiled) => { + let compile_time_ms = compiled.compile_time_ms; + compiled.cell_id = real_id; + println!( + "{}✓{} ({}ms)", + colors::GREEN, + colors::RESET, + compile_time_ms + ); + compiled_cells.insert( + real_id, + CompiledCellInfo { + compiled, + dep_count: cell.dependencies.len(), + compile_time_ms, + cached: false, + }, + ); + } + CompilationResult::Cached(mut compiled) => { + compiled.cell_id = real_id; + println!( + "{}✓{} {}(cached){}", + colors::GREEN, + colors::RESET, + colors::DIM, + colors::RESET + ); + compiled_cells.insert( + real_id, + CompiledCellInfo { + compiled, + dep_count: cell.dependencies.len(), + compile_time_ms: 0, + cached: true, + }, + ); + } + CompilationResult::Failed { cell_id: _, errors } => { + println!("{}✗{}", colors::RED, colors::RESET); + for error in &errors { + if let Some(rendered) = &error.rendered { + eprintln!("{}", rendered); + } else { + eprintln!("{}", error.format_terminal()); + } + } + compile_errors.push((cell.name.clone(), errors)); + } + } + } + + Ok(CompilationInfo { + cells: compiled_cells, + errors: compile_errors, + }) + } +} diff --git a/crates/venus/cli/src/executor/dependencies.rs b/crates/venus/cli/src/executor/dependencies.rs new file mode 100644 index 0000000..ba25504 --- /dev/null +++ b/crates/venus/cli/src/executor/dependencies.rs @@ -0,0 +1,27 @@ +use std::collections::HashMap; +use venus_core::graph::CellId; + +/// Check if `dep_id` is a transitive dependency of `target_id`. +pub fn is_transitive_dependency( + dep_id: CellId, + target_id: CellId, + deps: &HashMap>, +) -> bool { + let mut visited = std::collections::HashSet::new(); + let mut stack = vec![target_id]; + + while let Some(current) = stack.pop() { + if !visited.insert(current) { + continue; + } + + if let Some(current_deps) = deps.get(¤t) { + if current_deps.contains(&dep_id) { + return true; + } + stack.extend(current_deps.iter().copied()); + } + } + + false +} diff --git a/crates/venus/cli/src/executor/display.rs b/crates/venus/cli/src/executor/display.rs new file mode 100644 index 0000000..eb69424 --- /dev/null +++ b/crates/venus/cli/src/executor/display.rs @@ -0,0 +1,49 @@ +use super::NotebookExecutor; +use crate::colors; +use venus_core::graph::{CellId, CellInfo}; + +impl NotebookExecutor { + /// Get the notebook name (file stem). + pub fn notebook_name(&self) -> String { + self.notebook_path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .to_string() + } + + /// Get cell info by ID. + pub fn cell_by_id(&self, cell_id: CellId) -> Option<&CellInfo> { + self.cells + .iter() + .find(|c| self.cell_ids[&c.name] == cell_id) + } + + /// Print a setup step. + pub(super) fn print_step(name: &str) { + print!("{} ◆ {}{} ... ", colors::BLUE, name, colors::RESET); + colors::flush_stdout(); + } + + /// Print success for a step. + pub(super) fn print_success(extra: Option<&str>) { + match extra { + Some(s) => println!("{}✓{} ({})", colors::GREEN, colors::RESET, s), + None => println!("{}✓{}", colors::GREEN, colors::RESET), + } + } + + /// Print the header for a run. + pub fn print_header(&self, action: &str) { + println!( + "\n{}Venus{} - {} {}{}{}", + colors::BOLD, + colors::RESET, + action, + colors::CYAN, + self.notebook_name(), + colors::RESET + ); + println!("{}", "─".repeat(50)); + } +} diff --git a/crates/venus/cli/src/executor/execution.rs b/crates/venus/cli/src/executor/execution.rs new file mode 100644 index 0000000..5bf8ed6 --- /dev/null +++ b/crates/venus/cli/src/executor/execution.rs @@ -0,0 +1,125 @@ +use super::{ + CompilationInfo, ExecutionInfo, NotebookExecutor, ProgressCallback, is_transitive_dependency, +}; +use crate::colors; +use std::collections::HashMap; +use std::time::Instant; +use venus_core::execute::LinearExecutor; +use venus_core::graph::CellId; +use venus_core::state::StateManager; + +impl NotebookExecutor { + /// Execute cells with the given compilation info. + /// + /// If `cell_filter` is provided, only execute that cell and its dependencies. + pub fn execute( + &self, + compilation: &CompilationInfo, + cell_filter: Option<&str>, + ) -> anyhow::Result { + if !compilation.errors.is_empty() { + println!( + "\n{}Compilation failed for {} cell(s){}", + colors::RED, + compilation.errors.len(), + colors::RESET + ); + anyhow::bail!("Compilation failed"); + } + + println!("\n{}Executing cells...{}", colors::BOLD, colors::RESET); + + let state = StateManager::new(&self.dirs.state_dir)?; + let mut executor = LinearExecutor::with_state(state); + executor.set_callback(ProgressCallback::new()); + + // Load all compiled cells + for info in compilation.cells.values() { + executor.load_cell(info.compiled.clone(), info.dep_count)?; + } + + // Filter execution order if specific cell requested + let execution_order = self.filter_execution_order(cell_filter)?; + + // Execute + let exec_start = Instant::now(); + executor.execute_in_order(&execution_order, &self.deps)?; + let execution_time = exec_start.elapsed(); + + // Collect outputs + let mut outputs = HashMap::new(); + for &cell_id in &execution_order { + if let Some(output) = executor.state().get_output(cell_id) { + outputs.insert(cell_id, output); + } + } + + Ok(ExecutionInfo { + executed_cells: execution_order, + execution_time, + outputs, + }) + } + + /// Execute cells without a callback (for export mode). + pub fn execute_silent( + &self, + compilation: &CompilationInfo, + cell_filter: Option<&str>, + ) -> anyhow::Result { + if !compilation.errors.is_empty() { + anyhow::bail!("Compilation failed"); + } + + let state = StateManager::new(&self.dirs.state_dir)?; + let mut executor = LinearExecutor::with_state(state); + + // Load all compiled cells + for info in compilation.cells.values() { + executor.load_cell(info.compiled.clone(), info.dep_count)?; + } + + // Filter execution order if specific cell requested + let execution_order = self.filter_execution_order(cell_filter)?; + + // Execute + let exec_start = Instant::now(); + executor.execute_in_order(&execution_order, &self.deps)?; + let execution_time = exec_start.elapsed(); + + // Collect outputs + let mut outputs = HashMap::new(); + for &cell_id in &execution_order { + if let Some(output) = executor.state().get_output(cell_id) { + outputs.insert(cell_id, output); + } + } + + Ok(ExecutionInfo { + executed_cells: execution_order, + execution_time, + outputs, + }) + } + + /// Filter execution order based on cell filter. + fn filter_execution_order(&self, cell_filter: Option<&str>) -> anyhow::Result> { + if let Some(cell_name) = cell_filter { + let target_id = self + .cell_ids + .get(cell_name) + .ok_or_else(|| anyhow::anyhow!("Cell '{}' not found", cell_name))?; + + Ok(self + .order + .iter() + .copied() + .filter(|&id| { + id == *target_id || is_transitive_dependency(id, *target_id, &self.deps) + }) + .collect()) + } else { + Ok(self.order.clone()) + } + } +} diff --git a/crates/venus/cli/src/executor/mod.rs b/crates/venus/cli/src/executor/mod.rs new file mode 100644 index 0000000..ec41b46 --- /dev/null +++ b/crates/venus/cli/src/executor/mod.rs @@ -0,0 +1,13 @@ +//! Shared notebook execution pipeline for Venus CLI. + +mod compilation; +mod dependencies; +mod display; +mod execution; +mod model; +mod progress; +mod setup; + +pub use dependencies::is_transitive_dependency; +pub use model::{CompilationInfo, CompiledCellInfo, ExecutionInfo, NotebookExecutor}; +pub use progress::ProgressCallback; diff --git a/crates/venus/cli/src/executor/model.rs b/crates/venus/cli/src/executor/model.rs new file mode 100644 index 0000000..7254be5 --- /dev/null +++ b/crates/venus/cli/src/executor/model.rs @@ -0,0 +1,71 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use venus_core::compile::{CompiledCell, CompilerConfig, ToolchainManager, UniverseBuilder}; +use venus_core::graph::{CellId, CellInfo}; +use venus_core::paths::NotebookDirs; +use venus_core::state::BoxedOutput; + +/// Compiled cell with metadata for execution. +#[derive(Clone)] +pub struct CompiledCellInfo { + /// The compiled cell. + pub compiled: CompiledCell, + /// Number of dependencies. + pub dep_count: usize, + /// Compilation time in milliseconds. + #[allow(dead_code)] + pub compile_time_ms: u64, + /// Whether the cell was cached. + #[allow(dead_code)] + pub cached: bool, +} + +/// Result of cell compilation. +pub struct CompilationInfo { + /// Successfully compiled cells. + pub cells: HashMap, + /// Compilation errors by cell name. + pub errors: Vec<(String, Vec)>, +} + +/// Result of cell execution. +pub struct ExecutionInfo { + /// Cells that were executed (in order). + pub executed_cells: Vec, + /// Execution time. + pub execution_time: Duration, + /// Cell outputs (cell_id -> output). + pub outputs: HashMap>, +} + +/// Notebook executor that manages the full execution pipeline. +pub struct NotebookExecutor { + /// Absolute path to the notebook file. + pub notebook_path: PathBuf, + /// Notebook source code. + #[allow(dead_code)] + pub source: String, + /// Notebook directories. + pub dirs: NotebookDirs, + /// Toolchain manager. + pub toolchain: ToolchainManager, + /// Parsed cells. + pub cells: Vec, + /// Cell name to ID mapping. + pub cell_ids: HashMap, + /// Topological execution order. + pub order: Vec, + /// Dependency map (cell_id -> dependencies). + pub deps: HashMap>, + /// Compiler configuration. + pub config: CompilerConfig, + /// Universe builder (for dependency hash). + pub universe_builder: UniverseBuilder, + /// Path to compiled universe. + pub universe_path: PathBuf, + /// Whether using release mode. + #[allow(dead_code)] + pub release: bool, +} diff --git a/crates/venus/cli/src/executor/progress.rs b/crates/venus/cli/src/executor/progress.rs new file mode 100644 index 0000000..455881e --- /dev/null +++ b/crates/venus/cli/src/executor/progress.rs @@ -0,0 +1,79 @@ +use crate::colors; +use venus_core::Error; +use venus_core::execute::ExecutionCallback; +use venus_core::graph::CellId; + +/// Progress callback that prints execution status to the terminal. +pub struct ProgressCallback { + /// Whether to show verbose output. + verbose: bool, +} + +impl ProgressCallback { + /// Create a new progress callback. + pub fn new() -> Self { + Self { verbose: false } + } + + /// Create a verbose progress callback. + #[allow(dead_code)] + pub fn verbose() -> Self { + Self { verbose: true } + } +} + +impl Default for ProgressCallback { + fn default() -> Self { + Self::new() + } +} + +impl ExecutionCallback for ProgressCallback { + fn on_cell_started(&self, _cell_id: CellId, name: &str) { + print!( + "{} ▶ Running{} {}{}... ", + colors::CYAN, + colors::RESET, + colors::BOLD, + name + ); + colors::flush_stdout(); + } + + fn on_cell_completed(&self, _cell_id: CellId, _name: &str) { + println!("{}✓{}", colors::GREEN, colors::RESET); + } + + fn on_cell_error(&self, _cell_id: CellId, _name: &str, error: &Error) { + println!("{}✗{}", colors::RED, colors::RESET); + eprintln!("{} Error:{} {}", colors::RED, colors::RESET, error); + } + + fn on_level_started(&self, level: usize, cell_count: usize) { + if self.verbose && cell_count > 1 { + println!( + "{}Level {}:{} {} cells (parallel)", + colors::DIM, + level, + colors::RESET, + cell_count + ); + } + } + + fn on_level_completed(&self, _level: usize) {} +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_progress_callback_creation() { + let callback = ProgressCallback::new(); + assert!(!callback.verbose); + + let verbose = ProgressCallback::verbose(); + assert!(verbose.verbose); + } +} diff --git a/crates/venus/cli/src/executor/setup.rs b/crates/venus/cli/src/executor/setup.rs new file mode 100644 index 0000000..8521183 --- /dev/null +++ b/crates/venus/cli/src/executor/setup.rs @@ -0,0 +1,95 @@ +use super::NotebookExecutor; +use std::collections::HashMap; +use std::{fs, path::Path}; +use venus_core::compile::{CompilerConfig, ToolchainManager, UniverseBuilder}; +use venus_core::graph::{CellId, CellParser, GraphEngine}; +use venus_core::paths::NotebookDirs; + +impl NotebookExecutor { + /// Create a new notebook executor. + /// + /// This performs the setup phase: parsing, dependency resolution, and universe building. + pub fn new(notebook_path: &str, release: bool) -> anyhow::Result { + let path = Path::new(notebook_path); + if !path.exists() { + anyhow::bail!("Notebook not found: {}", notebook_path); + } + + let source = fs::read_to_string(path)?; + let abs_path = path.canonicalize()?; + let dirs = NotebookDirs::from_notebook_path(&abs_path)?; + + // Initialize toolchain + Self::print_step("Checking toolchain"); + let toolchain = ToolchainManager::new()?; + Self::print_success(None); + + // Parse cells + Self::print_step("Parsing cells"); + let mut parser = CellParser::new(); + let parse_result = parser.parse_str(&source, &abs_path)?; + let cells = parse_result.code_cells; + let definition_cells = parse_result.definition_cells; + Self::print_success(Some(&format!("{} code cells", cells.len()))); + + // Build dependency graph + Self::print_step("Building dependency graph"); + let mut graph = GraphEngine::new(); + let mut cell_ids: HashMap = HashMap::new(); + for cell in &cells { + let real_id = graph.add_cell(cell.clone()); + cell_ids.insert(cell.name.clone(), real_id); + } + graph.resolve_dependencies()?; + let order = graph.topological_order()?; + Self::print_success(None); + + // Build dependency map + let deps: HashMap> = cells + .iter() + .map(|cell| { + let real_id = cell_ids[&cell.name]; + let dep_ids: Vec = cell + .dependencies + .iter() + .filter_map(|dep| cell_ids.get(&dep.param_name).copied()) + .collect(); + (real_id, dep_ids) + }) + .collect(); + + // Build universe + Self::print_step("Building universe"); + let config = if release { + CompilerConfig::for_notebook_release(&dirs) + } else { + CompilerConfig::for_notebook(&dirs) + }; + + let mut universe_builder = + UniverseBuilder::for_notebook(config.clone(), toolchain.clone(), &abs_path)?; + universe_builder.parse_dependencies(&source, &definition_cells)?; + let universe_path = universe_builder.build()?; + + if universe_builder.dependencies().is_empty() { + Self::print_success(Some("runtime only")); + } else { + Self::print_success(None); + } + + Ok(Self { + notebook_path: abs_path, + source, + dirs, + toolchain, + cells, + cell_ids, + order, + deps, + config, + universe_builder, + universe_path, + release, + }) + } +} diff --git a/crates/venus/cli/tests/workspace.rs b/crates/venus/cli/tests/workspace.rs new file mode 100644 index 0000000..801ad22 --- /dev/null +++ b/crates/venus/cli/tests/workspace.rs @@ -0,0 +1,96 @@ +//! CLI execution with inherited workspace dependencies. + +use std::fs; +use std::path::Path; +use std::process::Command; + +use tempfile::TempDir; + +fn run_notebook(notebook: &Path, current_dir: &Path, expected: &str) -> std::io::Result<()> { + let output = Command::new(env!("CARGO_BIN_EXE_venus")) + .current_dir(current_dir) + .arg("run") + .arg(notebook) + .output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "venus run returned {}. stdout:\n{stdout}\nstderr:\n{stderr}", + output.status + ); + assert!( + stdout.contains(expected), + "Missing {expected}. stdout:\n{stdout}" + ); + assert!( + stdout.contains("Completed"), + "Missing completion. stdout:\n{stdout}" + ); + Ok(()) +} + +#[test] +fn runtime_consumes_inherited_internal_dependency() -> std::io::Result<()> { + let workspace = TempDir::new()?; + let unrelated_directory = TempDir::new()?; + let root = workspace.path(); + let notebook_directory = root.join("app/notebooks"); + let helper_directory = root.join("internal-helper/src"); + fs::create_dir_all(¬ebook_directory)?; + fs::create_dir_all(root.join("app/src"))?; + fs::create_dir_all(&helper_directory)?; + fs::write( + root.join("Cargo.toml"), + r#"[workspace] +members = ["app", "internal-helper"] +resolver = "2" + +[workspace.dependencies] +internal-helper = { path = "internal-helper" } +"#, + )?; + fs::write( + root.join("app/Cargo.toml"), + r#"[package] +name = "notebook-app" +version = "0.1.0" +edition = "2024" + +[dependencies] +internal-helper.workspace = true +"#, + )?; + fs::write(root.join("app/src/lib.rs"), "")?; + fs::write( + root.join("internal-helper/Cargo.toml"), + r#"[package] +name = "internal-helper" +version = "0.1.0" +edition = "2024" +"#, + )?; + let helper_source = helper_directory.join("lib.rs"); + fs::write( + &helper_source, + r#"pub fn value() -> &'static str { "workspace-first" } +"#, + )?; + let notebook = notebook_directory.join("runtime.rs"); + fs::write( + ¬ebook, + r#"#[venus::cell] +pub fn internal_value() -> String { + internal_helper::value().to_string() +} +"#, + )?; + + run_notebook(¬ebook, unrelated_directory.path(), "workspace-first")?; + fs::write( + &helper_source, + r#"pub fn value() -> &'static str { "workspace-updated" } +"#, + )?; + run_notebook(¬ebook, unrelated_directory.path(), "workspace-updated") +} diff --git a/docs/cells.md b/docs/cells.md index 67ff8c9..fa3192b 100644 --- a/docs/cells.md +++ b/docs/cells.md @@ -41,6 +41,58 @@ Parameters must match the return type of the dependency: | `Vec` | `&Vec` | | `CustomType` | `&CustomType` | +## Workspace Notebooks + +Venus reads the nearest parent `Cargo.toml` for each notebook. Package notebooks inherit declared normal dependencies and target dependencies. Both interactive execution and standalone builds use this context. + +Workspace members expand `workspace = true` from their owning workspace. Unused entries in `workspace.dependencies` stay excluded. Notebooks beneath a virtual workspace root use its dependency catalog. + +```toml +# Workspace Cargo.toml +[workspace] +members = ["analysis", "internal-model"] + +[workspace.dependencies] +model = { package = "internal-model", path = "internal-model", default-features = false } +``` + +```toml +# analysis/Cargo.toml +[package] +name = "analysis" +version = "0.1.0" +edition = "2021" + +[dependencies] +model.workspace = true +``` + +```rust +// analysis/notebooks/report.rs +#[venus::cell] +pub fn answer() -> i32 { + model::answer() +} +``` + +Notebook `cargo` blocks override matching dependency aliases. Paths in those blocks resolve from the notebook directory. Member dependency paths resolve from the member directory. Inherited dependency paths and root patch paths resolve from the workspace directory. + +````rust +//! ```cargo +//! [dependencies.model] +//! package = "internal-model" +//! path = "../../alternate-model" +//! features = ["reporting"] +//! ``` +```` + +- **Aliases and sources:** Cargo package aliases, Git selectors, registry selectors, and root patches survive generation. +- **Features:** Feature arrays, disabled defaults, optional dependencies, and feature activation graphs retain Cargo semantics. +- **Targets:** Target dependencies keep their target conditions. Dependency exports follow target and optional feature activation. +- **Runtime dependencies:** Venus requires compatible `venus`, `rkyv`, and `serde_json` aliases. Compatible additional features survive generation. Conflicting packages or sources return contextual errors. +- **Cache:** Manifest changes invalidate dependency hashes. Local dependency and patch paths bypass universe caching and refresh compiled cells after reloads. +- **Isolation:** Generated Cargo projects contain their own `[workspace]` table. + ## Doc Comments Doc comments become cell descriptions: