diff --git a/Cargo.lock b/Cargo.lock index 2b2b1b37a..9039cd8a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3838,6 +3838,7 @@ dependencies = [ "bigdecimal", "bip32", "byte-unit", + "camino", "camino-tempfile", "candid", "candid_parser", @@ -3889,6 +3890,7 @@ dependencies = [ "shellwords", "snafu", "sysinfo", + "tar", "test-tag", "time", "tiny-bip39", diff --git a/crates/icp-cli/Cargo.toml b/crates/icp-cli/Cargo.toml index f940f7034..bddc01ea8 100644 --- a/crates/icp-cli/Cargo.toml +++ b/crates/icp-cli/Cargo.toml @@ -21,6 +21,7 @@ base64.workspace = true bigdecimal.workspace = true bip32.workspace = true byte-unit.workspace = true +camino.workspace = true camino-tempfile.workspace = true candid_parser = { workspace = true, features = ["assist"] } candid.workspace = true @@ -32,6 +33,7 @@ dunce.workspace = true elliptic-curve.workspace = true flate2.workspace = true futures.workspace = true +tar.workspace = true hex.workspace = true httptest.workspace = true ic-agent.workspace = true diff --git a/crates/icp-cli/src/commands/project/bundle.rs b/crates/icp-cli/src/commands/project/bundle.rs new file mode 100644 index 000000000..0b758cbca --- /dev/null +++ b/crates/icp-cli/src/commands/project/bundle.rs @@ -0,0 +1,41 @@ +use anyhow::Context as _; +use clap::Args; +use icp::context::Context; +use icp::prelude::*; + +use crate::operations::bundle::create_bundle; + +/// Bundle a project into a self-contained deployable archive. +/// +/// Builds all project canisters and packages them with a rewritten manifest +/// into a `.tar.gz` file. The rewritten manifest replaces all build steps +/// with pre-built steps referencing the bundled WASM files. Asset sync +/// directories are included in the archive. +/// +/// Projects with script sync steps cannot be bundled. +#[derive(Args, Debug)] +pub(crate) struct BundleArgs { + /// Output path for the bundle archive (e.g. bundle.tar.gz) + #[arg(long, short)] + pub(crate) output: PathBuf, +} + +pub(crate) async fn exec(ctx: &Context, args: &BundleArgs) -> Result<(), anyhow::Error> { + let project = ctx.project.load().await.context("failed to load project")?; + + let canisters: Vec<_> = project.canisters.into_values().collect(); + + create_bundle( + &project.dir, + canisters, + ctx.builder.clone(), + ctx.artifacts.clone(), + &ctx.dirs.package_cache()?, + ctx.debug, + &args.output, + ) + .await + .context("failed to create bundle")?; + + Ok(()) +} diff --git a/crates/icp-cli/src/commands/project/mod.rs b/crates/icp-cli/src/commands/project/mod.rs index 48ae1348a..1c147a0ed 100644 --- a/crates/icp-cli/src/commands/project/mod.rs +++ b/crates/icp-cli/src/commands/project/mod.rs @@ -1,9 +1,12 @@ use clap::Subcommand; +pub(crate) mod bundle; pub(crate) mod show; -/// Display information about the current project +/// Manage the current project #[derive(Debug, Subcommand)] pub(crate) enum Command { Show(show::ShowArgs), + #[command(hide = true)] + Bundle(bundle::BundleArgs), } diff --git a/crates/icp-cli/src/main.rs b/crates/icp-cli/src/main.rs index 9d580708a..b883c6a91 100644 --- a/crates/icp-cli/src/main.rs +++ b/crates/icp-cli/src/main.rs @@ -406,6 +406,9 @@ async fn dispatch(ctx: &icp::context::Context, command: Command) -> Result<(), E commands::project::Command::Show(args) => { commands::project::show::exec(ctx, &args).await? } + commands::project::Command::Bundle(args) => { + commands::project::bundle::exec(ctx, &args).await? + } }, // Settings diff --git a/crates/icp-cli/src/operations/bundle.rs b/crates/icp-cli/src/operations/bundle.rs new file mode 100644 index 000000000..9da6b1d9f --- /dev/null +++ b/crates/icp-cli/src/operations/bundle.rs @@ -0,0 +1,815 @@ +use std::{ + collections::HashMap, + fs::File, + io::{BufWriter, Cursor, Write}, + sync::Arc, +}; + +use sha2::{Digest, Sha256}; + +use camino::Utf8Component; +use flate2::{Compression, write::GzEncoder}; +use icp::{ + Canister, InitArgs, + canister::{build::Build, wasm}, + fs, + manifest::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, EnvironmentManifest, Instructions, + Item, LoadManifestFromPathError, ManagedMode, ManifestInitArgs, Mode, NetworkManifest, + PROJECT_MANIFEST, ProjectManifest, SyncStep, SyncSteps, + assets::DirField, + load_manifest_from_path, plugin, prebuilt, + prebuilt::{LocalSource, SourceField}, + }, + package::PackageCache, + prelude::*, + store_artifact, +}; +use snafu::{ResultExt, Snafu}; +use tar::Builder; + +use crate::operations::build::{BuildManyError, build_many_with_progress_bar}; + +#[derive(Debug, Snafu)] +pub enum BundleError { + #[snafu(display( + "canister '{canister}' has a script sync step, which is not supported in bundles" + ))] + ScriptSyncStep { canister: String }, + + #[snafu(display( + "canister names {names:?} all sanitize to the same archive segment '{sanitized}'; \ + rename them to use distinct alphanumeric/-/_/. characters" + ))] + CanisterNameCollision { + sanitized: String, + names: Vec, + }, + + #[snafu(transparent)] + Build { source: BuildManyError }, + + #[snafu(display("failed to look up built artifact for canister '{canister}'"))] + LookupArtifact { + canister: String, + source: store_artifact::LookupArtifactError, + }, + + #[snafu(display("failed to load project manifest for bundle"))] + LoadManifest { source: LoadManifestFromPathError }, + + #[snafu(display("failed to load network manifest from '{path}'"))] + LoadNetwork { + path: PathBuf, + source: LoadManifestFromPathError, + }, + + #[snafu(display("failed to load environment manifest from '{path}'"))] + LoadEnvironment { + path: PathBuf, + source: LoadManifestFromPathError, + }, + + #[snafu(display("failed to read init_args file '{path}'"))] + ReadInitArgs { path: PathBuf, source: fs::IoError }, + + #[snafu(display("failed to serialize bundle manifest"))] + SerializeManifest { source: serde_yaml::Error }, + + #[snafu(display("failed to add '{path}' to bundle archive"))] + WriteArchiveEntry { + path: PathBuf, + source: std::io::Error, + }, + + #[snafu(display("failed to create bundle output file at '{path}'"))] + CreateOutput { + path: PathBuf, + source: std::io::Error, + }, + + #[snafu(display("failed to finalize bundle archive"))] + FlushArchive { source: std::io::Error }, + + #[snafu(display("failed to canonicalize path '{path}'"))] + CanonicalizePath { + path: PathBuf, + source: std::io::Error, + }, + + #[snafu(display( + "source path '{path}' for canister '{canister}' resolves outside the project directory \ + '{root}'; bundles cannot reference files outside the project" + ))] + SourceEscapesProject { + canister: String, + path: PathBuf, + root: PathBuf, + }, + + #[snafu(display( + "output path '{output}' is inside synced directory '{dir}'; bundling would include a \ + partial copy of the output file. Choose an output path outside this directory." + ))] + OutputOverlapsSyncDir { output: PathBuf, dir: PathBuf }, + + #[snafu(display( + "network '{network}' bind mount '{mount}' uses an absolute host path; \ + bundles require relative paths for portability" + ))] + AbsoluteBindMount { network: String, mount: String }, + + #[snafu(display("failed to resolve plugin wasm for canister '{canister}'"))] + ResolvePlugin { + canister: String, + source: wasm::WasmError, + }, + + #[snafu(display("failed to read plugin wasm for canister '{canister}'"))] + ReadPlugin { + canister: String, + source: fs::IoError, + }, + + #[snafu(display("failed to read plugin file '{file}' for canister '{canister}'"))] + ReadPluginFile { + canister: String, + file: String, + source: fs::IoError, + }, +} + +/// In-memory bytes destined for a single tar entry. +struct NamedBytes { + archive_path: String, + bytes: Vec, +} + +/// On-disk directory to be recursively appended at `archive_prefix`. +struct DirEntry { + src_path: PathBuf, + archive_prefix: String, +} + +/// Plugin input file. The canister/file metadata is carried so a read failure is attributable. +struct PluginFile { + src_path: PathBuf, + archive_path: String, + canister_name: String, + orig_file: String, +} + +/// init_args file referenced from an environment manifest. +struct InitArgsFile { + src_path: PathBuf, + archive_path: String, +} + +/// Everything the canister section contributes to the archive, separate from the manifest items. +#[derive(Default)] +struct BundleArtifacts { + wasms: Vec, + asset_dirs: Vec, + plugin_wasms: Vec, + plugin_dirs: Vec, + plugin_files: Vec, +} + +pub(crate) async fn create_bundle( + project_dir: &Path, + canisters: Vec<(PathBuf, Canister)>, + builder: Arc, + artifacts: Arc, + pkg_cache: &PackageCache, + debug: bool, + output: &Path, +) -> Result<(), BundleError> { + validate_canisters(&canisters)?; + let canonical_project_dir = canonicalize(project_dir)?; + let canonical_sync_dirs = validate_source_paths(&canisters, &canonical_project_dir)?; + validate_output_path(output, &canonical_sync_dirs)?; + + build_many_with_progress_bar( + canisters.clone(), + builder, + artifacts.clone(), + pkg_cache, + debug, + ) + .await?; + + // Re-read the raw manifest to preserve networks and environments verbatim. + let raw_manifest: ProjectManifest = + load_manifest_from_path(&project_dir.join(PROJECT_MANIFEST)) + .await + .context(LoadManifestSnafu)?; + + let (canister_items, bundle_artifacts) = + prepare_canisters(&canisters, &*artifacts, pkg_cache).await?; + let networks = inline_networks(raw_manifest.networks, project_dir).await?; + let (environments, init_args_files) = + inline_environments(raw_manifest.environments, project_dir, &canisters).await?; + + let bundle_manifest = ProjectManifest { + canisters: canister_items, + networks, + environments, + }; + + write_archive( + output, + &bundle_manifest, + &bundle_artifacts, + &init_args_files, + ) +} + +/// Build the per-canister manifest items and collect the archive artifacts they reference. +async fn prepare_canisters( + canisters: &[(PathBuf, Canister)], + artifacts: &dyn store_artifact::Access, + pkg_cache: &PackageCache, +) -> Result<(Vec>, BundleArtifacts), BundleError> { + let mut items = Vec::with_capacity(canisters.len()); + let mut out = BundleArtifacts::default(); + for (canister_path, canister) in canisters { + let item = + prepare_canister(canister_path, canister, artifacts, pkg_cache, &mut out).await?; + items.push(item); + } + Ok((items, out)) +} + +async fn prepare_canister( + canister_path: &Path, + canister: &Canister, + artifacts: &dyn store_artifact::Access, + pkg_cache: &PackageCache, + out: &mut BundleArtifacts, +) -> Result, BundleError> { + let path_name = path_segment(&canister.name); + let wasm = artifacts + .lookup(&canister.name) + .await + .context(LookupArtifactSnafu { + canister: canister.name.clone(), + })?; + let sha256 = hex::encode(Sha256::digest(&wasm)); + let wasm_filename = format!("{path_name}.wasm"); + + let mut bundle_sync_steps = Vec::with_capacity(canister.sync.steps.len()); + let mut plugin_idx: usize = 0; + + for step in &canister.sync.steps { + match step { + // validate_canisters rules this out + SyncStep::Script(_) => unreachable!("validated by validate_canisters"), + SyncStep::Assets(adapter) => { + bundle_sync_steps.push(prepare_asset_step(adapter, canister_path, &path_name, out)); + } + SyncStep::Plugin(adapter) => { + let idx = plugin_idx; + plugin_idx += 1; + bundle_sync_steps.push( + prepare_plugin_step( + adapter, + canister, + canister_path, + &path_name, + idx, + pkg_cache, + out, + ) + .await?, + ); + } + } + } + + let sync = (!bundle_sync_steps.is_empty()).then_some(SyncSteps { + steps: bundle_sync_steps, + }); + + out.wasms.push(NamedBytes { + archive_path: wasm_filename.clone(), + bytes: wasm, + }); + + Ok(Item::Manifest(CanisterManifest { + name: canister.name.clone(), + settings: canister.settings.clone(), + init_args: canister.init_args.as_ref().map(convert_init_args), + instructions: Instructions::BuildSync { + build: BuildSteps { + steps: vec![BuildStep::Prebuilt(prebuilt::Adapter { + source: prebuilt::SourceField::Local(prebuilt::LocalSource { + path: wasm_filename.as_str().into(), + }), + sha256: Some(sha256), + })], + }, + sync, + }, + })) +} + +fn prepare_asset_step( + adapter: &icp::manifest::assets::Adapter, + canister_path: &Path, + path_name: &str, + out: &mut BundleArtifacts, +) -> SyncStep { + let dirs = adapter.dir.as_vec(); + let mut prefixed: Vec = Vec::with_capacity(dirs.len()); + for d in &dirs { + let archive_prefix = format!("{path_name}/{}", normalize_archive_dir(d)); + out.asset_dirs.push(DirEntry { + src_path: canister_path.join(d), + archive_prefix: archive_prefix.clone(), + }); + prefixed.push(archive_prefix); + } + + let new_dir = if prefixed.len() == 1 { + DirField::Dir(prefixed.into_iter().next().unwrap()) + } else { + DirField::Dirs(prefixed) + }; + + SyncStep::Assets(icp::manifest::assets::Adapter { dir: new_dir }) +} + +async fn prepare_plugin_step( + adapter: &plugin::Adapter, + canister: &Canister, + canister_path: &Path, + path_name: &str, + idx: usize, + pkg_cache: &PackageCache, + out: &mut BundleArtifacts, +) -> Result { + let plugin_wasm_path = format!("plugins/{path_name}/{idx}.wasm"); + + let resolved = wasm::resolve( + &adapter.source, + canister_path, + adapter.sha256.as_deref(), + None, + pkg_cache, + ) + .await + .context(ResolvePluginSnafu { + canister: canister.name.clone(), + })?; + + let plugin_bytes = fs::read(&resolved).context(ReadPluginSnafu { + canister: canister.name.clone(), + })?; + let plugin_sha256 = hex::encode(Sha256::digest(&plugin_bytes)); + out.plugin_wasms.push(NamedBytes { + archive_path: plugin_wasm_path.clone(), + bytes: plugin_bytes, + }); + + // Plugin preopened dirs go under a `dirs/` subdir so a user-supplied dir literally named + // `files` cannot collide with the `files/` area used for plugin input files. + let bundle_dirs = adapter.dirs.as_ref().map(|dirs| { + dirs.iter() + .map(|d| { + let archive_prefix = format!( + "plugins/{path_name}/{idx}/dirs/{}", + normalize_archive_dir(d) + ); + out.plugin_dirs.push(DirEntry { + src_path: canister_path.join(d), + archive_prefix: archive_prefix.clone(), + }); + archive_prefix + }) + .collect::>() + }); + + let bundle_files = adapter.files.as_ref().map(|files| { + files + .iter() + .map(|f| { + let archive_path = format!( + "plugins/{path_name}/{idx}/files/{}", + normalize_archive_dir(f) + ); + out.plugin_files.push(PluginFile { + src_path: canister_path.join(f), + archive_path: archive_path.clone(), + canister_name: canister.name.clone(), + orig_file: f.clone(), + }); + archive_path + }) + .collect::>() + }); + + Ok(SyncStep::Plugin(plugin::Adapter { + source: SourceField::Local(LocalSource { + path: plugin_wasm_path.as_str().into(), + }), + sha256: Some(plugin_sha256), + dirs: bundle_dirs, + files: bundle_files, + })) +} + +async fn inline_networks( + items: Vec>, + project_dir: &Path, +) -> Result>, BundleError> { + let mut out = Vec::with_capacity(items.len()); + for item in items { + let inlined = match item { + Item::Manifest(_) => item, + Item::Path(ref path) => { + let full = project_dir.join(path); + let m = load_manifest_from_path::(&full) + .await + .context(LoadNetworkSnafu { path: full })?; + Item::Manifest(m) + } + }; + if let Item::Manifest(ref net) = inlined { + validate_network_for_bundle(net)?; + } + out.push(inlined); + } + Ok(out) +} + +async fn inline_environments( + items: Vec>, + project_dir: &Path, + canisters: &[(PathBuf, Canister)], +) -> Result<(Vec>, Vec), BundleError> { + // Inline canisters resolve init_args paths relative to the project dir (matches the + // Item::Manifest behavior in project.rs). + let canister_path_map: HashMap<&str, &Path> = canisters + .iter() + .map(|(path, canister)| (canister.name.as_str(), path.as_path())) + .collect(); + + let mut out = Vec::with_capacity(items.len()); + let mut init_args_files = Vec::new(); + + for item in items { + let mut inlined = match item { + Item::Manifest(_) => item, + Item::Path(ref path) => { + let full = project_dir.join(path); + let m = load_manifest_from_path::(&full) + .await + .context(LoadEnvironmentSnafu { path: full })?; + Item::Manifest(m) + } + }; + + if let Item::Manifest(ref mut env) = inlined + && let Some(ref mut overrides) = env.init_args + { + for (canister_name, mia) in overrides.iter_mut() { + if let ManifestInitArgs::Path { + path: orig_path, + format: fmt, + } = &*mia + { + let base = canister_path_map + .get(canister_name.as_str()) + .copied() + .unwrap_or(project_dir); + let archive_path = format!( + "init-args/{}/{}", + path_segment(canister_name), + normalize_archive_dir(orig_path) + ); + init_args_files.push(InitArgsFile { + src_path: base.join(orig_path), + archive_path: archive_path.clone(), + }); + *mia = ManifestInitArgs::Path { + path: archive_path, + format: fmt.clone(), + }; + } + } + } + + out.push(inlined); + } + + Ok((out, init_args_files)) +} + +fn write_archive( + output: &Path, + bundle_manifest: &ProjectManifest, + artifacts: &BundleArtifacts, + init_args_files: &[InitArgsFile], +) -> Result<(), BundleError> { + let manifest_yaml = serde_yaml::to_string(bundle_manifest).context(SerializeManifestSnafu)?; + + let file = File::create(output.as_std_path()).context(CreateOutputSnafu { + path: output.to_path_buf(), + })?; + let gz = GzEncoder::new(BufWriter::new(file), Compression::default()); + let mut archive = Builder::new(gz); + // Record symlinks as Symlink entries rather than slurping their targets — keeps secrets + // outside the project from leaking via a symlinked asset dir. + archive.follow_symlinks(false); + // Strip mtime/uid/gid from headers written via append_dir_all so the archive is + // byte-reproducible across machines. + archive.mode(tar::HeaderMode::Deterministic); + + append_bytes(&mut archive, "icp.yaml", manifest_yaml.as_bytes())?; + + for nb in &artifacts.wasms { + append_bytes(&mut archive, &nb.archive_path, &nb.bytes)?; + } + + for entry in init_args_files { + let data = fs::read(&entry.src_path).context(ReadInitArgsSnafu { + path: entry.src_path.clone(), + })?; + append_bytes(&mut archive, &entry.archive_path, &data)?; + } + + for d in &artifacts.asset_dirs { + append_dir(&mut archive, &d.src_path, &d.archive_prefix)?; + } + + for nb in &artifacts.plugin_wasms { + append_bytes(&mut archive, &nb.archive_path, &nb.bytes)?; + } + + for d in &artifacts.plugin_dirs { + append_dir(&mut archive, &d.src_path, &d.archive_prefix)?; + } + + for pf in &artifacts.plugin_files { + let data = fs::read(&pf.src_path).context(ReadPluginFileSnafu { + canister: pf.canister_name.clone(), + file: pf.orig_file.clone(), + })?; + append_bytes(&mut archive, &pf.archive_path, &data)?; + } + + // Finalize the tar trailer, the gzip trailer, and the underlying BufWriter — any of these + // may fail to write the last bytes to disk, and we want to surface that. + let gz = archive.into_inner().context(FlushArchiveSnafu)?; + let buf = gz.finish().context(FlushArchiveSnafu)?; + buf.into_inner().map_err(|e| BundleError::FlushArchive { + source: e.into_error(), + })?; + + Ok(()) +} + +fn append_bytes( + archive: &mut Builder, + archive_path: &str, + bytes: &[u8], +) -> Result<(), BundleError> { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, archive_path, Cursor::new(bytes)) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from(archive_path), + }) +} + +fn append_dir( + archive: &mut Builder, + src_path: &Path, + archive_prefix: &str, +) -> Result<(), BundleError> { + archive + .append_dir_all(archive_prefix, src_path.as_std_path()) + .context(WriteArchiveEntrySnafu { + path: PathBuf::from(archive_prefix), + }) +} + +/// Up-front validation that the canister set can be bundled: +/// - no sync step is a script (we cannot replay an arbitrary shell command from the bundle) +/// - all sanitized canister names are unique (otherwise archive paths collide silently) +fn validate_canisters(canisters: &[(PathBuf, Canister)]) -> Result<(), BundleError> { + for (_, canister) in canisters { + for step in &canister.sync.steps { + if matches!(step, SyncStep::Script(_)) { + return ScriptSyncStepSnafu { + canister: canister.name.clone(), + } + .fail(); + } + } + } + + let mut by_segment: HashMap> = HashMap::new(); + for (_, canister) in canisters { + by_segment + .entry(path_segment(&canister.name)) + .or_default() + .push(canister.name.clone()); + } + for (sanitized, mut names) in by_segment { + if names.len() > 1 { + names.sort(); + return CanisterNameCollisionSnafu { sanitized, names }.fail(); + } + } + + Ok(()) +} + +/// Canonicalize every asset/plugin source path and confirm it lives inside the canonical +/// project directory. Returns the canonical sync-directory paths for use in output-overlap +/// detection. +fn validate_source_paths( + canisters: &[(PathBuf, Canister)], + canonical_project_dir: &Path, +) -> Result, BundleError> { + let mut canonical_sync_dirs = Vec::new(); + for (canister_path, canister) in canisters { + for step in &canister.sync.steps { + match step { + SyncStep::Script(_) => {} + SyncStep::Assets(adapter) => { + for d in adapter.dir.as_vec() { + let src = canister_path.join(&d); + let canon = canonicalize_within_project( + &src, + canonical_project_dir, + &canister.name, + )?; + canonical_sync_dirs.push(canon); + } + } + SyncStep::Plugin(adapter) => { + if let Some(dirs) = &adapter.dirs { + for d in dirs { + let src = canister_path.join(d); + let canon = canonicalize_within_project( + &src, + canonical_project_dir, + &canister.name, + )?; + canonical_sync_dirs.push(canon); + } + } + if let Some(files) = &adapter.files { + for f in files { + let src = canister_path.join(f); + canonicalize_within_project( + &src, + canonical_project_dir, + &canister.name, + )?; + } + } + } + } + } + } + Ok(canonical_sync_dirs) +} + +/// Refuse to write the bundle output into a directory we are about to recursively archive — +/// otherwise the partial bundle file would be included in itself. +fn validate_output_path(output: &Path, canonical_sync_dirs: &[PathBuf]) -> Result<(), BundleError> { + let canonical_output = canonicalize_output(output)?; + for sync_dir in canonical_sync_dirs { + if canonical_output.starts_with(sync_dir) { + return OutputOverlapsSyncDirSnafu { + output: canonical_output, + dir: sync_dir.clone(), + } + .fail(); + } + } + Ok(()) +} + +fn validate_network_for_bundle(net: &NetworkManifest) -> Result<(), BundleError> { + let Mode::Managed(managed) = &net.configuration else { + return Ok(()); + }; + let ManagedMode::Image { + mounts: Some(mounts), + .. + } = managed.mode.as_ref() + else { + return Ok(()); + }; + for mount in mounts { + // Format documented in the manifest: relative_host_path:container_path[:options]. + // Split once on ':' and check the host-path side. + let host = mount.split(':').next().unwrap_or(""); + if Path::new(host).is_absolute() { + return AbsoluteBindMountSnafu { + network: net.name.clone(), + mount: mount.clone(), + } + .fail(); + } + } + Ok(()) +} + +fn canonicalize(path: &Path) -> Result { + path.canonicalize_utf8().context(CanonicalizePathSnafu { + path: path.to_path_buf(), + }) +} + +fn canonicalize_within_project( + src: &Path, + canonical_project_dir: &Path, + canister: &str, +) -> Result { + let canon = canonicalize(src)?; + if !canon.starts_with(canonical_project_dir) { + return SourceEscapesProjectSnafu { + canister: canister.to_owned(), + path: src.to_path_buf(), + root: canonical_project_dir.to_path_buf(), + } + .fail(); + } + Ok(canon) +} + +/// Resolve the canonical form of an output path that may not exist yet. We canonicalize its +/// parent (which must exist before we can write a file there anyway) and append the filename. +fn canonicalize_output(output: &Path) -> Result { + if output.exists() { + return canonicalize(output); + } + let parent = output + .parent() + .filter(|p| !p.as_str().is_empty()) + .unwrap_or(Path::new(".")); + let filename = output + .file_name() + .map(|s| s.to_string()) + .unwrap_or_default(); + let canon_parent = canonicalize(parent)?; + Ok(canon_parent.join(filename)) +} + +/// Normalizes a relative directory path for use as a tar archive prefix. +/// +/// Resolves `.` and `..` lexically, strips leading `..` that would escape the +/// canister root, and discards any absolute prefix. The result is a clean +/// forward-slash-separated relative path safe to embed in a tar entry name. +fn normalize_archive_dir(dir: &str) -> String { + // Treat `\` as a path separator regardless of host OS so cross-platform bundles don't + // produce archive entry names that decode as nested paths on Windows extraction. + let dir = dir.replace('\\', "/"); + let mut parts: Vec = Vec::new(); + for component in PathBuf::from(dir.as_str()).components() { + match component { + Utf8Component::Normal(s) => parts.push(s.to_owned()), + Utf8Component::CurDir => {} + Utf8Component::ParentDir => { + parts.pop(); + } + Utf8Component::RootDir | Utf8Component::Prefix(_) => parts.clear(), + } + } + parts.join("/") +} + +/// Converts a canister name into a cross-platform-safe path segment. +/// +/// Replaces any character that is not alphanumeric, `-`, `_`, or `.` with `_`. +/// This covers all characters prohibited on Windows (`< > : " / \ | ? *`), +/// path separators on Unix, and control characters. +fn path_segment(name: &str) -> String { + name.chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' => c, + _ => '_', + }) + .collect() +} + +fn convert_init_args(args: &InitArgs) -> ManifestInitArgs { + match args { + InitArgs::Text { content, format } => ManifestInitArgs::Value { + value: content.clone(), + format: format.clone(), + }, + InitArgs::Binary(bytes) => ManifestInitArgs::Value { + value: hex::encode(bytes), + format: ArgsFormat::Hex, + }, + } +} diff --git a/crates/icp-cli/src/operations/mod.rs b/crates/icp-cli/src/operations/mod.rs index f1d8ba2b7..a53b619a0 100644 --- a/crates/icp-cli/src/operations/mod.rs +++ b/crates/icp-cli/src/operations/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod binding_env_vars; pub(crate) mod build; +pub(crate) mod bundle; pub(crate) mod candid_compat; pub(crate) mod canister_migration; pub(crate) mod create; diff --git a/crates/icp-cli/tests/bundle_tests.rs b/crates/icp-cli/tests/bundle_tests.rs new file mode 100644 index 000000000..42f8bccb9 --- /dev/null +++ b/crates/icp-cli/tests/bundle_tests.rs @@ -0,0 +1,684 @@ +use std::{ + fs, + io::{BufReader, Read as _}, +}; + +use camino::Utf8Component; +use flate2::bufread::GzDecoder; +use icp::{ + fs::{create_dir_all, json, write, write_string}, + prelude::*, + store_id::IdMapping, +}; +use indoc::formatdoc; +use predicates::{ + ord::eq, + prelude::PredicateBooleanExt, + str::{PredicateStrExt, contains}, +}; +use sha2::{Digest, Sha256}; +use tar::Archive; + +use crate::common::{ENVIRONMENT_RANDOM_PORT, NETWORK_RANDOM_PORT, TestContext, clients}; + +mod common; + +/// Bundle a standard frontend-backend project: a script-built backend canister and an +/// asset-canister-recipe frontend canister with an assets sync step. +/// Verify archive structure, manifest content, and that the bundle deploys successfully. +#[tokio::test] +async fn bundle_and_deploy() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("icp"); + + // A small WASM to serve as the pre-built artifact for the backend. + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + // Create asset directory for the frontend canister. + let asset_dir = project_dir.join("www"); + create_dir_all(&asset_dir).expect("failed to create asset dir"); + write_string(&asset_dir.join("index.html"), "hello").expect("failed to write asset file"); + + let pm = formatdoc! {r#" + canisters: + - name: backend + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + + - name: frontend + recipe: + type: "@dfinity/asset-canister@v2.1.0" + configuration: + dir: www + + {NETWORK_RANDOM_PORT} + {ENVIRONMENT_RANDOM_PORT} + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + assert!(bundle_path.exists(), "bundle file was not created"); + + // Extract and inspect archive contents. + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_manifest = false; + let mut found_backend_wasm = false; + let mut found_frontend_wasm = false; + let mut found_asset = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + match path.as_str() { + "icp.yaml" => { + found_manifest = true; + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + "backend.wasm" => { + found_backend_wasm = true; + } + "frontend.wasm" => { + found_frontend_wasm = true; + } + p if p.starts_with("frontend/www/") => { + found_asset = true; + } + _ => {} + } + } + + assert!(found_manifest, "icp.yaml not found in bundle"); + assert!(found_backend_wasm, "backend.wasm not found in bundle"); + assert!(found_frontend_wasm, "frontend.wasm not found in bundle"); + assert!( + found_asset, + "asset file not found under frontend/www/ in bundle" + ); + + // Manifest must contain pre-built steps and no script or recipe steps. + assert!( + manifest_yaml.contains("pre-built"), + "bundle manifest should have pre-built build steps" + ); + assert!( + !manifest_yaml.contains("type: script"), + "bundle manifest should not contain script steps" + ); + assert!( + !manifest_yaml.contains("recipe:"), + "bundle manifest should not contain recipe sections" + ); + assert!( + manifest_yaml.contains("sha256:"), + "bundle manifest should include sha256 for pre-built wasms" + ); + + // Extract bundle to a fresh directory and deploy from it. + let bundle_dir = project_dir.join("bundle-extracted"); + create_dir_all(&bundle_dir).expect("failed to create bundle-extracted dir"); + + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + archive + .unpack(bundle_dir.as_std_path()) + .expect("failed to extract bundle"); + + let _g = ctx.start_network_in(&bundle_dir, "random-network").await; + ctx.ping_until_healthy(&bundle_dir, "random-network"); + + let network_port = ctx + .wait_for_network_descriptor(&bundle_dir, "random-network") + .gateway_port; + + clients::icp(&ctx, &bundle_dir, Some("random-environment".to_string())) + .mint_cycles(10 * TRILLION); + + ctx.icp() + .current_dir(&bundle_dir) + .args([ + "deploy", + "--subnet", + common::SUBNET_ID, + "--environment", + "random-environment", + ]) + .assert() + .success(); + + // Verify the backend canister responds to a query call. + ctx.icp() + .current_dir(&bundle_dir) + .args([ + "canister", + "call", + "--environment", + "random-environment", + "backend", + "greet", + "(\"world\")", + ]) + .assert() + .success() + .stdout(eq("(\"Hello, world!\")").trim()); + + // Verify the frontend canister serves the bundled asset. + let id_mapping: IdMapping = json::load( + &bundle_dir + .join(".icp") + .join("cache") + .join("mappings") + .join("random-environment.ids.json"), + ) + .expect("failed to read ID mapping"); + + let frontend_cid = id_mapping + .get("frontend") + .expect("frontend canister ID not found"); + + let resp = reqwest::get(format!( + "http://localhost:{network_port}/?canisterId={frontend_cid}" + )) + .await + .expect("request to frontend canister failed"); + + assert_eq!( + resp.text().await.expect("failed to read response body"), + "hello" + ); +} + +/// Bundle a canister whose environment override uses an external init_args file. +/// The file must be copied into the archive at a normalized path and the manifest +/// reference rewritten to match. +#[test] +fn bundle_inlines_external_init_args_file() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + write_string(&project_dir.join("args.idl"), "(\"world\")").expect("failed to write args file"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + + networks: + - name: random-network + mode: managed + gateway: + port: 0 + + environments: + - name: random-environment + network: random-network + init_args: + my-canister: + path: ./args.idl + format: candid + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_args_file = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + match path.as_str() { + "icp.yaml" => { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + "init-args/my-canister/args.idl" => { + found_args_file = true; + } + _ => {} + } + } + + assert!( + found_args_file, + "init-args/my-canister/args.idl not found in bundle" + ); + assert!( + manifest_yaml.contains("init-args/my-canister/args.idl"), + "bundle manifest should reference the relocated init_args file" + ); + assert!( + !manifest_yaml.contains("./args.idl"), + "bundle manifest should not contain the original relative path" + ); +} + +/// Canister names with characters invalid in file paths (spaces, `!`, `/`, etc.) +/// must be sanitized for archive entry names while the manifest preserves the +/// original name. +#[test] +fn bundle_sanitizes_canister_name_for_paths() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + let pm = formatdoc! {r#" + canisters: + - name: "my canister!" + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_wasm = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + match path.as_str() { + "icp.yaml" => { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + "my_canister_.wasm" => { + found_wasm = true; + } + _ => {} + } + } + + assert!(found_wasm, "my_canister_.wasm not found in bundle"); + assert!( + manifest_yaml.contains("my_canister_.wasm"), + "bundle manifest should reference sanitized wasm filename" + ); + assert!( + manifest_yaml.contains("my canister!"), + "bundle manifest should preserve original canister name" + ); +} + +/// An asset sync `dir` that contains `..` components but still resolves inside the project +/// must be accepted, and the `..` components must be lexically resolved before the path is +/// written into the archive or the rewritten manifest. +#[test] +fn bundle_normalizes_dotdot_within_project() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + // Assets live inside the project at ./shared-assets. The canister references them via + // a path that goes `tmp/..` and resolves to the same location — proof that `..` is + // handled lexically and doesn't have to point outside. + let assets_dir = project_dir.join("shared-assets"); + create_dir_all(&assets_dir).expect("failed to create shared-assets dir"); + write_string(&assets_dir.join("index.html"), "hello").expect("failed to write asset"); + create_dir_all(&project_dir.join("tmp")).expect("failed to create tmp dir"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: assets + dir: tmp/../shared-assets + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_asset = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + // Reject only `..` *path components* — substring matching would false-positive on + // a legitimate filename like `foo..bar.txt`. + if Path::new(&path) + .components() + .any(|c| matches!(c, Utf8Component::ParentDir)) + { + panic!("archive entry '{path}' contains a '..' path component"); + } + + match path.as_str() { + "icp.yaml" => { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + p if p.starts_with("my-canister/shared-assets/") => { + found_asset = true; + } + _ => {} + } + } + + assert!( + found_asset, + "asset not found under my-canister/shared-assets/ in bundle" + ); + + // Parse the rewritten manifest and inspect the actual sync dir field rather than + // substring-matching the whole YAML. + let parsed: serde_yaml::Value = + serde_yaml::from_str(&manifest_yaml).expect("manifest yaml is invalid"); + let dir = parsed["canisters"][0]["sync"]["steps"][0]["dir"] + .as_str() + .expect("expected sync step 0 to have a string `dir` field"); + assert_eq!(dir, "my-canister/shared-assets"); + assert!( + !Path::new(dir) + .components() + .any(|c| matches!(c, Utf8Component::ParentDir)), + "bundle manifest sync dir should not contain a '..' component: {dir}" + ); +} + +/// An asset sync step whose `dir` resolves *outside* the project directory must be rejected. +/// Bundles can only reference files inside the project so the produced archive is portable. +#[test] +fn bundle_rejects_source_outside_project() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + // Assets live one directory above the project — outside its tree. + let assets_dir = project_dir + .parent() + .expect("project dir has no parent") + .join("shared-assets"); + create_dir_all(&assets_dir).expect("failed to create sibling assets dir"); + write_string(&assets_dir.join("index.html"), "hello").expect("failed to write asset"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: assets + dir: ../shared-assets + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", "bundle.tar.gz"]) + .assert() + .failure() + .stderr(contains("my-canister").and(contains("outside the project directory"))); +} + +/// Bundle a canister with two plugin sync steps and verify the archive layout: per-plugin +/// wasm at `plugins/{canister}/{idx}.wasm`, preopened dirs at `plugins/{canister}/{idx}/dirs/`, +/// input files at `plugins/{canister}/{idx}/files/`. Also verify the rewritten manifest +/// references those paths and includes a sha256 matching the bundled plugin bytes. +#[test] +fn bundle_packages_plugin_sync_steps() { + let ctx = TestContext::new(); + let project_dir = ctx.create_project_dir("icp"); + let wasm_src = ctx.make_asset("example_icp_mo.wasm"); + + // Plugin bundling reads bytes and packages them — the wasm content does not need to be + // executable here, so any non-empty byte sequence works. + let plugin_a_bytes: &[u8] = b"\x00asm\x01\x00\x00\x00plugin-a"; + let plugin_b_bytes: &[u8] = b"\x00asm\x01\x00\x00\x00plugin-b"; + write(&project_dir.join("plugin-a.wasm"), plugin_a_bytes).expect("failed to write plugin-a"); + write(&project_dir.join("plugin-b.wasm"), plugin_b_bytes).expect("failed to write plugin-b"); + + let dir_a = project_dir.join("data-a"); + create_dir_all(&dir_a).expect("failed to create data-a"); + write_string(&dir_a.join("a.txt"), "alpha").expect("failed to write a.txt"); + + let dir_b = project_dir.join("data-b"); + create_dir_all(&dir_b).expect("failed to create data-b"); + write_string(&dir_b.join("b.txt"), "bravo").expect("failed to write b.txt"); + + write_string(&project_dir.join("config.toml"), "key=value").expect("failed to write config"); + + let pm = formatdoc! {r#" + canisters: + - name: my-canister + build: + steps: + - type: script + command: cp '{wasm_src}' "$ICP_WASM_OUTPUT_PATH" + sync: + steps: + - type: plugin + path: plugin-a.wasm + dirs: + - data-a + - type: plugin + path: plugin-b.wasm + dirs: + - data-b + files: + - config.toml + "#}; + + write_string(&project_dir.join("icp.yaml"), &pm).expect("failed to write project manifest"); + + let bundle_path = project_dir.join("bundle.tar.gz"); + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", bundle_path.as_str()]) + .assert() + .success(); + + let bundle_bytes = fs::read(bundle_path.as_std_path()).expect("failed to read bundle"); + let gz = GzDecoder::new(BufReader::new(bundle_bytes.as_slice())); + let mut archive = Archive::new(gz); + + let mut found_plugin_a: Option> = None; + let mut found_plugin_b: Option> = None; + let mut found_a_dir = false; + let mut found_b_dir = false; + let mut found_config = false; + let mut manifest_yaml = String::new(); + + for entry in archive.entries().expect("failed to read archive entries") { + let mut entry = entry.expect("failed to read archive entry"); + let path = entry + .path() + .expect("failed to get entry path") + .to_string_lossy() + .into_owned(); + + match path.as_str() { + "icp.yaml" => { + entry + .read_to_string(&mut manifest_yaml) + .expect("failed to read icp.yaml"); + } + "plugins/my-canister/0.wasm" => { + let mut buf = Vec::new(); + entry + .read_to_end(&mut buf) + .expect("failed to read plugin-a wasm"); + found_plugin_a = Some(buf); + } + "plugins/my-canister/1.wasm" => { + let mut buf = Vec::new(); + entry + .read_to_end(&mut buf) + .expect("failed to read plugin-b wasm"); + found_plugin_b = Some(buf); + } + "plugins/my-canister/0/dirs/data-a/a.txt" => found_a_dir = true, + "plugins/my-canister/1/dirs/data-b/b.txt" => found_b_dir = true, + "plugins/my-canister/1/files/config.toml" => found_config = true, + _ => {} + } + } + + assert_eq!( + found_plugin_a.as_deref(), + Some(plugin_a_bytes), + "plugin-a wasm bytes in archive don't match source" + ); + assert_eq!( + found_plugin_b.as_deref(), + Some(plugin_b_bytes), + "plugin-b wasm bytes in archive don't match source" + ); + assert!( + found_a_dir, + "plugin-a preopened dir not at plugins/my-canister/0/dirs/data-a/" + ); + assert!( + found_b_dir, + "plugin-b preopened dir not at plugins/my-canister/1/dirs/data-b/" + ); + assert!( + found_config, + "plugin-b input file not at plugins/my-canister/1/files/config.toml" + ); + + let parsed: serde_yaml::Value = + serde_yaml::from_str(&manifest_yaml).expect("manifest yaml is invalid"); + let steps = &parsed["canisters"][0]["sync"]["steps"]; + let plugin_a_sha = hex::encode(Sha256::digest(plugin_a_bytes)); + let plugin_b_sha = hex::encode(Sha256::digest(plugin_b_bytes)); + + assert_eq!( + steps[0]["path"].as_str(), + Some("plugins/my-canister/0.wasm") + ); + assert_eq!(steps[0]["sha256"].as_str(), Some(plugin_a_sha.as_str())); + assert_eq!( + steps[0]["dirs"][0].as_str(), + Some("plugins/my-canister/0/dirs/data-a") + ); + + assert_eq!( + steps[1]["path"].as_str(), + Some("plugins/my-canister/1.wasm") + ); + assert_eq!(steps[1]["sha256"].as_str(), Some(plugin_b_sha.as_str())); + assert_eq!( + steps[1]["dirs"][0].as_str(), + Some("plugins/my-canister/1/dirs/data-b") + ); + assert_eq!( + steps[1]["files"][0].as_str(), + Some("plugins/my-canister/1/files/config.toml") + ); +} + +/// Projects with script sync steps must be rejected with a clear error. +#[test] +fn bundle_rejects_script_sync_step() { + let ctx = TestContext::new(); + + let project_dir = ctx.create_project_dir("icp"); + + let pm = r#" +canisters: + - name: my-canister + build: + steps: + - type: script + command: echo build + sync: + steps: + - type: script + command: echo sync +"#; + + write_string(&project_dir.join("icp.yaml"), pm).expect("failed to write project manifest"); + + ctx.icp() + .current_dir(&project_dir) + .args(["project", "bundle", "--output", "bundle.tar.gz"]) + .assert() + .failure() + .stderr(contains("my-canister").and(contains("script sync step"))); +} diff --git a/crates/icp/src/canister/mod.rs b/crates/icp/src/canister/mod.rs index 12d8c3646..cb8aa81ce 100644 --- a/crates/icp/src/canister/mod.rs +++ b/crates/icp/src/canister/mod.rs @@ -12,7 +12,7 @@ pub mod recipe; pub mod sync; mod script; -mod wasm; +pub mod wasm; /// Controls who can read canister logs. /// Supports both string format ("controllers", "public") and object format ({ allowed_viewers: [...] }). @@ -145,40 +145,48 @@ impl From for LogVisibility { #[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] pub struct Settings { /// Controls who can read canister logs. + #[serde(skip_serializing_if = "Option::is_none")] pub log_visibility: Option, /// Compute allocation (0 to 100). Represents guaranteed compute capacity. + #[serde(skip_serializing_if = "Option::is_none")] pub compute_allocation: Option, /// Memory allocation in bytes. If unset, memory is allocated dynamically. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub memory_allocation: Option, /// Freezing threshold in seconds. Controls how long a canister can be inactive before being frozen. /// Supports duration suffixes in YAML: s, m, h, d, w (e.g. "30d" or "4w"). + #[serde(skip_serializing_if = "Option::is_none")] pub freezing_threshold: Option, /// Upper limit on cycles reserved for future resource payments. /// Memory allocations that would push the reserved balance above this limit will fail. /// Supports suffixes in YAML: k, m, b, t (e.g. "4t" or "4.3t"). - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub reserved_cycles_limit: Option, /// Wasm memory limit in bytes. Sets an upper bound for Wasm heap growth. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub wasm_memory_limit: Option, /// Wasm memory threshold in bytes. Triggers a callback when exceeded. /// Supports suffixes in YAML: kb, kib, mb, mib, gb, gib (e.g. "4gib" or "2.5kb"). + #[serde(skip_serializing_if = "Option::is_none")] pub wasm_memory_threshold: Option, /// Log memory limit in bytes (max 2 MiB). Oldest logs are purged when usage exceeds this value. /// Supports suffixes in YAML: kb, kib, mb, mib (e.g. "2mib" or "256kib"). Canister default is 4096 bytes. + #[serde(skip_serializing_if = "Option::is_none")] pub log_memory_limit: Option, /// Environment variables for the canister as key-value pairs. /// These variables are accessible within the canister and can be used to configure /// behavior without hardcoding values in the WASM module. + #[serde(skip_serializing_if = "Option::is_none")] pub environment_variables: Option>, } diff --git a/crates/icp/src/manifest/canister.rs b/crates/icp/src/manifest/canister.rs index ebafac6a2..f35496cb9 100644 --- a/crates/icp/src/manifest/canister.rs +++ b/crates/icp/src/manifest/canister.rs @@ -61,17 +61,17 @@ pub enum ManifestInitArgs { /// Represents the manifest describing a single canister. /// This struct is typically loaded from a `canister.yaml` file and defines /// the canister's name and how it should be built into WebAssembly. -#[derive(Clone, Debug, PartialEq, JsonSchema)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] pub struct CanisterManifest { /// The unique name of the canister as defined in this manifest. pub name: String, /// The configuration specifying the various settings when creating the canister. #[serde(default)] - #[schemars(with = "Option")] pub settings: Settings, /// Initialization arguments passed to the canister during installation. + #[serde(skip_serializing_if = "Option::is_none")] pub init_args: Option, #[serde(flatten)] @@ -236,7 +236,7 @@ impl<'de> Deserialize<'de> for CanisterManifest { } } -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(untagged)] pub enum Instructions { Recipe { @@ -249,6 +249,7 @@ pub enum Instructions { build: BuildSteps, /// The configuration specifying how to sync the canister + #[serde(skip_serializing_if = "Option::is_none")] sync: Option, }, } diff --git a/crates/icp/src/manifest/environment.rs b/crates/icp/src/manifest/environment.rs index abe73cc4d..3b9d67d29 100644 --- a/crates/icp/src/manifest/environment.rs +++ b/crates/icp/src/manifest/environment.rs @@ -1,18 +1,22 @@ use std::collections::HashMap; use schemars::JsonSchema; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, Serialize}; use crate::{canister::Settings, prelude::LOCAL}; use super::canister::ManifestInitArgs; -#[derive(Clone, Debug, PartialEq, Deserialize, JsonSchema)] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, JsonSchema)] pub struct EnvironmentInner { pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub network: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub canisters: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub settings: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub init_args: Option>, } @@ -100,6 +104,36 @@ impl<'de> Deserialize<'de> for EnvironmentManifest { } } +impl From<&EnvironmentManifest> for EnvironmentInner { + fn from(env: &EnvironmentManifest) -> Self { + let network = if env.network == LOCAL { + None + } else { + Some(env.network.clone()) + }; + + let canisters = match &env.canisters { + CanisterSelection::Everything => None, + CanisterSelection::Named(names) => Some(names.clone()), + CanisterSelection::None => Some(vec![]), + }; + + EnvironmentInner { + name: env.name.clone(), + network, + canisters, + settings: env.settings.clone(), + init_args: env.init_args.clone(), + } + } +} + +impl Serialize for EnvironmentManifest { + fn serialize(&self, serializer: S) -> Result { + EnvironmentInner::from(self).serialize(serializer) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/icp/src/manifest/mod.rs b/crates/icp/src/manifest/mod.rs index fe11aac7e..789fe4a86 100644 --- a/crates/icp/src/manifest/mod.rs +++ b/crates/icp/src/manifest/mod.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use snafu::prelude::*; use crate::fs; @@ -16,9 +16,15 @@ pub(crate) mod recipe; pub(crate) mod serde_helpers; pub use { - canister::{ArgsFormat, CanisterManifest, ManifestInitArgs}, + adapter::assets, + adapter::plugin, + adapter::prebuilt, + canister::{ + ArgsFormat, BuildStep, BuildSteps, CanisterManifest, Instructions, ManifestInitArgs, + SyncStep, SyncSteps, + }, environment::EnvironmentManifest, - network::NetworkManifest, + network::{ManagedMode, Mode, NetworkManifest}, project::ProjectManifest, }; @@ -41,6 +47,19 @@ pub enum Item { Manifest(T), } +/// Items in path form serialize back to a bare path string, *not* to the contents of the +/// referenced file. Callers that need a self-contained YAML output (e.g. `icp project bundle`) +/// must convert any `Item::Path` to `Item::Manifest` themselves by loading the referenced +/// manifest first. +impl Serialize for Item { + fn serialize(&self, serializer: S) -> Result { + match self { + Item::Path(p) => p.serialize(serializer), + Item::Manifest(m) => m.serialize(serializer), + } + } +} + impl<'de, T> Deserialize<'de> for Item where T: Deserialize<'de>, diff --git a/crates/icp/src/manifest/network.rs b/crates/icp/src/manifest/network.rs index 16ddc05d6..d892f9aa2 100644 --- a/crates/icp/src/manifest/network.rs +++ b/crates/icp/src/manifest/network.rs @@ -1,11 +1,11 @@ use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use url::Url; use crate::network::SubnetKind; /// A network definition for the project -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] pub struct NetworkManifest { pub name: String, @@ -13,20 +13,20 @@ pub struct NetworkManifest { pub configuration: Mode, } -#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(rename_all = "lowercase", tag = "mode")] pub enum Mode { Managed(Managed), Connected(Connected), } -#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, JsonSchema, Serialize)] pub struct Managed { #[serde(flatten)] pub mode: Box, } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema, Serialize)] #[serde(untagged, rename_all_fields = "kebab-case")] #[allow(clippy::large_enum_variant)] pub enum ManagedMode { @@ -36,46 +36,68 @@ pub enum ManagedMode { /// Port mappings in the format "host_port:container_port" port_mapping: Vec, /// Whether to delete the container when the network stops + #[serde(skip_serializing_if = "Option::is_none")] rm_on_exit: Option, /// Command line arguments to pass to the container's entrypoint - #[serde(alias = "cmd", alias = "command")] + #[serde( + alias = "cmd", + alias = "command", + skip_serializing_if = "Option::is_none" + )] args: Option>, /// Entrypoint to use for the container + #[serde(skip_serializing_if = "Option::is_none")] entrypoint: Option>, /// Environment variables to set in the container in VAR=VALUE format (or VAR to inherit from host) + #[serde(skip_serializing_if = "Option::is_none")] environment: Option>, /// Volumes to mount into the container in the format name:container_path[:options] + #[serde(skip_serializing_if = "Option::is_none")] volumes: Option>, /// The platform to use for the container (e.g. linux/amd64) + #[serde(skip_serializing_if = "Option::is_none")] platform: Option, /// The user to run the container as in the format user[:group] + #[serde(skip_serializing_if = "Option::is_none")] user: Option, /// The size of /dev/shm in bytes + #[serde(skip_serializing_if = "Option::is_none")] shm_size: Option, /// The status directory inside the container. Defaults to /app/status + #[serde(skip_serializing_if = "Option::is_none")] status_dir: Option, /// Bind mounts to add to the container in the format relative_host_path:container_path[:options] + #[serde(skip_serializing_if = "Option::is_none")] mounts: Option>, /// Extra hosts entries for Docker networking (e.g. "host.docker.internal:host-gateway") + #[serde(skip_serializing_if = "Option::is_none")] extra_hosts: Option>, }, Launcher { /// HTTP gateway configuration + #[serde(skip_serializing_if = "Option::is_none")] gateway: Option, /// Artificial delay to add to every update call + #[serde(skip_serializing_if = "Option::is_none")] artificial_delay_ms: Option, /// Set up the Internet Identity canister. Makes internet identity available at /// id.ai.localhost: + #[serde(skip_serializing_if = "Option::is_none")] ii: Option, /// Set up the NNS + #[serde(skip_serializing_if = "Option::is_none")] nns: Option, /// Configure the list of subnets (one application subnet by default) + #[serde(skip_serializing_if = "Option::is_none")] subnets: Option>, /// Bitcoin P2P node addresses to connect to (e.g. "127.0.0.1:18444") + #[serde(skip_serializing_if = "Option::is_none")] bitcoind_addr: Option>, /// Dogecoin P2P node addresses to connect to + #[serde(skip_serializing_if = "Option::is_none")] dogecoind_addr: Option>, /// The version of icp-cli-network-launcher to use. Defaults to the latest released version. Launcher versions correspond to published PocketIC or IC-OS releases. + #[serde(skip_serializing_if = "Option::is_none")] version: Option, }, } @@ -95,18 +117,19 @@ impl Default for ManagedMode { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(rename_all = "kebab-case")] pub struct Connected { #[serde(flatten)] pub endpoints: Endpoints, /// The root key of this network + #[serde(skip_serializing_if = "Option::is_none")] #[schemars(with = "Option", regex(pattern = "^[0-9a-f]{266}$"))] pub root_key: Option, } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(untagged, rename_all_fields = "kebab-case")] pub enum Endpoints { Explicit { @@ -114,6 +137,7 @@ pub enum Endpoints { /// otherwise icp-cli will fall back to ?canisterId= query parameters which are frequently brittle in frontend code. /// /// If no HTTP gateway endpoint is provided, canister URLs will not be printed in deploy operations. + #[serde(skip_serializing_if = "Option::is_none")] http_gateway_url: Option, /// The URL of the API endpoint. Should support the standard API routes (e.g. /api/v3). api_url: Url, @@ -126,7 +150,7 @@ pub enum Endpoints { }, } -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(try_from = "String", into = "String")] pub struct RootKey(pub Vec); @@ -145,13 +169,16 @@ impl From for String { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct Gateway { /// Network interface for the gateway. Defaults to 127.0.0.1 + #[serde(skip_serializing_if = "Option::is_none")] pub bind: Option, /// Domains the gateway should respond to. Automatically includes localhost if applicable. + #[serde(skip_serializing_if = "Option::is_none")] pub domains: Option>, /// Port for the gateway to listen on. Defaults to 8000 + #[serde(skip_serializing_if = "Option::is_none")] pub port: Option, } diff --git a/crates/icp/src/manifest/project.rs b/crates/icp/src/manifest/project.rs index b158cbbd0..310b57f9f 100644 --- a/crates/icp/src/manifest/project.rs +++ b/crates/icp/src/manifest/project.rs @@ -1,23 +1,20 @@ use schemars::JsonSchema; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::manifest::{ Item, canister::CanisterManifest, environment::EnvironmentManifest, network::NetworkManifest, }; -#[derive(Debug, PartialEq, JsonSchema, Deserialize)] +#[derive(Debug, PartialEq, JsonSchema, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct ProjectManifest { #[serde(default)] - #[schemars(with = "Option>>")] pub canisters: Vec>, #[serde(default)] - #[schemars(with = "Option>>")] pub networks: Vec>, #[serde(default)] - #[schemars(with = "Option>>")] pub environments: Vec>, } diff --git a/crates/icp/src/manifest/recipe.rs b/crates/icp/src/manifest/recipe.rs index 107ba0fbe..e8bff9d09 100644 --- a/crates/icp/src/manifest/recipe.rs +++ b/crates/icp/src/manifest/recipe.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize, de::Error as _}; /// Represents the accepted values for a recipe type in /// the canister manifest -#[derive(Clone, Debug, PartialEq, JsonSchema, Serialize)] +#[derive(Clone, Debug, PartialEq, JsonSchema)] #[schemars(from = "String")] pub enum RecipeType { /// path to a locally defined recipe @@ -91,7 +91,13 @@ impl From for String { } } -#[derive(Clone, Debug, Deserialize, PartialEq, JsonSchema)] +impl Serialize for RecipeType { + fn serialize(&self, serializer: S) -> Result { + String::from(self.clone()).serialize(serializer) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct Recipe { /// An identifier for a recipe, it can have one of the following formats: /// @@ -108,8 +114,8 @@ pub struct Recipe { #[schemars(with = "String")] pub recipe_type: RecipeType, - #[serde(default)] - #[schemars(with = "Option>")] + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[schemars(with = "HashMap")] pub configuration: HashMap, /// Optional sha256 checksum for the recipe template. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 67dcd9c0a..624749c43 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -89,7 +89,7 @@ This document contains the help content for the `icp` command-line program. * `identity` — Manage your identities * `network` — Launch and manage local test networks * `new` — Create a new ICP project from a template -* `project` — Display information about the current project +* `project` — Manage the current project * `settings` — Configure user settings * `sync` — Synchronize canisters * `token` — Perform token transactions @@ -1449,7 +1449,7 @@ Under the hood templates are generated with `cargo-generate`. See the cargo-gene ## `icp project` -Display information about the current project +Manage the current project **Usage:** `icp project ` diff --git a/docs/schemas/canister-yaml-schema.json b/docs/schemas/canister-yaml-schema.json index 0dcda0934..5bf99c59b 100644 --- a/docs/schemas/canister-yaml-schema.json +++ b/docs/schemas/canister-yaml-schema.json @@ -327,11 +327,7 @@ "properties": { "configuration": { "additionalProperties": true, - "default": {}, - "type": [ - "object", - "null" - ] + "type": "object" }, "sha256": { "description": "Optional sha256 checksum for the recipe template.\nIf provided, the integrity of the recipe will be verified against this hash", @@ -437,7 +433,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { @@ -584,25 +579,8 @@ "type": "string" }, "settings": { - "anyOf": [ - { - "$ref": "#/$defs/Settings" - }, - { - "type": "null" - } - ], - "default": { - "compute_allocation": null, - "environment_variables": null, - "freezing_threshold": null, - "log_memory_limit": null, - "log_visibility": null, - "memory_allocation": null, - "reserved_cycles_limit": null, - "wasm_memory_limit": null, - "wasm_memory_threshold": null - }, + "$ref": "#/$defs/Settings", + "default": {}, "description": "The configuration specifying the various settings when creating the canister." } }, diff --git a/docs/schemas/environment-yaml-schema.json b/docs/schemas/environment-yaml-schema.json index 825958121..0d111b82f 100644 --- a/docs/schemas/environment-yaml-schema.json +++ b/docs/schemas/environment-yaml-schema.json @@ -206,7 +206,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { diff --git a/docs/schemas/icp-yaml-schema.json b/docs/schemas/icp-yaml-schema.json index 0f0325c76..aef787025 100644 --- a/docs/schemas/icp-yaml-schema.json +++ b/docs/schemas/icp-yaml-schema.json @@ -255,25 +255,8 @@ "type": "string" }, "settings": { - "anyOf": [ - { - "$ref": "#/$defs/Settings" - }, - { - "type": "null" - } - ], - "default": { - "compute_allocation": null, - "environment_variables": null, - "freezing_threshold": null, - "log_memory_limit": null, - "log_visibility": null, - "memory_allocation": null, - "reserved_cycles_limit": null, - "wasm_memory_limit": null, - "wasm_memory_threshold": null - }, + "$ref": "#/$defs/Settings", + "default": {}, "description": "The configuration specifying the various settings when creating the canister." } }, @@ -811,11 +794,7 @@ "properties": { "configuration": { "additionalProperties": true, - "default": {}, - "type": [ - "object", - "null" - ] + "type": "object" }, "sha256": { "description": "Optional sha256 checksum for the recipe template.\nIf provided, the integrity of the recipe will be verified against this hash", @@ -921,7 +900,6 @@ "type": "null" } ], - "default": null, "description": "Upper limit on cycles reserved for future resource payments.\nMemory allocations that would push the reserved balance above this limit will fail.\nSupports suffixes in YAML: k, m, b, t (e.g. \"4t\" or \"4.3t\")." }, "wasm_memory_limit": { @@ -1030,31 +1008,25 @@ "description": "Schema for ProjectManifest", "properties": { "canisters": { + "default": [], "items": { "$ref": "#/$defs/Item" }, - "type": [ - "array", - "null" - ] + "type": "array" }, "environments": { + "default": [], "items": { "$ref": "#/$defs/Item3" }, - "type": [ - "array", - "null" - ] + "type": "array" }, "networks": { + "default": [], "items": { "$ref": "#/$defs/Item2" }, - "type": [ - "array", - "null" - ] + "type": "array" } }, "title": "ProjectManifest",