Include Markdown and notebook files with scoped references

README.md

@@ -283,6 +283,58 @@ signed = fill_md(src, dict(amt="$1", name="Sam", grants=[dict(d="Jan", n="100"),
 
 The result remains `md` that can contain unresolved template tokens. A partially filled document is still a valid template. For example, fill the grant details now and the signing dates in a later call.
 
+### Include a file
+
+`mdhtml.fill.include(path)` returns Markdown from a `.md` file or a notebook's exported note cells. It excludes frontmatter and notebook code and outputs. It does not execute the included file. Paths are relative to the current working directory.
+
+Given `camera.md`:
+
+```md
+# Camera
+
+## Setup {#sec-setup}
+
+{{operator}} sets {{resolution}}. See [@sec-setup].
+```
+
+Include it in a recording guide:
+
+```python
+from mdhtml.fill import include
+
+camera = include("camera.md", keep={"operator": "camera_operator"})
+```
+
+The result is:
+
+```md
+::: {.include from="camera" scope="__camera"}
+
+# Camera
+
+## Setup {#sec-setup}
+
+{{camera_operator}} sets ________________. See [@sec-setup].
+
+:::
+```
+
+`md2mdhtml(camera)` changes the heading ID to `sec-setup__camera` and the link target to `#sec-setup__camera`. Another file can use the same local ID under a different scope. Write `[@sec-setup__camera]` to refer to this heading from outside the include.
+
+Fields become sixteen underscores by default. Pass `keep=["operator"]` to retain a field or a mapping to rename it. Ordinary conditional and loop markers remain available for later filling. The mapping can rename those markers too.
+
+`scope` is appended to each local ID as written. It defaults to `__` followed by the file stem. Pass a distinct scope when including the same file twice. A scope cannot contain whitespace. For a file named `Camera guide.md`, use `scope="__camera"`.
+
+For notebooks, `skip=1` drops the first exported note after frontmatter removal. Hidden notes and code outputs are excluded. `skip` does not change Markdown files.
+
+Page breaks belong in the parent document:
+
+```python
+source = f'{camera}\n\n<br type="page">\n\n{include("mic.md")}'
+```
+
+The returned `Md` string displays as prose in notebook output cells. Use `template_md(path, skip=0)` to load the source without blanking or wrapping. Use `rewrite(md, keep=())` to blank or rename fields in an existing string.
+
 ### Mutable MDHTML DOM
 
 `md2dom` converts user-authored `md` to a mutable [fast5ever](https://github.com/AnswerDotAI/fast5ever) DOM. This forgiving import path normalizes provisional HTML, including malformed nesting; that recovery behavior is not the definition of valid `md`. fast5ever uses html5ever's WHATWG parsing and serialization with an arena tree:
@@ -630,3 +682,4 @@ maturin develop && pytest -q
 ```
 
 `tests/test_conformance.py` renders the fixtures under `tests/source/` and compares normalized HTML trees. Run `pytest tests/test_conformance.py -v` to see results by example id.
+

python/mdhtml/export.py

@@ -4,6 +4,7 @@
 from html import escape
 from pathlib import Path
 
+from fast5ever import parse_fragment
 from ._native import HeadingNums, Resolver as _Resolver, group_plan, anchors, ref_tokens, ref_variant, target_kind
 from ._native import REFTYPES, SCHEMES, decode_raw as _decode_raw, dialect_css, export_html as _export_html, math_js as _math_js
 

python/mdhtml/fill.py

@@ -21,7 +21,8 @@
 from untrusted sources must be sanitized upstream, since a value containing `{{other_field}}`
 resolves against the data (injected code never runs). A literal `{{` in prose belongs in a
 backtick code span, which the scanner never enters."""
-import yaml, sys
+import yaml, sys, re
+from html import escape
 from bisect import bisect_left
 from dataclasses import astuple, is_dataclass
 from pathlib import Path
@@ -36,7 +37,7 @@
 from .md import Md, _normalize_offsets
 from ._cli import read_src
 
-__all__ = ["tokens", "fill_md", "instantiate", "instantiate_nb", "frontmatter_data"]
+__all__ = ["tokens", "fill_md", "instantiate", "instantiate_nb", "frontmatter_data", "BLANK", "template_md", "rewrite", "include"]
 
 _MAX_DEPTH = 10
 _MISSING = object()
@@ -270,6 +271,42 @@ def frontmatter_data(src):
 
 
 
+BLANK = '_' * 16
+
+
+def template_md(path, skip=0):
+    "Read Markdown or exported notebook notes, excluding frontmatter and the first `skip` notes."
+    path = Path(path)
+    if path.suffix == '.ipynb':
+        notes = [m for m in read_ipynb(path).messages if m.msg_type == 'note' and m.exported]
+        if notes: notes[0].content = frontmatter(notes[0].content)[1]
+        return dlg2md([m for m in notes if m.content.strip()][skip:])
+    return frontmatter(path.read_text(encoding='utf-8'))[1]
+
+
+def rewrite(md, keep=()):
+    "Blank fields unless kept or renamed; leave ranges available for later filling."
+    md, _ = _normalize_offsets(md)
+    kept = dict(keep) if isinstance(keep, dict) else {k: k for k in keep}
+    for t in reversed(tokens(md)):
+        name = t['name']
+        if t['kind'] == 'var': rep = '{{' + kept[name] + '}}' if name in kept else BLANK
+        elif name in kept and kept[name] != name: rep = t['source'].replace(name, kept[name], 1)
+        else: continue
+        md = md[:t['start']] + rep + md[t['end']:]
+    return md
+
+
+def include(path, keep=(), skip=0, scope=None):
+    "Include Markdown or exported notebook notes, blanking fields unless kept; never execute code."
+    path = Path(path)
+    body = rewrite(template_md(path, skip), keep)
+    scope = '__' + path.stem if scope is None else scope
+    if any(c.isspace() for c in scope): raise ValueError('include scope must not contain whitespace; pass scope explicitly')
+    name, scope = escape(path.stem, quote=True), escape(scope, quote=True)
+    return Md(f'::: {{.include from="{name}" scope="{scope}"}}\n\n{body}\n\n:::', [])
+
+
 def _capture_shell():
     "A `CaptureShell`, imported lazily: a bare install carries no execnb or IPython (the `fill` extra provides them)."
     try: from execnb.shell import CaptureShell
@@ -359,3 +396,6 @@ def main(
     else: res = instantiate(read_src(file), values, strict=not lenient, dest=out)
     for w in res.warnings: print(w, file=sys.stderr)
     if out is None: sys.stdout.write(res)
+
+
+

python/mdhtml/md.py

@@ -50,6 +50,7 @@ def __new__(cls, s, warnings):
         return self
 
     def __getnewargs__(self): return (str(self), self.warnings)
+    def _repr_markdown_(self): return str(self)
 
 
 def _is_ial(line):

src/block.rs

@@ -2083,6 +2083,7 @@ impl<'a> ContainerBuilder<'a> {
                 let start_line = self.nodes[idx].start_line;
                 let mut blocks = self.finish_children(idx, parser, depth);
                 if spliced { return blocks; }
+                if tag == "div" && let Some(attrs) = crate::scopes::attrs(&open) { return vec![DraftBlock::Div { attrs, children: blocks }]; }
                 let mut out = vec![Self::draft_raw(&format!("{open}\n"), start_line, parser)];
                 out.append(&mut blocks);
                 if closed { out.push(Self::draft_raw(&format!("</{tag}>\n"), start_line, parser)); }

src/export_html.rs

@@ -317,7 +317,7 @@ impl Exporter {
             .iter()
             .map(|&a| {
                 let href = self.dom.attr(a, "href").unwrap_or("#");
-                href.get(1..).unwrap_or("").split('-').next().unwrap_or("").to_string()
+                href.trim_start_matches('#').split('-').next().unwrap_or("").to_string()
             })
             .collect();
         let mut out = Vec::new();

src/lib.rs

@@ -19,6 +19,7 @@ pub mod markdown;
 mod render;
 pub mod resolve;
 pub mod scan;
+mod scopes;
 pub mod template;
 pub mod wikitext;
 mod wrap;
@@ -106,11 +107,12 @@ pub fn parse(src: &str, options: &Options) -> Document {
     };
     let mut doc = block::parse_document(owned.as_deref().unwrap_or(src), options);
     doc.meta = meta;
+    scopes::validate(&mut doc);
     doc
 }
 pub fn block_spans(src: &str, options: &Options) -> Vec<BlockSpan> { block::parse_block_spans(src, options) }
 
-/// Serialize a parsed [`Document`] to its MDHTML fragment.
+/// Serialize a parsed [`Document`] to MDHTML, qualifying include-local IDs and links.
 pub fn render(doc: &Document) -> String { render::render_document(doc) }
 
 /// Inline edit nodes (images, math, xrefs, attrs, raw inlines, template tokens)

src/render.rs

@@ -112,13 +112,17 @@ impl<'a> Renderer<'a> {
                 out.push('\n');
             }
             Block::Html { raw, tokens } => {
+                let start = out.len();
                 let mut at = 0;
                 for t in tokens {
                     out.push_str(&raw[at..t.start]);
                     template_html(&t.syntax, &t.body, t.kind, &t.name, out);
                     at = t.end;
                 }
                 out.push_str(&raw[at..]);
+                let html = crate::scopes::qualify(&out[start..], None);
+                out.truncate(start);
+                out.push_str(&html);
             }
             Block::ThematicBreak { attrs } => {
                 out.push_str("<hr");
@@ -141,10 +145,20 @@ impl<'a> Renderer<'a> {
                 out.push_str("\n</figure>\n");
             }
             Block::Div { attrs, children } => {
+                let scope = attrs.classes.iter().any(|c| c == "include")
+                    .then(|| attrs.pairs.iter().find(|(k, _)| k == "scope").map(|(_, v)| v.as_str())).flatten();
+                let mut attrs = attrs.clone();
+                if scope.is_some() { attrs.pairs.retain(|(k, _)| k != "scope"); }
                 out.push_str("<div");
-                attrs_html(attrs, out);
+                attrs_html(&attrs, out);
                 out.push_str(">\n");
+                let start = out.len();
                 self.blocks(children, out);
+                if let Some(scope) = scope {
+                    let html = crate::scopes::qualify(&out[start..], Some(scope));
+                    out.truncate(start);
+                    out.push_str(&html);
+                }
                 out.push_str("</div>\n");
             }
             Block::Math { attrs, tex, .. } => {

src/scopes.rs

@@ -0,0 +1,77 @@
+use std::collections::{HashMap, HashSet};
+
+use fast5ever::{DOCUMENT, parse_fragment};
+use crate::{Attr, Block, Diagnostic, Document};
+
+pub(crate) fn qualify(src: &str, suffix: Option<&str>) -> String {
+    if suffix.is_none() && !src.to_ascii_lowercase().contains("scope") { return src.into(); }
+    let mut dom = parse_fragment(src, "body");
+    let mut includes: Vec<_> = dom.descendants(DOCUMENT).into_iter().filter_map(|e| {
+        if !dom.has_class(e, "include") { return None; }
+        dom.attr(e, "scope").map(|s| (e, s.to_string()))
+    }).collect();
+    includes.reverse();
+    if let Some(suffix) = suffix { includes.push((DOCUMENT, suffix.into())); }
+    if includes.is_empty() { return src.into(); }
+    for (inc, scope) in includes {
+        let _ = dom.remove_attr(inc, "scope");
+        let els: Vec<_> = dom.descendants(inc).into_iter().skip(1).collect();
+        let ids: HashMap<_, _> = els.iter().filter_map(|&e| {
+            dom.attr(e, "id").map(|id| (id.to_string(), format!("{id}{scope}")))
+        }).collect();
+        for e in els {
+            if let Some(id) = dom.attr(e, "id").and_then(|id| ids.get(id)) { let _ = dom.set_attr(e, "id", id); }
+            if let Some(id) = dom.attr(e, "href").and_then(|s| s.strip_prefix('#')).and_then(|id| ids.get(id)) {
+                let _ = dom.set_attr(e, "href", &format!("#{id}"));
+            }
+        }
+    }
+    dom.to_html(DOCUMENT)
+}
+
+pub(crate) fn validate(doc: &mut Document) {
+    fn check(scope: &str, seen: &mut HashSet<String>, warnings: &mut Vec<Diagnostic>) {
+        let message = if scope.is_empty() { Some("include scope must not be empty") }
+            else if scope.chars().any(char::is_whitespace) { Some("include scope must not contain whitespace") }
+            else if !seen.insert(scope.into()) { Some("include scopes must be unique") }
+            else { None };
+        if let Some(message) = message { warnings.push(Diagnostic::warning("include-scope", message)); }
+    }
+    fn walk(blocks: &[Block], seen: &mut HashSet<String>, warnings: &mut Vec<Diagnostic>) {
+        for block in blocks {
+            match block {
+                Block::Div { attrs, children } => {
+                    if attrs.classes.iter().any(|c| c == "include") {
+                        for (_, scope) in attrs.pairs.iter().filter(|(k, _)| k == "scope") { check(scope, seen, warnings); }
+                    }
+                    walk(children, seen, warnings);
+                }
+                Block::BlockQuote { children, .. } => walk(children, seen, warnings),
+                Block::List { items, .. } => for item in items { walk(&item.blocks, seen, warnings); },
+                Block::Html { raw, .. } if raw.to_ascii_lowercase().contains("scope") => {
+                    let dom = parse_fragment(raw, "body");
+                    for e in dom.descendants(DOCUMENT) {
+                        if dom.has_class(e, "include") && let Some(scope) = dom.attr(e, "scope") { check(scope, seen, warnings); }
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+    let mut seen = HashSet::new();
+    walk(&doc.blocks, &mut seen, &mut doc.diagnostics);
+    for note in &doc.footnotes { walk(&note.blocks, &mut seen, &mut doc.diagnostics); }
+}
+
+pub(crate) fn attrs(open: &str) -> Option<Attr> {
+    if !open.to_ascii_lowercase().contains("scope") { return None; }
+    let dom = parse_fragment(open, "body");
+    let e = dom.descendants(DOCUMENT).into_iter().find(|&e| dom.has_class(e, "include") && dom.attr(e, "scope").is_some())?;
+    let fast5ever::NodeData::Element { attrs, .. } = &dom.get(e).data else { return None; };
+    let mut result = Attr::default();
+    for (name, value) in attrs {
+        let key = name.prefix.as_ref().map_or_else(|| name.local.to_string(), |p| format!("{p}:{}", name.local));
+        result.set_pair(key, value);
+    }
+    Some(result)
+}

tests/canonical.rs

@@ -129,3 +129,33 @@ decision-making stays plain.
     assert!(canonical.contains("  \\- continuation that looks like a nested list"));
     assert_eq!(render(&parse(&canonical, &options)), render(&document), "canonical Markdown:\n{canonical}");
 }
+
+#[test]
+fn include_scopes_qualify_ids_before_export() {
+    let source = r#":::: {.include scope=__camera}
+# Camera {#sec-camera}
+See [@sec-setup__lens].
+
+::: {.include scope=__lens}
+## Lens {#sec-setup}
+See [@sec-setup] and [@sec-camera].
+:::
+::::
+
+See [@sec-setup__lens__camera].
+"#;
+    let html = render(&parse(source, &Options::default()));
+    assert!(html.contains(r#"id="sec-setup__lens__camera""#));
+    assert!(html.contains(r#"id="sec-camera__camera""#));
+    assert_eq!(html.matches(r##"href="#sec-setup__lens__camera""##).count(), 3);
+    assert!(html.contains(r##"href="#sec-camera__camera""##));
+    assert!(!html.contains("scope="));
+    for (scope, error) in [("__camera", "must be unique"), ("\"\"", "must not be empty")] {
+        let invalid = source.replace("scope=__lens", &format!("scope={scope}"));
+        assert!(parse(&invalid, &Options::default()).diagnostics.iter().any(|d| d.message.contains(error)));
+    }
+    let raw = r##"<div class="include" SCOPE = "__mic"><h2 id="sec-setup">Setup</h2><a href="#sec-setup">Here</a></div>"##;
+    let html = render(&parse(raw, &Options::default()));
+    assert!(html.contains(r#"id="sec-setup__mic""#));
+    assert!(html.contains(r##"href="#sec-setup__mic""##));
+}