diff --git a/README.md b/README.md index 235a3ba..47c77f0 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,58 @@ signed = fill_md(src, dict(amt="$1", name="Sam", grants=[dict(d="Jan", n="100"), The result remains `md` that can contain unresolved template tokens. A partially filled document is still a valid template. For example, fill the grant details now and the signing dates in a later call. +### Include a file + +`mdhtml.fill.include(path)` returns Markdown from a `.md` file or a notebook's exported note cells. It excludes frontmatter and notebook code and outputs. It does not execute the included file. Paths are relative to the current working directory. + +Given `camera.md`: + +```md +# Camera + +## Setup {#sec-setup} + +{{operator}} sets {{resolution}}. See [@sec-setup]. +``` + +Include it in a recording guide: + +```python +from mdhtml.fill import include + +camera = include("camera.md", keep={"operator": "camera_operator"}) +``` + +The result is: + +```md +::: {.include from="camera" scope="__camera"} + +# Camera + +## Setup {#sec-setup} + +{{camera_operator}} sets ________________. See [@sec-setup]. + +::: +``` + +`md2mdhtml(camera)` changes the heading ID to `sec-setup__camera` and the link target to `#sec-setup__camera`. Another file can use the same local ID under a different scope. Write `[@sec-setup__camera]` to refer to this heading from outside the include. + +Fields become sixteen underscores by default. Pass `keep=["operator"]` to retain a field or a mapping to rename it. Ordinary conditional and loop markers remain available for later filling. The mapping can rename those markers too. + +`scope` is appended to each local ID as written. It defaults to `__` followed by the file stem. Pass a distinct scope when including the same file twice. A scope cannot contain whitespace. For a file named `Camera guide.md`, use `scope="__camera"`. + +For notebooks, `skip=1` drops the first exported note after frontmatter removal. Hidden notes and code outputs are excluded. `skip` does not change Markdown files. + +Page breaks belong in the parent document: + +```python +source = f'{camera}\n\n
\n\n{include("mic.md")}' +``` + +The returned `Md` string displays as prose in notebook output cells. Use `template_md(path, skip=0)` to load the source without blanking or wrapping. Use `rewrite(md, keep=())` to blank or rename fields in an existing string. + ### Mutable MDHTML DOM `md2dom` converts user-authored `md` to a mutable [fast5ever](https://github.com/AnswerDotAI/fast5ever) DOM. This forgiving import path normalizes provisional HTML, including malformed nesting; that recovery behavior is not the definition of valid `md`. fast5ever uses html5ever's WHATWG parsing and serialization with an arena tree: @@ -630,3 +682,4 @@ maturin develop && pytest -q ``` `tests/test_conformance.py` renders the fixtures under `tests/source/` and compares normalized HTML trees. Run `pytest tests/test_conformance.py -v` to see results by example id. + diff --git a/python/mdhtml/export.py b/python/mdhtml/export.py index 7679d66..9367491 100644 --- a/python/mdhtml/export.py +++ b/python/mdhtml/export.py @@ -4,6 +4,7 @@ from html import escape from pathlib import Path +from fast5ever import parse_fragment from ._native import HeadingNums, Resolver as _Resolver, group_plan, anchors, ref_tokens, ref_variant, target_kind from ._native import REFTYPES, SCHEMES, decode_raw as _decode_raw, dialect_css, export_html as _export_html, math_js as _math_js diff --git a/python/mdhtml/fill.py b/python/mdhtml/fill.py index 9452331..cb17a26 100644 --- a/python/mdhtml/fill.py +++ b/python/mdhtml/fill.py @@ -21,7 +21,8 @@ from untrusted sources must be sanitized upstream, since a value containing `{{other_field}}` resolves against the data (injected code never runs). A literal `{{` in prose belongs in a backtick code span, which the scanner never enters.""" -import yaml, sys +import yaml, sys, re +from html import escape from bisect import bisect_left from dataclasses import astuple, is_dataclass from pathlib import Path @@ -36,7 +37,7 @@ from .md import Md, _normalize_offsets from ._cli import read_src -__all__ = ["tokens", "fill_md", "instantiate", "instantiate_nb", "frontmatter_data"] +__all__ = ["tokens", "fill_md", "instantiate", "instantiate_nb", "frontmatter_data", "BLANK", "template_md", "rewrite", "include"] _MAX_DEPTH = 10 _MISSING = object() @@ -270,6 +271,42 @@ def frontmatter_data(src): +BLANK = '_' * 16 + + +def template_md(path, skip=0): + "Read Markdown or exported notebook notes, excluding frontmatter and the first `skip` notes." + path = Path(path) + if path.suffix == '.ipynb': + notes = [m for m in read_ipynb(path).messages if m.msg_type == 'note' and m.exported] + if notes: notes[0].content = frontmatter(notes[0].content)[1] + return dlg2md([m for m in notes if m.content.strip()][skip:]) + return frontmatter(path.read_text(encoding='utf-8'))[1] + + +def rewrite(md, keep=()): + "Blank fields unless kept or renamed; leave ranges available for later filling." + md, _ = _normalize_offsets(md) + kept = dict(keep) if isinstance(keep, dict) else {k: k for k in keep} + for t in reversed(tokens(md)): + name = t['name'] + if t['kind'] == 'var': rep = '{{' + kept[name] + '}}' if name in kept else BLANK + elif name in kept and kept[name] != name: rep = t['source'].replace(name, kept[name], 1) + else: continue + md = md[:t['start']] + rep + md[t['end']:] + return md + + +def include(path, keep=(), skip=0, scope=None): + "Include Markdown or exported notebook notes, blanking fields unless kept; never execute code." + path = Path(path) + body = rewrite(template_md(path, skip), keep) + scope = '__' + path.stem if scope is None else scope + if any(c.isspace() for c in scope): raise ValueError('include scope must not contain whitespace; pass scope explicitly') + name, scope = escape(path.stem, quote=True), escape(scope, quote=True) + return Md(f'::: {{.include from="{name}" scope="{scope}"}}\n\n{body}\n\n:::', []) + + def _capture_shell(): "A `CaptureShell`, imported lazily: a bare install carries no execnb or IPython (the `fill` extra provides them)." try: from execnb.shell import CaptureShell @@ -359,3 +396,6 @@ def main( else: res = instantiate(read_src(file), values, strict=not lenient, dest=out) for w in res.warnings: print(w, file=sys.stderr) if out is None: sys.stdout.write(res) + + + diff --git a/python/mdhtml/md.py b/python/mdhtml/md.py index 3f9db06..0700c57 100644 --- a/python/mdhtml/md.py +++ b/python/mdhtml/md.py @@ -50,6 +50,7 @@ def __new__(cls, s, warnings): return self def __getnewargs__(self): return (str(self), self.warnings) + def _repr_markdown_(self): return str(self) def _is_ial(line): diff --git a/src/block.rs b/src/block.rs index 91e44bb..2d74f77 100644 --- a/src/block.rs +++ b/src/block.rs @@ -2083,6 +2083,7 @@ impl<'a> ContainerBuilder<'a> { let start_line = self.nodes[idx].start_line; let mut blocks = self.finish_children(idx, parser, depth); if spliced { return blocks; } + if tag == "div" && let Some(attrs) = crate::scopes::attrs(&open) { return vec![DraftBlock::Div { attrs, children: blocks }]; } let mut out = vec![Self::draft_raw(&format!("{open}\n"), start_line, parser)]; out.append(&mut blocks); if closed { out.push(Self::draft_raw(&format!("\n"), start_line, parser)); } diff --git a/src/export_html.rs b/src/export_html.rs index 0981f17..8030248 100644 --- a/src/export_html.rs +++ b/src/export_html.rs @@ -317,7 +317,7 @@ impl Exporter { .iter() .map(|&a| { let href = self.dom.attr(a, "href").unwrap_or("#"); - href.get(1..).unwrap_or("").split('-').next().unwrap_or("").to_string() + href.trim_start_matches('#').split('-').next().unwrap_or("").to_string() }) .collect(); let mut out = Vec::new(); diff --git a/src/lib.rs b/src/lib.rs index dbec41f..b3e964a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ pub mod markdown; mod render; pub mod resolve; pub mod scan; +mod scopes; pub mod template; pub mod wikitext; mod wrap; @@ -106,11 +107,12 @@ pub fn parse(src: &str, options: &Options) -> Document { }; let mut doc = block::parse_document(owned.as_deref().unwrap_or(src), options); doc.meta = meta; + scopes::validate(&mut doc); doc } pub fn block_spans(src: &str, options: &Options) -> Vec { block::parse_block_spans(src, options) } -/// Serialize a parsed [`Document`] to its MDHTML fragment. +/// Serialize a parsed [`Document`] to MDHTML, qualifying include-local IDs and links. pub fn render(doc: &Document) -> String { render::render_document(doc) } /// Inline edit nodes (images, math, xrefs, attrs, raw inlines, template tokens) diff --git a/src/render.rs b/src/render.rs index 95aaa66..a208e6a 100644 --- a/src/render.rs +++ b/src/render.rs @@ -112,6 +112,7 @@ impl<'a> Renderer<'a> { out.push('\n'); } Block::Html { raw, tokens } => { + let start = out.len(); let mut at = 0; for t in tokens { out.push_str(&raw[at..t.start]); @@ -119,6 +120,9 @@ impl<'a> Renderer<'a> { at = t.end; } out.push_str(&raw[at..]); + let html = crate::scopes::qualify(&out[start..], None); + out.truncate(start); + out.push_str(&html); } Block::ThematicBreak { attrs } => { out.push_str(" Renderer<'a> { out.push_str("\n\n"); } Block::Div { attrs, children } => { + let scope = attrs.classes.iter().any(|c| c == "include") + .then(|| attrs.pairs.iter().find(|(k, _)| k == "scope").map(|(_, v)| v.as_str())).flatten(); + let mut attrs = attrs.clone(); + if scope.is_some() { attrs.pairs.retain(|(k, _)| k != "scope"); } out.push_str("\n"); + let start = out.len(); self.blocks(children, out); + if let Some(scope) = scope { + let html = crate::scopes::qualify(&out[start..], Some(scope)); + out.truncate(start); + out.push_str(&html); + } out.push_str("\n"); } Block::Math { attrs, tex, .. } => { diff --git a/src/scopes.rs b/src/scopes.rs new file mode 100644 index 0000000..219b76b --- /dev/null +++ b/src/scopes.rs @@ -0,0 +1,77 @@ +use std::collections::{HashMap, HashSet}; + +use fast5ever::{DOCUMENT, parse_fragment}; +use crate::{Attr, Block, Diagnostic, Document}; + +pub(crate) fn qualify(src: &str, suffix: Option<&str>) -> String { + if suffix.is_none() && !src.to_ascii_lowercase().contains("scope") { return src.into(); } + let mut dom = parse_fragment(src, "body"); + let mut includes: Vec<_> = dom.descendants(DOCUMENT).into_iter().filter_map(|e| { + if !dom.has_class(e, "include") { return None; } + dom.attr(e, "scope").map(|s| (e, s.to_string())) + }).collect(); + includes.reverse(); + if let Some(suffix) = suffix { includes.push((DOCUMENT, suffix.into())); } + if includes.is_empty() { return src.into(); } + for (inc, scope) in includes { + let _ = dom.remove_attr(inc, "scope"); + let els: Vec<_> = dom.descendants(inc).into_iter().skip(1).collect(); + let ids: HashMap<_, _> = els.iter().filter_map(|&e| { + dom.attr(e, "id").map(|id| (id.to_string(), format!("{id}{scope}"))) + }).collect(); + for e in els { + if let Some(id) = dom.attr(e, "id").and_then(|id| ids.get(id)) { let _ = dom.set_attr(e, "id", id); } + if let Some(id) = dom.attr(e, "href").and_then(|s| s.strip_prefix('#')).and_then(|id| ids.get(id)) { + let _ = dom.set_attr(e, "href", &format!("#{id}")); + } + } + } + dom.to_html(DOCUMENT) +} + +pub(crate) fn validate(doc: &mut Document) { + fn check(scope: &str, seen: &mut HashSet, warnings: &mut Vec) { + let message = if scope.is_empty() { Some("include scope must not be empty") } + else if scope.chars().any(char::is_whitespace) { Some("include scope must not contain whitespace") } + else if !seen.insert(scope.into()) { Some("include scopes must be unique") } + else { None }; + if let Some(message) = message { warnings.push(Diagnostic::warning("include-scope", message)); } + } + fn walk(blocks: &[Block], seen: &mut HashSet, warnings: &mut Vec) { + for block in blocks { + match block { + Block::Div { attrs, children } => { + if attrs.classes.iter().any(|c| c == "include") { + for (_, scope) in attrs.pairs.iter().filter(|(k, _)| k == "scope") { check(scope, seen, warnings); } + } + walk(children, seen, warnings); + } + Block::BlockQuote { children, .. } => walk(children, seen, warnings), + Block::List { items, .. } => for item in items { walk(&item.blocks, seen, warnings); }, + Block::Html { raw, .. } if raw.to_ascii_lowercase().contains("scope") => { + let dom = parse_fragment(raw, "body"); + for e in dom.descendants(DOCUMENT) { + if dom.has_class(e, "include") && let Some(scope) = dom.attr(e, "scope") { check(scope, seen, warnings); } + } + } + _ => {} + } + } + } + let mut seen = HashSet::new(); + walk(&doc.blocks, &mut seen, &mut doc.diagnostics); + for note in &doc.footnotes { walk(¬e.blocks, &mut seen, &mut doc.diagnostics); } +} + +pub(crate) fn attrs(open: &str) -> Option { + if !open.to_ascii_lowercase().contains("scope") { return None; } + let dom = parse_fragment(open, "body"); + let e = dom.descendants(DOCUMENT).into_iter().find(|&e| dom.has_class(e, "include") && dom.attr(e, "scope").is_some())?; + let fast5ever::NodeData::Element { attrs, .. } = &dom.get(e).data else { return None; }; + let mut result = Attr::default(); + for (name, value) in attrs { + let key = name.prefix.as_ref().map_or_else(|| name.local.to_string(), |p| format!("{p}:{}", name.local)); + result.set_pair(key, value); + } + Some(result) +} diff --git a/tests/canonical.rs b/tests/canonical.rs index a9b134f..46e59b8 100644 --- a/tests/canonical.rs +++ b/tests/canonical.rs @@ -129,3 +129,33 @@ decision-making stays plain. assert!(canonical.contains(" \\- continuation that looks like a nested list")); assert_eq!(render(&parse(&canonical, &options)), render(&document), "canonical Markdown:\n{canonical}"); } + +#[test] +fn include_scopes_qualify_ids_before_export() { + let source = r#":::: {.include scope=__camera} +# Camera {#sec-camera} +See [@sec-setup__lens]. + +::: {.include scope=__lens} +## Lens {#sec-setup} +See [@sec-setup] and [@sec-camera]. +::: +:::: + +See [@sec-setup__lens__camera]. +"#; + let html = render(&parse(source, &Options::default())); + assert!(html.contains(r#"id="sec-setup__lens__camera""#)); + assert!(html.contains(r#"id="sec-camera__camera""#)); + assert_eq!(html.matches(r##"href="#sec-setup__lens__camera""##).count(), 3); + assert!(html.contains(r##"href="#sec-camera__camera""##)); + assert!(!html.contains("scope=")); + for (scope, error) in [("__camera", "must be unique"), ("\"\"", "must not be empty")] { + let invalid = source.replace("scope=__lens", &format!("scope={scope}")); + assert!(parse(&invalid, &Options::default()).diagnostics.iter().any(|d| d.message.contains(error))); + } + let raw = r##"

Setup

Here
"##; + let html = render(&parse(raw, &Options::default())); + assert!(html.contains(r#"id="sec-setup__mic""#)); + assert!(html.contains(r##"href="#sec-setup__mic""##)); +} diff --git a/tests/test_export.py b/tests/test_export.py index 8d86ee2..9f7c376 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -2,7 +2,7 @@ import pytest -from mdhtml import TemplateDelimiter, dialect_css, math_js, mdhtml2dom, mdhtml2html, mdhtml2typst, md2gfm, md2mdhtml +from mdhtml import TemplateDelimiter, dialect_css, math_js, mdhtml2dom, mdhtml2html, mdhtml2typst, md2gfm, md2mdhtml, md2dom from mdhtml.mustache import MUSTACHE, mustache_pill from mdhtml.export import SCHEMES, _headnums from test_conformance import normalize_html @@ -574,3 +574,92 @@ def test_lenient_is_the_only_forgiving_numbering_mode(): with pytest.raises(ValueError, match='not found'): mdhtml2html(md2mdhtml(LENIENT_MD), refs='resolve') assert not mdhtml2html(md2mdhtml(LENIENT_MD), refs='ids').warnings # ids mode has nothing to fail at with pytest.raises(ValueError, match='unknown refs mode'): mdhtml2html('

x

', refs='lax') + + + +def test_included_document_scopes(): + source = r'''# Recording guide +## Overview {#sec-overview} +## Equipment {#sec-equipment} + +::: {.include scope=__camera} +# Camera +## Setup {#sec-setup} +See [@sec-setup] and [overview](#sec-overview). +::: + +::: {.include scope=__mic} +# Microphone +## Setup {#sec-setup} +See [@sec-setup]. +::: + +## Record {#sec-record} +See [@sec-setup__camera; @sec-setup__mic]. +''' + doc = md2mdhtml(source) + from xml.etree import ElementTree as ET + early = ET.fromstring('' + doc + '') + ids = [e.attrib['id'] for e in early.iter() if 'id' in e.attrib] + assert len(ids) == len(set(ids)) + assert {'sec-setup__camera', 'sec-setup__mic'} <= set(ids) + assert early.find('.//a[@href="#sec-setup__mic"]') is not None + assert 'scope=' not in doc + assert 'id="sec-setup__mic"' in md2dom(source).to_html() + with pytest.raises(ValueError, match='target #sec-setup not found'): + mdhtml2html(md2mdhtml(source + '\nSee [@sec-setup].')) + html = mdhtml2html(doc, number_headings='decimal') + result = ET.fromstring('' + html + '') + for scope in ('camera', 'mic'): + heading = result.find(f'.//*[@id="sec-setup__{scope}"]') + assert ''.join(heading.itertext()).startswith('1. ') + assert result.find(f'.//a[@href="#sec-setup__{scope}"]').text == 'Section 1' + assert result.find('.//a[@href="#sec-overview"]').text == 'overview' + assert ''.join(result.find('.//*[@id="sec-record"]').itertext()).startswith('2. ') + assert 'Sections 1 and 1' in html + assert 'heading-number' not in mdhtml2html(doc, number_headings=False, refs='ids') + for invalid, message in [(source.replace('scope=__mic', 'scope=__camera'), 'must be unique'), + (source.replace('scope=__mic', 'scope=""'), 'must not be empty')]: + assert any(message in warning for warning in md2mdhtml(invalid).warnings) + + named = '# Camera\n## Setup {#sec-setup__camera}\n# Microphone\n## Setup {#sec-setup__mic}\nSee [@sec-setup__camera; @sec-setup__mic].' + assert 'Sections 1 and 1' in md2gfm(named, number_headings='decimal') + typst = mdhtml2typst(md2mdhtml(named), number_headings='decimal') + assert 'Sections' in typst and '[Section]' not in typst + + +def test_scoped_mdhtml_retains_parse_context_and_callback_ids(): + source = r'''--- +number-headings: decimal +--- +::: {.include scope=__mic} +# Microphone +## Setup {#sec-setup} +See [@sec-setup] and [@sec-setup__mic]. +''' + doc = md2mdhtml(source) + assert doc.meta == {'number-headings': 'decimal'} + assert len(doc.warnings) == 1 and 'unclosed fenced div' in doc.warnings[0] + assert doc.count('href="#sec-setup__mic"') == 2 + assert '__mic__mic' not in mdhtml2html(doc) + from mdhtml._native import md2mdhtml as native + html, warnings, meta = native(source) + assert 'id="sec-setup__mic"' in html + assert warnings == doc.warnings and dict(meta) == doc.meta + callback = md2mdhtml(source, callbacks={'heading': lambda node, html: html.replace('sec-setup', 'sec-sound')}) + assert 'id="sec-sound__mic"' in callback + assert 'id="sec-setup__mic"' in md2mdhtml(source, callbacks={'div': lambda node, html: html}) + + +def test_scoped_template_contents(): + source = r'''
''' + doc = md2mdhtml(source) + assert '