diff --git a/docs/how-to/generated_docs.rst b/docs/how-to/generated_docs.rst index cf060c957..264189977 100644 --- a/docs/how-to/generated_docs.rst +++ b/docs/how-to/generated_docs.rst @@ -21,7 +21,7 @@ When a script or tool creates documentation content at build time, collect it with a ``docs_bundle`` and mount the bundle into your documentation tree. You find a `complete working example `_ -in the :ref:`metamodel-types-visualization`. +in the :ref:`metamodel-reference`. Which ``data`` attribute? ------------------------- diff --git a/src/extensions/score_metamodel/docs/BUILD b/src/extensions/score_metamodel/docs/BUILD index 389c9a57d..bb7621a9f 100644 --- a/src/extensions/score_metamodel/docs/BUILD +++ b/src/extensions/score_metamodel/docs/BUILD @@ -20,7 +20,10 @@ genrule( srcs = [ "//src/extensions/score_metamodel:metamodel_yaml", ], - outs = ["generated/index.rst", "generated/metamodel_classes.mmd"], + outs = [ + "generated/index.rst", + "generated/metamodel_classes.mmd", + ], cmd = "$(location :generate_metamodel_rst_bin) --rst-output $(location generated/index.rst) --mmd-output $(location generated/metamodel_classes.mmd) $(location //src/extensions/score_metamodel:metamodel_yaml)", tools = [":generate_metamodel_rst_bin"], visibility = ["//visibility:private"], diff --git a/src/extensions/score_metamodel/docs/generate_metamodel_rst.py b/src/extensions/score_metamodel/docs/generate_metamodel_rst.py index 0e2ef5ee3..50b231db1 100644 --- a/src/extensions/score_metamodel/docs/generate_metamodel_rst.py +++ b/src/extensions/score_metamodel/docs/generate_metamodel_rst.py @@ -11,7 +11,30 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Generate an RST file with a list-table and Mermaid class diagram from metamodel.yaml. +"""Generate an RST file with per-type tables and Mermaid class diagrams. + +The generator is intentionally split into three layers: + +* :class:`Metamodel` and :class:`NeedType` are the domain model. They convert + the YAML dictionaries into typed objects and own metamodel-specific rules, + such as inherited options and direct dependencies. +* :class:`NeedDiagramBuilder` translates the domain model into the generic + Mermaid model. :class:`MermaidRenderer` is the only component that knows + Mermaid syntax. +* :class:`RstRenderer` turns the same domain model into the generated RST. + +The generated RST contains one overview diagram and one focused diagram per +need type. The focused diagrams are embedded directly in the RST. Strings +are still assembled in the two renderers because RST and Mermaid are text +formats; the domain and rendering decisions are kept out of the CLI and out +of the YAML representation. Focused diagrams also receive a compact display +height when they contain only incoming, only outgoing, or no links. This is +implemented through scoped CSS because diagram height is controlled by the +Sphinx Mermaid integration rather than by the Mermaid class-diagram syntax. + +The Options and Links tables are rendered below each focused diagram inside a +closed dropdown, so the diagrams remain the primary content when the page is +opened. Usage: generate_metamodel_rst.py --rst-output FILE --mmd-output FILE [METAMODEL_YAML] @@ -21,129 +44,553 @@ import argparse import sys +from collections.abc import Iterable, Iterator, Mapping +from dataclasses import dataclass, replace from pathlib import Path +from typing import Any, cast import ruamel.yaml -def _parse_yaml(path: Path) -> dict: +def _parse_yaml(path: Path) -> Mapping[str, Any]: yaml = ruamel.yaml.YAML() yaml.preserve_quotes = True - with Path(path).open(encoding="utf-8") as fh: - return yaml.load(fh) # type: ignore[no-any-return] + with path.open(encoding="utf-8") as fh: + return cast(Mapping[str, Any], yaml.load(fh)) + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + """Return *value* as a mapping or an empty mapping for missing sections.""" + if isinstance(value, Mapping): + return value + return {} + + +def _as_string_mapping(value: Any) -> dict[str, str]: + """Convert a YAML mapping into the string mapping used by the model.""" + return {str(key): str(item) for key, item in _as_mapping(value).items()} def _split_targets(targets: str) -> list[str]: """Split a comma-separated target spec, ignoring the ``ANY`` wildcard.""" if targets == "ANY": return [] - return [t.strip() for t in targets.split(",") if t.strip()] - - -def _incoming_mandatory_links(types: dict) -> dict[str, list[str]]: - """Map each type to the source types that point at it via mandatory links.""" - incoming: dict[str, list[str]] = {name: [] for name in types} - for source, ty in types.items(): - for targets in ty.get("mandatory_links", {}).values(): - for target in _split_targets(targets): - if target in incoming and source not in incoming[target]: - incoming[target].append(source) - return {name: sorted(sources) for name, sources in incoming.items()} - - -def _build_table(types: dict) -> list[str]: - incoming = _incoming_mandatory_links(types) - lines: list[str] = [] - lines.append(".. list-table:: Need Types") - lines.append(" :header-rows: 1") - lines.append("") - lines.append(" * - Type") - lines.append(" - Title") - lines.append(" - Mandatory Options") - lines.append(" - Links") - lines.append(" - Incoming Mandatory Links") - for name, ty in sorted(types.items()): - title = ty.get("title", name) - mandatory = ( - ", ".join(sorted(ty.get("mandatory_options", {}).keys())) or "\u2014" - ) - optional_links = ty.get("optional_links", {}) - mandatory_links = ty.get("mandatory_links", {}) - link_strs = [ - f"**{n}**" if n in mandatory_links else n - for n in sorted(set(optional_links) | set(mandatory_links)) + return [target.strip() for target in targets.split(",") if target.strip()] + + +@dataclass(frozen=True) +class NeedLink: + """A link definition belonging to one need type.""" + + name: str + targets: tuple[str, ...] + optional: bool + + +@dataclass(frozen=True) +class NeedType: + """A normalized need type from the metamodel YAML.""" + + name: str + title: str + prefix: str + mandatory_options: Mapping[str, str] + optional_options: Mapping[str, str] + mandatory_links: Mapping[str, str] + optional_links: Mapping[str, str] + color: str | None = None + + @property + def document_id(self) -> str: + """Return the section ID generated by Docutils for this type.""" + # Docutils normalizes underscores in heading IDs to hyphens. + return self.name.replace("_", "-") + + @classmethod + def from_yaml( + cls, + name: str, + raw_type: Mapping[str, Any], + base_options: Mapping[str, Any], + ) -> NeedType: + """Create one type and apply the global option defaults.""" + mandatory_options = _as_string_mapping( + base_options.get("mandatory_options", {}) + ) + mandatory_options.update( + _as_string_mapping(raw_type.get("mandatory_options", {})) + ) + + optional_options = _as_string_mapping(base_options.get("optional_options", {})) + optional_options.update( + _as_string_mapping(raw_type.get("optional_options", {})) + ) + + prefix = str(raw_type.get("prefix", f"{name}__")) + mandatory_options.setdefault("id", f"^{prefix}[0-9a-z_]+$") + + return cls( + name=name, + title=str(raw_type.get("title", name)), + prefix=prefix, + mandatory_options=mandatory_options, + optional_options=optional_options, + mandatory_links=_as_string_mapping(raw_type.get("mandatory_links", {})), + optional_links=_as_string_mapping(raw_type.get("optional_links", {})), + color=( + raw_type.get("color") + if isinstance(raw_type.get("color"), str) + else None + ), + ) + + def iter_links(self) -> Iterator[NeedLink]: + """Yield all link definitions with their resolved target names.""" + for name, targets in self.mandatory_links.items(): + yield NeedLink( + name=name, + targets=tuple(_split_targets(targets)), + optional=False, + ) + for name, targets in self.optional_links.items(): + yield NeedLink( + name=name, + targets=tuple(_split_targets(targets)), + optional=True, + ) + + +@dataclass(frozen=True) +class Metamodel: + """The normalized metamodel used by both documentation renderers.""" + + types: tuple[NeedType, ...] + + @classmethod + def from_yaml(cls, data: Mapping[str, Any]) -> Metamodel: + """Build a metamodel from the top-level YAML mapping.""" + base_options = _as_mapping(data.get("needs_types_base_options", {})) + raw_types = _as_mapping(data.get("needs_types", {})) + types = tuple( + NeedType.from_yaml(str(name), _as_mapping(raw_type), base_options) + for name, raw_type in sorted( + raw_types.items(), key=lambda item: str(item[0]) + ) + ) + return cls(types=types) + + @property + def types_by_name(self) -> dict[str, NeedType]: + """Return the types indexed by their directive name.""" + return {need_type.name: need_type for need_type in self.types} + + def direct_dependencies(self, name: str) -> set[str]: + """Return the types directly linked to *name* in either direction.""" + dependencies = {name} + known_names = set(self.types_by_name) + + # Include incoming links as well as outgoing links. Otherwise a type + # that is only referenced by other types would get an unhelpful + # one-node graph. + for source in self.types: + for link in source.iter_links(): + for target in link.targets: + if target not in known_names: + continue + if source.name == name: + dependencies.add(target) + if target == name: + dependencies.add(source.name) + return dependencies + + +@dataclass(frozen=True) +class MermaidNode: + """A generic class-diagram node.""" + + name: str + attributes: tuple[str, ...] = () + optional_attributes: tuple[str, ...] = () + color: str | None = None + style_override: str | None = None + href: str | None = None + tooltip: str | None = None + + +@dataclass(frozen=True) +class MermaidEdge: + """A generic directed class-diagram edge.""" + + source: str + target: str + label: str + optional: bool = False + + +@dataclass(frozen=True) +class MermaidDiagram: + """A Mermaid class diagram independent of the source metamodel.""" + + nodes: tuple[MermaidNode, ...] + edges: tuple[MermaidEdge, ...] + direction: str = "BT" + layout: str = "elk" + security_level: str | None = "loose" + + def compact_height_for(self, focal_name: str) -> str | None: + """Return a compact CSS height for a one-directional focal diagram. + + ``sphinxcontrib-mermaid`` uses a 500px default SVG height. The values + below are two-thirds and one-third of that default. A diagram with + both incoming and outgoing links keeps the default height. + """ + has_incoming = any(edge.target == focal_name for edge in self.edges) + has_outgoing = any(edge.source == focal_name for edge in self.edges) + + if not has_incoming and not has_outgoing: + return "167px" + if has_incoming != has_outgoing: + return "333px" + return None + + +class MermaidRenderer: + """Serialize a :class:`MermaidDiagram` into Mermaid source text.""" + + def render(self, diagram: MermaidDiagram) -> str: + """Build a complete Mermaid document.""" + lines = ( + self._class_declarations(diagram) + + self._edge_declarations(diagram) + + self._style_declarations(diagram) + + self._link_declarations(diagram) + ) + document = [ + "---", + "config:", + f" layout: {diagram.layout}", ] - links = ", ".join(link_strs) or "\u2014" - inc = ", ".join(incoming.get(name, [])) or "\u2014" - lines.append(" * - " + name) - lines.append(" - " + title) - lines.append(" - " + mandatory) - lines.append(" - " + links) - lines.append(" - " + inc) - return lines - - -def _class_declarations(types: dict) -> list[str]: - """Declare every type, listing mandatory options as class members.""" - lines: list[str] = [] - for name in sorted(types): - mandatory_opts = sorted(types[name].get("mandatory_options", {}).keys()) - if mandatory_opts: - lines.append(f"class {name} {{") - lines.extend(f" +{opt}" for opt in mandatory_opts) - lines.append("}") - else: - lines.append(f"class {name}") - return lines - - -def _link_edges(types: dict) -> list[str]: - """Edges for all (mandatory + optional) links between known types.""" - lines: list[str] = [] - seen: set[tuple[str, str, str]] = set() - for name, ty in sorted(types.items()): - links = list(ty.get("mandatory_links", {}).items()) + list( - ty.get("optional_links", {}).items() - ) - for link_name, targets in sorted(links): - for target in _split_targets(targets): - if target not in types: - continue - key = (name, target, link_name) - if key not in seen: + if diagram.security_level is not None: + document.append(f" securityLevel: {diagram.security_level}") + document.extend( + [ + "---", + "classDiagram", + f" direction {diagram.direction}", + ] + ) + return "\n".join(document + lines) + + def _class_declarations(self, diagram: MermaidDiagram) -> list[str]: + lines: list[str] = [] + for node in sorted(diagram.nodes, key=lambda item: item.name): + if node.attributes or node.optional_attributes: + lines.append(f"class {node.name} {{") + lines.extend(f" +{attribute}" for attribute in sorted(node.attributes)) + # Mermaid's trailing '*' member classifier renders the complete + # attribute line in italics. Keep the textual marker as part + # of the line so the optional status remains unambiguous. + lines.extend( + f" +{attribute} [optional]*" + for attribute in sorted( + set(node.optional_attributes) - set(node.attributes) + ) + ) + lines.append("}") + else: + lines.append(f"class {node.name}") + return lines + + def _edge_declarations(self, diagram: MermaidDiagram) -> list[str]: + lines: list[str] = [] + seen: set[tuple[str, str, str]] = set() + for edge in diagram.edges: + key = (edge.source, edge.target, edge.label) + if key in seen: + continue + seen.add(key) + arrow = "..>" if edge.optional else "-->" + lines.append(f"{edge.source} {arrow} {edge.target} : {edge.label}") + return lines + + def _style_declarations(self, diagram: MermaidDiagram) -> list[str]: + lines = [ + f"style {node.name} fill:{node.color},stroke:#666,color:#000" + for node in sorted(diagram.nodes, key=lambda item: item.name) + if node.color and node.style_override is None + ] + lines.extend( + f"style {node.name} {node.style_override}" + for node in sorted(diagram.nodes, key=lambda item: item.name) + if node.style_override + ) + return lines + + def _link_declarations(self, diagram: MermaidDiagram) -> list[str]: + lines: list[str] = [] + for node in sorted(diagram.nodes, key=lambda item: item.name): + if node.href is None: + continue + line = f'click {node.name} href "{node.href}"' + if node.tooltip: + line += f' "{node.tooltip}"' + lines.append(line) + return lines + + +@dataclass(frozen=True) +class NeedDiagramBuilder: + """Translate metamodel types into generic Mermaid diagrams.""" + + metamodel: Metamodel + + def build(self, focal_name: str | None = None) -> MermaidDiagram: + """Build an overview or a focused dependency diagram.""" + type_map = self.metamodel.types_by_name + included_names = set(type_map) + if focal_name is not None: + included_names = self.metamodel.direct_dependencies(focal_name) + + selected_types = tuple( + type_map[name] for name in sorted(included_names) if name in type_map + ) + nodes = tuple(self._node(need_type) for need_type in selected_types) + + if focal_name is not None: + root_color = type_map[focal_name].color or "#FFFFFF" + root_style = f"fill:{root_color},stroke:#000,stroke-width:3px,color:#000" + nodes = tuple( + replace(node, style_override=root_style) + if node.name == focal_name + else node + for node in nodes + ) + + return MermaidDiagram( + nodes=nodes, + edges=self._edges(selected_types, included_names, focal_name), + ) + + @staticmethod + def _node(need_type: NeedType) -> MermaidNode: + return MermaidNode( + name=need_type.name, + attributes=tuple(need_type.mandatory_options), + optional_attributes=tuple(need_type.optional_options), + color=need_type.color, + href=f"#{need_type.document_id}", + tooltip=f"Open {need_type.name} details", + ) + + @staticmethod + def _edges( + selected_types: tuple[NeedType, ...], + included_names: set[str], + focal_name: str | None, + ) -> tuple[MermaidEdge, ...]: + edges: list[MermaidEdge] = [] + seen: set[tuple[str, str, str]] = set() + for source in selected_types: + links = sorted( + source.iter_links(), + key=lambda link: (link.optional, link.name, link.targets), + ) + for link in links: + for target in link.targets: + if target not in included_names: + continue + # A focused diagram should show only links touching its + # root; links between two neighbours add unrelated detail. + if ( + focal_name is not None + and source.name != focal_name + and target != focal_name + ): + continue + key = (source.name, target, link.name) + if key in seen: + continue seen.add(key) - if link_name.endswith("_by"): - # for layouting, reverse link direction for "passive" verbs - lines.append(f"{target} <-- {name} : {link_name}") - else: - lines.append(f"{name} --> {target} : {link_name}") - return lines + # Keep every edge in source-to-target form. Together with + # the BT direction this places the target above its source. + edges.append( + MermaidEdge( + source=source.name, + target=target, + label=link.name, + optional=link.optional, + ) + ) + return tuple(edges) -def _node_styles(types: dict) -> list[str]: - """Color nodes per the ``color`` option in metamodel.yaml. +@dataclass(frozen=True) +class RstRenderer: + """Render the normalized metamodel as an RST document.""" - Dark text and a visible border keep pastel-filled nodes readable - in both light and dark mermaid themes. - """ - lines: list[str] = [] - for name, ty in sorted(types.items()): - color = ty.get("color") - if color: - lines.append(f"style {name} fill:{color},stroke:#666,color:#000") - return lines + metamodel: Metamodel + diagram_builder: NeedDiagramBuilder + mermaid_renderer: MermaidRenderer + overview_filename: str + def render(self) -> str: + """Build the complete generated RST document.""" + focused_diagrams = { + need_type.name: self.diagram_builder.build(focal_name=need_type.name) + for need_type in self.metamodel.types + } + lines = [ + ".. _metamodel-reference:", + "", + "Metamodel", + "=========", + "", + "Overview", + "~~~~~~~~", + "", + f".. mermaid:: {self.overview_filename}", + " :name: metamodel-overview-diagram", + "", + ] + lines.extend(self._render_compact_height_styles(focused_diagrams)) + for need_type in self.metamodel.types: + lines.extend(self._render_type(need_type, focused_diagrams[need_type.name])) + return "\n".join(lines) -def _build_mermaid(types: dict) -> list[str]: - return _class_declarations(types) + _link_edges(types) + _node_styles(types) + def _render_type(self, need_type: NeedType, diagram: MermaidDiagram) -> list[str]: + lines = [ + need_type.name, + "~" * len(need_type.name), + "", + f"**{need_type.title}** (``{need_type.prefix}``)", + "", + ] + lines.extend( + [ + ".. mermaid::", + f" :name: {self._diagram_name(need_type)}", + "", + ] + ) + # Inline diagrams keep the generated assets self-contained. The type + # names therefore come directly from the metamodel and need no second + # list of declared output files in BUILD. + lines.extend( + f" {line}" for line in self.mermaid_renderer.render(diagram).splitlines() + ) + lines.append("") + lines.extend(self._render_collapsible_details(need_type)) + return lines + @staticmethod + def _diagram_name(need_type: NeedType) -> str: + """Return the normalized HTML id for a focused metamodel diagram.""" + return f"metamodel-type-diagram-{need_type.document_id}" -def main() -> int: + def _render_compact_height_styles( + self, focused_diagrams: Mapping[str, MermaidDiagram] + ) -> list[str]: + """Render scoped HTML styles for compact focused diagrams.""" + styles: list[str] = [] + for need_type in self.metamodel.types: + diagram = focused_diagrams[need_type.name] + height = diagram.compact_height_for(need_type.name) + if height is None: + continue + styles.extend( + [ + f"pre#{self._diagram_name(need_type)}.mermaid > svg {{", + f" height: {height} !important;", + "}", + ] + ) + + if not styles: + return [] + return [ + ".. raw:: html", + "", + " ", + "", + ] + + def _render_collapsible_details(self, need_type: NeedType) -> list[str]: + """Render the type's tables in a closed dropdown below its diagram.""" + details = self._render_mapping_section( + heading="Options", + mandatory_values=need_type.mandatory_options, + optional_values=need_type.optional_options, + value_title="Option", + ) + details.extend( + self._render_mapping_section( + heading="Links", + mandatory_values=need_type.mandatory_links, + optional_values=need_type.optional_links, + value_title="Link", + ) + ) + if not details: + return [] + + return [ + ".. dropdown:: Options and Links", + "", + *self._indent_rst(details), + ] + + @staticmethod + def _indent_rst(lines: Iterable[str], prefix: str = " ") -> list[str]: + """Indent generated RST as content of a directive.""" + return [f"{prefix}{line}" if line else "" for line in lines] + + @staticmethod + def _render_mapping_section( + heading: str, + mandatory_values: Mapping[str, str], + optional_values: Mapping[str, str], + value_title: str, + ) -> list[str]: + """Render one table that distinguishes mandatory and optional values.""" + if not mandatory_values and not optional_values: + return [] + + # These are supporting details of a type, so an inline label keeps them + # visually subordinate to the type heading. + lines = [ + f"**{heading}**", + "", + ".. list-table::", + " :header-rows: 1", + "", + " * - Type", + f" - {value_title}", + " - Definition", + ] + for value_type, values in ( + ("Mandatory", mandatory_values), + ("Optional", optional_values), + ): + for key, value in sorted(values.items()): + lines.append(f" * - {value_type}") + lines.append(f" - ``{key}``") + lines.append(f" - ``{value}``") + lines.append("") + return lines + + +def _argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Generate RST from metamodel.yaml") parser.add_argument("--rst-output", type=Path, required=True) parser.add_argument("--mmd-output", type=Path, required=True) parser.add_argument("metamodel", nargs="?", default=None) - args = parser.parse_args() + return parser + + +def main() -> int: + args = _argument_parser().parse_args() meta_path = ( Path(args.metamodel) @@ -160,45 +607,29 @@ def main() -> int: print(f"Error parsing YAML: {exc}", file=sys.stderr) return 1 - types = data.get("needs_types", {}) - if not types: + if not _as_mapping(data.get("needs_types", {})): print( - "Error: 'needs_types' section not found in metamodel.yaml", file=sys.stderr + "Error: 'needs_types' section not found in metamodel.yaml", + file=sys.stderr, ) return 1 - table = _build_table(types) - mermaid_lines = [ - "---", - "config:", - " layout: elk", - "---", - "classDiagram", - " direction BT", - ] + _build_mermaid(types) - args.mmd_output.write_text("\n".join(mermaid_lines) + "\n", encoding="utf-8") - output = "\n".join( - [ - "..", - " # (generated \u2014 do not edit)", - " # SPDX-License-Identifier: Apache-2.0", - "", - ".. _metamodel-types-visualization:", - "", - "Metamodel Types Visualization", - "==============================", - "", - f".. mermaid:: {args.mmd_output.name}", - "", - "Need Types", - "----------", - "", - ] - + table - + [""] + metamodel = Metamodel.from_yaml(data) + diagram_builder = NeedDiagramBuilder(metamodel) + mermaid_renderer = MermaidRenderer() + + args.mmd_output.write_text( + mermaid_renderer.render(diagram_builder.build()) + "\n", + encoding="utf-8", ) - args.rst_output.write_text(output, encoding="utf-8") + rst_renderer = RstRenderer( + metamodel=metamodel, + diagram_builder=diagram_builder, + mermaid_renderer=mermaid_renderer, + overview_filename=args.mmd_output.name, + ) + args.rst_output.write_text(rst_renderer.render(), encoding="utf-8") return 0