Add reusable llms/markdown docs builder - #361
Conversation
|
Added HoloViews config file: holoviews/scripts/llms_config.py"""Config for building HoloViews markdown docs and llms.txt
from the nbsite llms builder.
"""
from __future__ import annotations
from pathlib import Path
from nbsite.scripts import LlmsBuildConfig, LlmsSection, MarkdownSource
ROOT = Path(__file__).parent.parent
DOC_DIR = ROOT / "doc"
BUILTDOCS_DIR = ROOT / "builtdocs"
OUTPUT_DIR = BUILTDOCS_DIR / "markdown"
MARKDOWN_BASE_URL = "/markdown"
def _section_label(path: Path) -> str:
if path.stem == "index":
return "home" if path.parent == Path(".") else path.parent.as_posix().replace("-", " ")
return path.stem.replace("_", " ")
def _api_label(path: Path) -> str:
name = path.stem
for prefix in (
"holoviews.element.",
"holoviews.core.",
"holoviews.ipython.",
"holoviews.plotting.",
"holoviews.operation.",
"holoviews.util.",
):
if name.startswith(prefix):
name = name.removeprefix(prefix)
break
return name.replace("_", " ")
CONFIG = LlmsBuildConfig(
project_title="HoloViews",
project_description=(
"HoloViews is an open-source Python library designed to make data analysis"
" and visualization seamless and simple. \n"
"This file points to the selected markdown documentation for code-writing utility."
),
markdown_root=OUTPUT_DIR,
llms_output_path=BUILTDOCS_DIR / "llms.txt",
markdown_base_url=MARKDOWN_BASE_URL,
sources=(
MarkdownSource(
source_dir=DOC_DIR,
output_dir=OUTPUT_DIR,
rendered_source_dir=BUILTDOCS_DIR,
),
),
sections=(
LlmsSection(
title="Home",
description="Top-level pages in the HoloViews docs tree.",
path_prefix=Path("."),
label_builder=_section_label,
path_filter=lambda path: len(path.parts) == 1,
),
LlmsSection(
title="Getting Started",
description="Step-by-step guides to get you using HoloViews productively as quickly as possible.",
path_prefix=Path("getting_started"),
),
LlmsSection(
title="User Guide",
description="Key concepts that will help you use HoloViews in your work.",
path_prefix=Path("user_guide"),
),
LlmsSection(
title="Gallery",
description="Example visualizations using HoloViews with different backends and datasets.",
path_prefix=Path("gallery"),
),
LlmsSection(
title="Reference Gallery",
description="More gallery examples using different Holoviews element types.",
path_prefix=Path("reference"),
label_builder=_section_label,
),
LlmsSection(
title="API",
description="HoloViews plotting APIs.",
path_prefix=Path("reference_manual"),
label_builder=_api_label,
),
),
) |
This reverts commit a28ff18.
|
Can you show some output for holoviews? Also can you clarify why rst is needed |
|
Here's a zip file of the generated markdown directory. |
|
Hmm a couple issues I noticed immediately:
|
This can be easily resolved by deciding what goes into each repo's config file
OK, will try to fix that. Also important to note here that this is a specifically HoloViews problem. |
Okay if you can update it accordingly!
So Panel / hvPlot doesn't encounter this? Thanks! |
No. here's Panel's markdown zip: |
All below 100 lines now. |
|
Fixed the HoloViews markdown files |
|
I think the llms.txt should have a sentence saying not all are shown, like I think is only a subset of the many charts hvplot supports. 
|
There was a problem hiding this comment.
Pull request overview
Copilot reviewed 5 out of 5 changed files in this pull request and generated no new comments.
Suppressed comments (3)
nbsite/scripts/_build_llms_txt.py:585
_build_url_pattern_body()buildsrelswithstr(Path(...)), which will use OS-specific separators (e.g. backslashes on Windows). Since this content is emitted intollms.txtas URL/path examples, it should be stable and use forward slashes regardless of platform. Use.as_posix()when computing the relative path string.
def _rel(path: Path) -> str:
try:
return str(path.relative_to(section.path_prefix).with_suffix(""))
except ValueError:
return path.stem
nbsite/main.py:28
_load_config_object()will raise a bareAttributeErrorif the requested config attribute is missing from the loaded module. For a CLI, this makes the error harder to interpret. Catch the missing-attribute case and raise aValueErrorwith a clear message including the module/path and attribute name.
config = getattr(module, attr)
return config() if callable(config) else config
nbsite/scripts/_build_llms_txt.py:548
generate_index_pages()passes absolute paths tocategory.label_builder(), but the label builder contract elsewhere (e.g. section label builders) appears to receive paths relative tomarkdown_root. With a common label builder likepath.parent == Path('.')(as in the PR description example), absolute paths will produce incorrect labels. Passmd_filerelative tomarkdown_rootinto the label builder for consistent behavior.
for md_file in md_files:
rel_md = md_file.relative_to(markdown_root).as_posix()
lines.append(f"- [{category.label_builder(md_file)}]({markdown_base_url}/{rel_md})")
| "style", | ||
| ) | ||
|
|
||
| MARKDOWN_STRIP_TAGS = ( |
There was a problem hiding this comment.
Can these be imported from an standard html lib?
There was a problem hiding this comment.
Unfortunately there's no single package that can replace the values in the constants.
| def _pandoc_command( | ||
| input_format: str, | ||
| output_path: Path, | ||
| input_path: Path, | ||
| ) -> list[str]: | ||
| return [ | ||
| "pandoc", | ||
| "-f", | ||
| input_format, | ||
| "-t", | ||
| "gfm", | ||
| "-o", | ||
| str(output_path), | ||
| str(input_path), | ||
| ] |
There was a problem hiding this comment.
I think we can just import pandoc?
This PR:
nbsite build-llms --config module:CONFIGfor individual reposUsage in individual repos
scripts/llmsconfig.pyCONFIG = LlmsBuildConfig(...)nbsite build-llms --config scripts/llms_config.py**Example script for hvPlot **
See holoviz/hvplot#1732
scripts/llms_config.py: