From 90fdc355d7aa1b92daf3bd236b9c30310aab27c5 Mon Sep 17 00:00:00 2001 From: Xin Han Date: Sun, 16 Aug 2026 04:15:56 +1000 Subject: [PATCH] Generate llms.txt and per-page Markdown for the docs Adds mkdocs-llmstxt, which publishes /llms.txt as an index of the documentation and a .md counterpart for every page. Feeding docs to an LLM currently means copying rendered HTML, which loses the structure that makes it useful. The plugin renders each page to HTML and converts it back to Markdown, rather than reading the source. That matters here: the API pages are a single `::: ikpykit.X` directive each, so a source-level export would produce nothing. Going through the rendered output captures the full mkdocstrings result -- signature, description, and parameter tables. It also means the plugin has to run after mkdocstrings and mkdocs-jupyter, hence its position last in the plugins list. Sections mirror the nav. The INNE notebook is listed explicitly because the user_guides/*.md glob does not match .ipynb, and authors.md is included so every page in the nav is covered; verified by diffing the nav against the generated index. full_output produces llms-full.txt (208K) with every page expanded, for pasting a whole corpus in one go. Two caveats worth recording. The plugin is in maintenance mode upstream, its author having moved to Zensical, so it works but will not gain features. And mike deploys per version, so these files land under each version prefix rather than at the site root. --- mkdocs.yml | 26 +++++++++++++++++++++ pyproject.toml | 1 + uv.lock | 63 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 1808399..04d223b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -100,6 +100,32 @@ plugins: version_selector: true css_dir: css javascript_dir: js + # Must come after mkdocstrings and mkdocs-jupyter: this plugin converts the + # rendered HTML back to Markdown, so it needs their output to already exist. + # Otherwise the API pages would export as bare `::: ikpykit.X` directives. + - llmstxt: + markdown_description: >- + IKPyKit implements Isolation Kernel methods for anomaly detection, + clustering, and representation learning, over points, graphs, groups, + streams, trajectories, and time series. The estimators follow the + scikit-learn API. + full_output: llms-full.txt + sections: + Getting started: + - README.md + - quick-start/how-to-install.md + User guides: + - user_guides/*.md + - user_guides/inne.ipynb + Examples: + - examples/*.md + API reference: + - api/**/*.md + Project: + - releases/releases.md + - faq/table-of-contents.md + - contributing/contribution.md + - authors/authors.md theme: name: material custom_dir: docs/overrides diff --git a/pyproject.toml b/pyproject.toml index 1292d21..8633ffa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,7 @@ docs = [ "mike==2.2.0", "mkdocs==1.6.1", "mkdocs-jupyter==0.26.3", + "mkdocs-llmstxt==0.5.0", "mkdocstrings==1.0.6", "mkdocstrings-python==2.0.5", "notebook==7.6.2", diff --git a/uv.lock b/uv.lock index b59b57b..436e0de 100644 --- a/uv.lock +++ b/uv.lock @@ -820,6 +820,7 @@ docs = [ { name = "mike" }, { name = "mkdocs" }, { name = "mkdocs-jupyter" }, + { name = "mkdocs-llmstxt" }, { name = "mkdocstrings" }, { name = "mkdocstrings-python" }, { name = "notebook" }, @@ -852,6 +853,7 @@ docs = [ { name = "mike", specifier = "==2.2.0" }, { name = "mkdocs", specifier = "==1.6.1" }, { name = "mkdocs-jupyter", specifier = "==0.26.3" }, + { name = "mkdocs-llmstxt", specifier = "==0.5.0" }, { name = "mkdocstrings", specifier = "==1.0.6" }, { name = "mkdocstrings-python", specifier = "==2.0.5" }, { name = "notebook", specifier = "==7.6.2" }, @@ -1371,14 +1373,27 @@ wheels = [ [[package]] name = "markdown-it-py" -version = "4.0.0" +version = "3.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload-time = "2023-06-03T06:41:11.019Z" }, +] + +[[package]] +name = "markdownify" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/ab/d1297139c0e2ceb151ae564c8c4f57ac0155d8f1f8b4cbd5d6523c82ea36/markdownify-1.2.3.tar.gz", hash = "sha256:1a176f05522c8a2cb1dd3ab9d307dcdadbed5c26ae717855bfc42b3b6d38d937", size = 18852, upload-time = "2026-06-30T20:27:39.06Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, + { url = "https://files.pythonhosted.org/packages/04/10/fa543d484e8b1199243fe20eedd02cc5af050edebce98a7293a5773df592/markdownify-1.2.3-py3-none-any.whl", hash = "sha256:a189a0bedfd14009030fde5f85bb6f77c56897cb839b5c25315dd7d4e3e290ba", size = 15732, upload-time = "2026-06-30T20:27:38.094Z" }, ] [[package]] @@ -1531,6 +1546,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, ] +[[package]] +name = "mdformat" +version = "0.7.22" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/eb/b5cbf2484411af039a3d4aeb53a5160fae25dd8c84af6a4243bc2f3fedb3/mdformat-0.7.22.tar.gz", hash = "sha256:eef84fa8f233d3162734683c2a8a6222227a229b9206872e6139658d99acb1ea", size = 34610, upload-time = "2025-01-30T18:00:51.418Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/6f/94a7344f6d634fe3563bea8b33bccedee37f2726f7807e9a58440dc91627/mdformat-0.7.22-py3-none-any.whl", hash = "sha256:61122637c9e1d9be1329054f3fa216559f0d1f722b7919b060a8c2a4ae1850e5", size = 34447, upload-time = "2025-01-30T18:00:48.708Z" }, +] + +[[package]] +name = "mdformat-tables" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdformat" }, + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/fc/995ba209096bdebdeb8893d507c7b32b7e07d9a9f2cdc2ec07529947794b/mdformat_tables-1.0.0.tar.gz", hash = "sha256:a57db1ac17c4a125da794ef45539904bb8a9592e80557d525e1f169c96daa2c8", size = 6106, upload-time = "2024-08-23T23:41:33.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/37/d78e37d14323da3f607cd1af7daf262cb87fe614a245c15ad03bb03a2706/mdformat_tables-1.0.0-py3-none-any.whl", hash = "sha256:94cd86126141b2adc3b04c08d1441eb1272b36c39146bab078249a41c7240a9a", size = 5104, upload-time = "2024-08-23T23:41:31.863Z" }, +] + [[package]] name = "mdit-py-plugins" version = "0.5.0" @@ -1656,6 +1696,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/95/cf3f7fe4910cf0365fa8ea0c731f4b8a624d97cd76ea777913ac8d0868e2/mkdocs_jupyter-0.26.3-py3-none-any.whl", hash = "sha256:cd6644fb578131157194d750fd4d10fc2fd8f1e84e00036ee62df3b5b4b84c82", size = 1459740, upload-time = "2026-04-17T18:56:30.031Z" }, ] +[[package]] +name = "mkdocs-llmstxt" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "markdownify" }, + { name = "mdformat" }, + { name = "mdformat-tables" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f5/4c31cdffa7c09bf48d8c7a50d8342dc100abac98ac4150826bc11afc0c9f/mkdocs_llmstxt-0.5.0.tar.gz", hash = "sha256:b2fa9e6d68df41d7467e948a4745725b6c99434a36b36204857dbd7bb3dfe041", size = 33909, upload-time = "2025-11-20T14:02:24.861Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/2b/82928cc9e8d9269cd79e7ebf015efdc4945e6c646e86ec1d4dba1707f215/mkdocs_llmstxt-0.5.0-py3-none-any.whl", hash = "sha256:753c699913d2d619a9072604b26b6dc9f5fb6d257d9b107857f80c8a0b787533", size = 12040, upload-time = "2025-11-20T14:02:23.483Z" }, +] + [[package]] name = "mkdocs-material" version = "9.7.0" @@ -2063,7 +2118,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [