From afaa80afb133c25d1d8a2525ca350a7703e42ed0 Mon Sep 17 00:00:00 2001 From: Xin Han Date: Sun, 16 Aug 2026 20:05:29 +1000 Subject: [PATCH] Generate the estimator tables from one source The tables listing what IKPyKit implements existed in three places: the root README, docs/README.md, and, since the API reference became generated, api/index.md. They had drifted, and the drift is the kind nobody notices: - IForest was published at TKDD in 2012. The summary matrix said TKDD'12 and the table below it said TKDD2022, on the same page. - STREAMKHC was described as "Isolation Distribution Kernel for Trajectory Anomaly Detections", which is IKAT's description, copied. - TIDKC's said "Tajectory". - The matrix called IsoDisKernel "IsodisKernel", and the class STREAMKHC "StreaKHC". Only half of each table can come from the code. The class names come from the __all__ of each module and the descriptions from the docstrings, but the application label and the publication venue are nowhere in the package. So scripts/estimators.py holds those, one entry per estimator, and renders every table from them. Adding an estimator means one entry instead of three tables, and check_complete() fails the build when a public class has no entry, rather than leaving it silently absent the way the old tables did. The root README never passes through MkDocs, so no plugin can reach it. Both READMEs carry the tables between markers instead, filled in by scripts/sync_readme_tables.py. The lint job runs it with --check; a pre-commit hook runs the same check for anyone who installed the hooks. The comparison ignores the padding around table pipes, so a formatter realigning them does not read as drift. Content that changes as a result, beyond the fixes above: sections use the names the docs navigation already uses ("Point Anomaly Detection" rather than "Point Anomaly detection", "Graph Mining" rather than "Graph Data"), rows follow __all__ order, and estimators sharing a cell in the summary matrix are separated by
rather than spilling into blank rows. The API overview gained a publication column from the same data. Co-Authored-By: Claude Opus 5 --- .github/workflows/lint.yml | 6 + .pre-commit-config.yaml | 11 ++ README.md | 102 +++++------ docs/README.md | 102 +++++------ scripts/estimators.py | 319 ++++++++++++++++++++++++++++++++++ scripts/gen_api_pages.py | 120 ++++--------- scripts/sync_readme_tables.py | 99 +++++++++++ 7 files changed, 573 insertions(+), 186 deletions(-) create mode 100644 scripts/estimators.py create mode 100644 scripts/sync_readme_tables.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e60776e..aafb17c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,3 +28,9 @@ jobs: - name: Run linter run: uv run ruff check . + + # The pre-commit hook only helps people who installed it. This is the gate: + # the root README is outside MkDocs, so nothing else notices it drifting + # from the package. + - name: Check the estimator tables match the package + run: uv run --no-sync python scripts/sync_readme_tables.py --check diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6ca98c..3473d2f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,17 @@ repos: hooks: - id: black + - repo: local + hooks: + # The root README never passes through MkDocs, so no plugin can keep its + # estimator tables in step with the docs. CI runs the same check. + - id: sync-readme-tables + name: estimator tables are current + entry: uv run python scripts/sync_readme_tables.py --check + language: system + pass_filenames: false + files: ^(README\.md|docs/README\.md|scripts/estimators\.py|ikpykit/.*/__init__\.py)$ + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: diff --git a/README.md b/README.md index a55dfb1..53b7b8c 100644 --- a/README.md +++ b/README.md @@ -82,74 +82,76 @@ clf.predict([[0.1, 0.3], [0, 0.7], [90, 85]]) ## Implemented Algorithms + + #### Summary -| Algorithms | Kernel Similarity | Anomaly Detection | Clustering | Change Detection | -| --------------- | ------------------------------ | -------------------------- | -------------------- | ---------------- | -| Point Data | IsoKernel (AAAI'19, SIGKDD'18) | IForest (ICDM'08, TKDD'12) | IDKC (IS'23) | | -| | | INNE (CIJ'18) | PSKC (TKDE'23) | | -| | | IDKD (TKDE'22) | IKAHC (PRJ'23) | | -| Graph Data | IsoGraphKernel (AAAI'21) | IKGOD (SIAM'23) | | | -| Group Data | IsodisKernel (SIGKDD'20) | IKGAD (TKDE'22) | | | -| Stream Data | | | StreaKHC (SIGKDD'22) | ICID (JAIR'24) | -| Time Series | | IKTOD (VLDB'22) | | | -| Trajectory Data | | IKAT (JAIR'24) | TIDKC (ICDM'23) | | +| Algorithms | Kernel Similarity | Anomaly Detection | Clustering | Change Detection | +| --- | --- | --- | --- | --- | +| Point Data | IsoKernel (AAAI'19, SIGKDD'18) | IForest (ICDM'08, TKDD'12)
INNE (CIJ'18)
IDKD (TKDE'22) | IDKC (IS'23)
PSKC (TKDE'23)
IKAHC (PR'23) | | +| Graph Data | IsoGraphKernel (AAAI'21) | IKGOD (SIAM'23) | | | +| Group Data | IsoDisKernel (SIGKDD'20) | IKGAD (TKDE'22) | | | +| Stream Data | | | STREAMKHC (SIGKDD'22) | ICID (JAIR'24) | +| Time Series | | IKTOD (VLDB'22) | | | +| Trajectory Data | | IKAT (JAIR'24) | TIDKC (ICDM'23) | | + +**(i) Isolation Kernel**: -**(i) Isolation Kernel** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IsoDisKernel](https://isolationkernel.github.io/ikpykit/latest/api/kernel/isodiskernel.html) | Isolation Distribution Kernel | Distribution similarity calculating | SIGKDD2020 | +| [IsoKernel](https://isolationkernel.github.io/ikpykit/latest/api/kernel/isokernel.html) | Isolation Kernel | IK feature mapping and similarity calculating | AAAI2019, SIGKDD2018 | -| Abbr | Algorithm | Application | Publication | -| ---------------------------------------------------------------------------------------------- | ----------------------------- | --------------------------------------------- | -------------------- | -| [IsoKernel](https://isolationkernel.github.io/ikpykit/latest/api/kernel/isolation_kernel.html) | Isolation Kernel | IK feature mapping and similarity calculating | AAAI2019, SIGKDD2018 | -| [IsoDisKernel](https://isolationkernel.github.io/ikpykit/latest/api/kernel/isolation_dis_kernel.html) | Isolation Distribution Kernel | Distribution similarity calculating | SIGKDD2020 | +**(ii) Point Anomaly Detection**: -**(ii) Point Anomaly detection** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IDKD](https://isolationkernel.github.io/ikpykit/latest/api/anomaly/idkd.html) | Isolation Distributional Kernel for point anomaly detections | Anomaly Detection | TKDE2022 | +| [INNE](https://isolationkernel.github.io/ikpykit/latest/api/anomaly/inne.html) | Isolation-based anomaly detection using nearest-neighbor ensembles | Anomaly Detection | CIJ2018 | +| [IForest](https://isolationkernel.github.io/ikpykit/latest/api/anomaly/iforest.html) | Isolation forest | Anomaly Detection | ICDM2008, TKDD2012 | -| Abbr | Algorithm | Application | Publication | -| ----------------------------------------------------------------------------- | ------------------------------------------------------------------ | ----------------- | ------------------ | -| [IForest](https://isolationkernel.github.io/ikpykit/latest/api/anomaly/iforest.html) | Isolation forest | Anomaly Detection | ICDM2008, TKDD2022 | -| [INNE](https://isolationkernel.github.io/ikpykit/latest/api/anomaly/inne.html) | Isolation-based anomaly detection using nearest-neighbor ensembles | Anomaly Detection | CIJ2018 | -| [IDKD](https://isolationkernel.github.io/ikpykit/latest/api/anomaly/idkd.html) | Isolation Distributional Kernel for point anomaly detections | Anomaly Detection | TKDE2022 | +**(iii) Point Clustering**: -**(iii) Point Clustering** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IDKC](https://isolationkernel.github.io/ikpykit/latest/api/cluster/idkc.html) | Kernel-based Clustering via Isolation Distributional Kernel | Point Clustering | IS2023 | +| [IKAHC](https://isolationkernel.github.io/ikpykit/latest/api/cluster/ikahc.html) | Isolation Kernel for Agglomerative Hierarchical Clustering | Hierarchical Clustering | PR2023 | +| [PSKC](https://isolationkernel.github.io/ikpykit/latest/api/cluster/pskc.html) | Point-set Kernel Clustering | Point Clustering | TKDE2023 | -| Abbr | Algorithm | Application | Publication | -| ------------------------------------------------------------------------- | ------------------------------------------------------------ | ----------------------- | ----------- | -| [IDKC](https://isolationkernel.github.io/ikpykit/latest/api/cluster/idkc.html) | Kernel-based Clustering via Isolation Distributional Kernel. | Point Clustering | IS2023 | -| [PSKC](https://isolationkernel.github.io/ikpykit/latest/api/cluster/pskc.html) | Point-set Kernel Clustering | Point Clustering | TKDE2023 | -| [IKAHC](https://isolationkernel.github.io/ikpykit/latest/api/cluster/ikahc.html) | Isolation Kernel for Agglomerative Hierarchical Clustering | Hierarchical Clustering | PR2023 | +**(iv) Graph Mining**: -**(IV) Graph Data** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IKGOD](https://isolationkernel.github.io/ikpykit/latest/api/graph/ikgod.html) | Subgraph Centralization: A Necessary Step for Graph Anomaly Detection | Graph Anomaly Detection | SIAM2023 | +| [IsoGraphKernel](https://isolationkernel.github.io/ikpykit/latest/api/graph/isographkernel.html) | Isolation Graph Kernel | Graph IK embedding and similarity calculating | AAAI2021 | -| Abbr | Algorithm | Application | Publication | -| ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | --------------------------------------------- | ----------- | -| [IKGOD](https://isolationkernel.github.io/ikpykit/latest/api/graph/ikgod.html) | Subgraph Centralization: A Necessary Step for Graph Anomaly Detection. | Graph Anomaly Detection | SIAM2023 | -| [IsoGraphKernel](https://isolationkernel.github.io/ikpykit/latest/api/graph/IsoGraphKernel.html) | Isolation Graph Kernel | Graph IK embedding and similarity calculating | AAAI2021 | +**(v) Group Mining**: -**(V) Group Data** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IKGAD](https://isolationkernel.github.io/ikpykit/latest/api/group/ikgad.html) | Isolation Distributional Kernel for group anomaly detections | Group Anomaly Detection | TKDE2022 | -| Abbr | Algorithm | Application | Publication | -| ----------------------------------------------------------------------- | ------------------------------------------------------------ | ----------------------- | ----------- | -| [IKGAD](https://isolationkernel.github.io/ikpykit/latest/api/group/ikgad.html) | Isolation Distributional Kernel for group anomaly detections | Group Anomaly Detection | TKDE2022 | +**(vi) Stream Mining**: -**(VI) Stream Data** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [ICID](https://isolationkernel.github.io/ikpykit/latest/api/stream/icid.html) | Detecting change intervals with isolation distributional kernel | Change Intervals Detection | JAIR2024 | +| [STREAMKHC](https://isolationkernel.github.io/ikpykit/latest/api/stream/streamkhc.html) | Streaming Hierarchical Clustering Based on Point-Set Kernel | Online Hierarchical Clustering | SIGKDD2022 | -| Abbr | Algorithm | Application | Publication | -| ------------------------------------------------------------------------------ | --------------------------------------------------------------- | ------------------------------ | ----------- | -| [StreaKHC](https://isolationkernel.github.io/ikpykit/latest/api/stream/streakhc.html) | Isolation Distribution Kernel for Trajectory Anomaly Detections | Online Hierarchical Clustering | SIGKDD2022 | -| [ICID](https://isolationkernel.github.io/ikpykit/latest/api/stream/icid.html) | Detecting change intervals with isolation distributional kernel | Change Intervals Detection | JAIR2024 | +**(vii) Trajectory Mining**: -**(VII) Trajectory Data** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IKAT](https://isolationkernel.github.io/ikpykit/latest/api/trajectory/ikat.html) | Isolation Distribution Kernel for Trajectory Anomaly Detections | Trajectory Anomaly Detection | JAIR2024 | +| [TIDKC](https://isolationkernel.github.io/ikpykit/latest/api/trajectory/tidkc.html) | Distribution-based Trajectory Clustering | Trajectory Clustering | ICDM2023 | -| Abbr | Algorithm | Application | Publication | -| ---------------------------------------------------------------------------- | --------------------------------------------------------------- | ---------------------------- | ----------- | -| [TIDKC](https://isolationkernel.github.io/ikpykit/latest/api/trajectory/tidkc.html) | Distribution-based Tajectory Clustering | Trajectory Clustering | ICDM2023 | -| [IKAT](https://isolationkernel.github.io/ikpykit/latest/api/trajectory/ikat.html) | Isolation Distribution Kernel for Trajectory Anomaly Detections | Trajectory Anomaly Detection | JAIR2024 | +**(viii) Time Series Mining**: -**(VIII) Time Series** +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IKTOD](https://isolationkernel.github.io/ikpykit/latest/api/timeseries/iktod.html) | Isolation distribution kernel for Time Series Anomaly Detection | Anomaly Detection | VLDB2022 | -| Abbr | Algorithm | Application | Publication | -| ----------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------- | ----------- | -| [IKTOD](https://isolationkernel.github.io/ikpykit/latest/api/time_series/iktod.html) | Isolation distribution kernel for Time Series Anomaly Detection | Anomaly detection | VLDB2022 | + --- diff --git a/docs/README.md b/docs/README.md index f24f0ff..b8cf912 100644 --- a/docs/README.md +++ b/docs/README.md @@ -87,74 +87,76 @@ clf.predict([[0.1, 0.3], [0, 0.7], [90, 85]]) ## Implemented Algorithms + + #### Summary -| Algorithms | Kernel Similarity | Anomaly Detection | Clustering | Change Detection | -| --------------- | ------------------------------ | -------------------------- | -------------------- | ---------------- | -| Point Data | IsoKernel (AAAI'19, SIGKDD'18) | IForest (ICDM'08, TKDD'12) | IDKC (IS'23) | | -| | | INNE (CIJ'18) | PSKC (TKDE'23) | | -| | | IDKD (TKDE'22) | IKAHC (PRJ'23) | | -| Graph Data | IsoGraphKernel (AAAI'21) | IKGOD (SIAM'23) | | | -| Group Data | IsodisKernel (SIGKDD'20) | IKGAD (TKDE'22) | | | -| Stream Data | | | StreaKHC (SIGKDD'22) | ICID (JAIR'24) | -| Time Series | | IKTOD (VLDB'22) | | | -| Trajectory Data | | IKAT (JAIR'24) | TIDKC (ICDM'23) | | +| Algorithms | Kernel Similarity | Anomaly Detection | Clustering | Change Detection | +| --- | --- | --- | --- | --- | +| Point Data | IsoKernel (AAAI'19, SIGKDD'18) | IForest (ICDM'08, TKDD'12)
INNE (CIJ'18)
IDKD (TKDE'22) | IDKC (IS'23)
PSKC (TKDE'23)
IKAHC (PR'23) | | +| Graph Data | IsoGraphKernel (AAAI'21) | IKGOD (SIAM'23) | | | +| Group Data | IsoDisKernel (SIGKDD'20) | IKGAD (TKDE'22) | | | +| Stream Data | | | STREAMKHC (SIGKDD'22) | ICID (JAIR'24) | +| Time Series | | IKTOD (VLDB'22) | | | +| Trajectory Data | | IKAT (JAIR'24) | TIDKC (ICDM'23) | | + +**(i) Isolation Kernel**: -**(i) Isolation Kernel** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IsoDisKernel](./api/kernel/isodiskernel.md) | Isolation Distribution Kernel | Distribution similarity calculating | SIGKDD2020 | +| [IsoKernel](./api/kernel/isokernel.md) | Isolation Kernel | IK feature mapping and similarity calculating | AAAI2019, SIGKDD2018 | -| Abbr | Algorithm | Application | Publication | -| ---------------------------------------------------------------------------------------------- | ----------------------------- | --------------------------------------------- | -------------------- | -| [IsoKernel](./api/kernel/isokernel.md) | Isolation Kernel | IK feature mapping and similarity calculating | AAAI2019, SIGKDD2018 | -| [IsoDisKernel](./api/kernel/isodiskernel.md) | Isolation Distribution Kernel | Distribution similarity calculating | SIGKDD2020 | +**(ii) Point Anomaly Detection**: -**(ii) Point Anomaly detection** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IDKD](./api/anomaly/idkd.md) | Isolation Distributional Kernel for point anomaly detections | Anomaly Detection | TKDE2022 | +| [INNE](./api/anomaly/inne.md) | Isolation-based anomaly detection using nearest-neighbor ensembles | Anomaly Detection | CIJ2018 | +| [IForest](./api/anomaly/iforest.md) | Isolation forest | Anomaly Detection | ICDM2008, TKDD2012 | -| Abbr | Algorithm | Application | Publication | -| ----------------------------------------------------------------------------- | ------------------------------------------------------------------ | ----------------- | ------------------ | -| [IForest](./api/anomaly/iforest.md) | Isolation forest | Anomaly Detection | ICDM2008, TKDD2022 | -| [INNE](./api/anomaly/inne.md) | Isolation-based anomaly detection using nearest-neighbor ensembles | Anomaly Detection | CIJ2018 | -| [IDKD](./api/anomaly/idkd.md) | Isolation Distributional Kernel for point anomaly detections | Anomaly Detection | TKDE2022 | +**(iii) Point Clustering**: -**(iii) Point Clustering** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IDKC](./api/cluster/idkc.md) | Kernel-based Clustering via Isolation Distributional Kernel | Point Clustering | IS2023 | +| [IKAHC](./api/cluster/ikahc.md) | Isolation Kernel for Agglomerative Hierarchical Clustering | Hierarchical Clustering | PR2023 | +| [PSKC](./api/cluster/pskc.md) | Point-set Kernel Clustering | Point Clustering | TKDE2023 | -| Abbr | Algorithm | Application | Publication | -| ------------------------------------------------------------------------- | ------------------------------------------------------------ | ----------------------- | ----------- | -| [IDKC](./api/cluster/idkc.md) | Kernel-based Clustering via Isolation Distributional Kernel. | Point Clustering | IS2023 | -| [PSKC](./api/cluster/pskc.md) | Point-set Kernel Clustering | Point Clustering | TKDE2023 | -| [IKAHC](./api/cluster/ikahc.md) | Isolation Kernel for Agglomerative Hierarchical Clustering | Hierarchical Clustering | PR2023 | +**(iv) Graph Mining**: -**(IV) Graph Data** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IKGOD](./api/graph/ikgod.md) | Subgraph Centralization: A Necessary Step for Graph Anomaly Detection | Graph Anomaly Detection | SIAM2023 | +| [IsoGraphKernel](./api/graph/isographkernel.md) | Isolation Graph Kernel | Graph IK embedding and similarity calculating | AAAI2021 | -| Abbr | Algorithm | Application | Publication | -| ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | --------------------------------------------- | ----------- | -| [IKGOD](./api/graph/ikgod.md) | Subgraph Centralization: A Necessary Step for Graph Anomaly Detection. | Graph Anomaly Detection | SIAM2023 | -| [IsoGraphKernel](./api/graph/isographkernel.md) | Isolation Graph Kernel | Graph IK embedding and similarity calculating | AAAI2021 | +**(v) Group Mining**: -**(V) Group Data** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IKGAD](./api/group/ikgad.md) | Isolation Distributional Kernel for group anomaly detections | Group Anomaly Detection | TKDE2022 | -| Abbr | Algorithm | Application | Publication | -| ----------------------------------------------------------------------- | ------------------------------------------------------------ | ----------------------- | ----------- | -| [IKGAD](./api/group/ikgad.md) | Isolation Distributional Kernel for group anomaly detections | Group Anomaly Detection | TKDE2022 | +**(vi) Stream Mining**: -**(VI) Stream Data** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [ICID](./api/stream/icid.md) | Detecting change intervals with isolation distributional kernel | Change Intervals Detection | JAIR2024 | +| [STREAMKHC](./api/stream/streamkhc.md) | Streaming Hierarchical Clustering Based on Point-Set Kernel | Online Hierarchical Clustering | SIGKDD2022 | -| Abbr | Algorithm | Application | Publication | -| ------------------------------------------------------------------------------ | --------------------------------------------------------------- | ------------------------------ | ----------- | -| [StreaKHC](./api/stream/streamkhc.md) | Isolation Distribution Kernel for Trajectory Anomaly Detections | Online Hierarchical Clustering | SIGKDD2022 | -| [ICID](./api/stream/icid.md) | Detecting change intervals with isolation distributional kernel | Change Intervals Detection | JAIR2024 | +**(vii) Trajectory Mining**: -**(VII) Trajectory Data** : +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IKAT](./api/trajectory/ikat.md) | Isolation Distribution Kernel for Trajectory Anomaly Detections | Trajectory Anomaly Detection | JAIR2024 | +| [TIDKC](./api/trajectory/tidkc.md) | Distribution-based Trajectory Clustering | Trajectory Clustering | ICDM2023 | -| Abbr | Algorithm | Application | Publication | -| ---------------------------------------------------------------------------- | --------------------------------------------------------------- | ---------------------------- | ----------- | -| [TIDKC](./api/trajectory/tidkc.md) | Distribution-based Tajectory Clustering | Trajectory Clustering | ICDM2023 | -| [IKAT](./api/trajectory/ikat.md) | Isolation Distribution Kernel for Trajectory Anomaly Detections | Trajectory Anomaly Detection | JAIR2024 | +**(viii) Time Series Mining**: -**(VIII) Time Series** +| Abbr | Algorithm | Application | Publication | +| --- | --- | --- | --- | +| [IKTOD](./api/timeseries/iktod.md) | Isolation distribution kernel for Time Series Anomaly Detection | Anomaly Detection | VLDB2022 | -| Abbr | Algorithm | Application | Publication | -| ----------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------- | ----------- | -| [IKTOD](./api/timeseries/iktod.md) | Isolation distribution kernel for Time Series Anomaly Detection | Anomaly detection | VLDB2022 | + --- diff --git a/scripts/estimators.py b/scripts/estimators.py new file mode 100644 index 0000000..04dff0a --- /dev/null +++ b/scripts/estimators.py @@ -0,0 +1,319 @@ +"""The single description of what IKPyKit implements. + +The estimator tables existed in three places that had drifted apart: the root +``README.md``, ``docs/README.md`` and, since the API reference was generated, +``api/index.md``. This module holds the parts that cannot be read off the code +and renders every one of those tables from them, so adding an estimator means +editing one entry rather than three tables. + +What the code already states is not repeated here. The class list comes from the +``__all__`` of each module, and the one-line description from the first +paragraph of its docstring. +""" + +from __future__ import annotations + +import importlib +import inspect +import re +from collections.abc import Callable +from dataclasses import dataclass + +# Longest description to put in a page's front matter. Social cards and search +# engines both cut off around here, so a longer one is only truncated elsewhere. +DESCRIPTION_LIMIT = 160 + +API_ROOT = "api" + + +@dataclass(frozen=True) +class Section: + """A group of estimators shown under one heading.""" + + title: str + module: str + subsections: tuple[Section, ...] = () + # Dataset loaders belong in the API reference but not in a table of + # algorithms, so a section can opt out of the README tables. + in_readme: bool = True + + +SECTIONS = ( + Section("Isolation Kernel", "ikpykit.kernel"), + Section("Point Anomaly Detection", "ikpykit.anomaly"), + Section("Point Clustering", "ikpykit.cluster"), + Section("Graph Mining", "ikpykit.graph"), + Section("Group Mining", "ikpykit.group"), + Section("Stream Mining", "ikpykit.stream"), + Section( + "Trajectory Mining", + "ikpykit.trajectory", + subsections=( + Section("DataLoader", "ikpykit.trajectory.dataloader", in_readme=False), + ), + ), + Section("Time Series Mining", "ikpykit.timeseries"), +) + + +@dataclass(frozen=True) +class Estimator: + """What the tables say about an estimator that the code does not. + + ``data_type`` and ``capability`` place it in the summary matrix; they are the + row and the column it appears in. + """ + + algorithm: str + application: str + publications: tuple[str, ...] + data_type: str + capability: str + + +# Every public class outside a section marked `in_readme=False` needs an entry +# here. `check_complete` enforces that, so a new estimator cannot quietly end up +# missing from the tables the way it used to. +ESTIMATORS = { + "IsoKernel": Estimator( + algorithm="Isolation Kernel", + application="IK feature mapping and similarity calculating", + publications=("AAAI2019", "SIGKDD2018"), + data_type="Point Data", + capability="Kernel Similarity", + ), + "IsoDisKernel": Estimator( + algorithm="Isolation Distribution Kernel", + application="Distribution similarity calculating", + publications=("SIGKDD2020",), + data_type="Group Data", + capability="Kernel Similarity", + ), + "IForest": Estimator( + algorithm="Isolation forest", + application="Anomaly Detection", + publications=("ICDM2008", "TKDD2012"), + data_type="Point Data", + capability="Anomaly Detection", + ), + "INNE": Estimator( + algorithm="Isolation-based anomaly detection using nearest-neighbor ensembles", + application="Anomaly Detection", + publications=("CIJ2018",), + data_type="Point Data", + capability="Anomaly Detection", + ), + "IDKD": Estimator( + algorithm="Isolation Distributional Kernel for point anomaly detections", + application="Anomaly Detection", + publications=("TKDE2022",), + data_type="Point Data", + capability="Anomaly Detection", + ), + "IDKC": Estimator( + algorithm="Kernel-based Clustering via Isolation Distributional Kernel", + application="Point Clustering", + publications=("IS2023",), + data_type="Point Data", + capability="Clustering", + ), + "PSKC": Estimator( + algorithm="Point-set Kernel Clustering", + application="Point Clustering", + publications=("TKDE2023",), + data_type="Point Data", + capability="Clustering", + ), + "IKAHC": Estimator( + algorithm="Isolation Kernel for Agglomerative Hierarchical Clustering", + application="Hierarchical Clustering", + publications=("PR2023",), + data_type="Point Data", + capability="Clustering", + ), + "IKGOD": Estimator( + algorithm="Subgraph Centralization: A Necessary Step for Graph Anomaly Detection", + application="Graph Anomaly Detection", + publications=("SIAM2023",), + data_type="Graph Data", + capability="Anomaly Detection", + ), + "IsoGraphKernel": Estimator( + algorithm="Isolation Graph Kernel", + application="Graph IK embedding and similarity calculating", + publications=("AAAI2021",), + data_type="Graph Data", + capability="Kernel Similarity", + ), + "IKGAD": Estimator( + algorithm="Isolation Distributional Kernel for group anomaly detections", + application="Group Anomaly Detection", + publications=("TKDE2022",), + data_type="Group Data", + capability="Anomaly Detection", + ), + "ICID": Estimator( + algorithm="Detecting change intervals with isolation distributional kernel", + application="Change Intervals Detection", + publications=("JAIR2024",), + data_type="Stream Data", + capability="Change Detection", + ), + "STREAMKHC": Estimator( + algorithm="Streaming Hierarchical Clustering Based on Point-Set Kernel", + application="Online Hierarchical Clustering", + publications=("SIGKDD2022",), + data_type="Stream Data", + capability="Clustering", + ), + "IKAT": Estimator( + algorithm="Isolation Distribution Kernel for Trajectory Anomaly Detections", + application="Trajectory Anomaly Detection", + publications=("JAIR2024",), + data_type="Trajectory Data", + capability="Anomaly Detection", + ), + "TIDKC": Estimator( + algorithm="Distribution-based Trajectory Clustering", + application="Trajectory Clustering", + publications=("ICDM2023",), + data_type="Trajectory Data", + capability="Clustering", + ), + "IKTOD": Estimator( + algorithm="Isolation distribution kernel for Time Series Anomaly Detection", + application="Anomaly Detection", + publications=("VLDB2022",), + data_type="Time Series", + capability="Anomaly Detection", + ), +} + +MATRIX_ROWS = ( + "Point Data", + "Graph Data", + "Group Data", + "Stream Data", + "Time Series", + "Trajectory Data", +) +MATRIX_COLUMNS = ( + "Kernel Similarity", + "Anomaly Detection", + "Clustering", + "Change Detection", +) + +ROMAN = ("i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix", "x") + + +def summarize_text(text: str) -> str: + """Collapse text to a single line, keeping whole sentences within the limit. + + Text longer than the limit is cut at the last sentence that still fits, + rather than mid-word. + """ + line = " ".join(text.split()) + if len(line) <= DESCRIPTION_LIMIT: + return line + + kept = "" + for sentence in re.findall(r"[^.]*\.(?:\s|$)", line): + if len(kept) + len(sentence) > DESCRIPTION_LIMIT: + break + kept += sentence + # A first sentence that is itself over the limit leaves nothing to keep. + return kept.strip() or line[:DESCRIPTION_LIMIT].rstrip() + + +def summarize(obj: object) -> str: + """Return the first paragraph of an object's docstring as a single line.""" + doc = inspect.getdoc(obj) or "" + return summarize_text(doc.split("\n\n")[0]) + + +def page_path(module: str, name: str) -> str: + """Return the doc path for a class, mirroring its location in the package.""" + package = module.removeprefix("ikpykit.").replace(".", "/") + return f"{package}/{name.lower()}.md" + + +def collect(section: Section) -> list[tuple[str, str, str]]: + """Return (module, name, description) for every public class in a section.""" + module = importlib.import_module(section.module) + return [ + (section.module, name, summarize(getattr(module, name))) + for name in module.__all__ + ] + + +def readme_sections() -> list[tuple[Section, list[tuple[str, str, str]]]]: + """Return the sections that appear in the README tables, with their classes.""" + return [(s, collect(s)) for s in SECTIONS if s.in_readme] + + +def check_complete() -> None: + """Fail if an estimator has no table entry, or an entry has no estimator.""" + public = {name for _, entries in readme_sections() for _, name, _ in entries} + missing = sorted(public - ESTIMATORS.keys()) + if missing: + raise SystemExit( + f"No entry in ESTIMATORS for {', '.join(missing)}. Add one in " + f"{__file__} so the tables include it." + ) + extra = sorted(ESTIMATORS.keys() - public) + if extra: + raise SystemExit( + f"ESTIMATORS has entries for {', '.join(extra)}, which no module " + f"exports. Remove them from {__file__}." + ) + + +def short_publication(publication: str) -> str: + """Shorten a publication for the summary matrix: AAAI2019 -> AAAI'19.""" + return re.sub(r"^(\D+)\d{2}(\d{2})$", r"\1'\2", publication) + + +def render_summary_matrix() -> list[str]: + """Render the matrix of which estimators cover which task on which data.""" + cells: dict[tuple[str, str], list[str]] = {} + for name, meta in ESTIMATORS.items(): + short = ", ".join(short_publication(p) for p in meta.publications) + cells.setdefault((meta.data_type, meta.capability), []).append( + f"{name} ({short})" + ) + + lines = [ + "| Algorithms | " + " | ".join(MATRIX_COLUMNS) + " |", + "| --- | " + " | ".join("---" for _ in MATRIX_COLUMNS) + " |", + ] + for row in MATRIX_ROWS: + # Several estimators can share a cell;
keeps them in one row rather + # than spilling into blank rows underneath. + filled = ["
".join(cells.get((row, col), [])) for col in MATRIX_COLUMNS] + lines.append(f"| {row} | " + " | ".join(filled) + " |") + return lines + + +def render_tables(link: Callable[[str, str], str]) -> str: + """Render the whole generated block, linking each class through `link`. + + `link` takes the module and the class name and returns a URL, which differs + between the two READMEs: one is read on GitHub, the other inside the site. + """ + lines = ["#### Summary", "", *render_summary_matrix(), ""] + for number, (section, entries) in zip(ROMAN, readme_sections(), strict=False): + lines += [ + f"**({number}) {section.title}**:", + "", + "| Abbr | Algorithm | Application | Publication |", + "| --- | --- | --- | --- |", + ] + for module, name, _ in entries: + meta = ESTIMATORS[name] + lines.append( + f"| [{name}]({link(module, name)}) | {meta.algorithm} | " + f"{meta.application} | {', '.join(meta.publications)} |" + ) + lines.append("") + return "\n".join(lines).rstrip() diff --git a/scripts/gen_api_pages.py b/scripts/gen_api_pages.py index db31de9..51d6f53 100644 --- a/scripts/gen_api_pages.py +++ b/scripts/gen_api_pages.py @@ -2,92 +2,38 @@ Every public estimator used to need a one-line stub under ``docs/api/`` and a matching entry in the ``nav`` of ``mkdocs.yml``. Both were easy to forget, and a -missing one failed silently: the class simply had no page. The package already -states what is public, through the ``__all__`` of each module, so that is what -this script reads. - -The only thing it cannot derive is the section names below, which are editorial, -and their order, which is meaningful. Adding an estimator to a module that is -already listed needs no change here. +missing one failed silently: the class simply had no page. What is public, and +how each class describes itself, is read from the package by estimators.py. Run by mkdocs-gen-files during the build; the pages exist only in the built site. """ from __future__ import annotations -import importlib -import inspect import json -import re +import pathlib +import sys import textwrap -from dataclasses import dataclass, field import mkdocs_gen_files from mkdocs.structure.files import InclusionLevel -# Longest description to put in front matter. Social cards and search engines -# both cut off around here, so a longer one is only truncated elsewhere. -DESCRIPTION_LIMIT = 160 - -API_ROOT = "api" - - -@dataclass(frozen=True) -class Section: - """A group of estimators shown under one heading in the navigation.""" - - title: str - module: str - subsections: tuple[Section, ...] = field(default_factory=tuple) - - -SECTIONS = ( - Section("Isolation Kernel", "ikpykit.kernel"), - Section("Point Anomaly Detection", "ikpykit.anomaly"), - Section("Point Clustering", "ikpykit.cluster"), - Section("Graph Mining", "ikpykit.graph"), - Section("Group Mining", "ikpykit.group"), - Section("Stream Mining", "ikpykit.stream"), - Section( - "Trajectory Mining", - "ikpykit.trajectory", - subsections=(Section("DataLoader", "ikpykit.trajectory.dataloader"),), - ), - Section("Time Series Mining", "ikpykit.timeseries"), +# mkdocs-gen-files runs this through runpy, which does not put the script's own +# directory on the path the way running it as a program would. +sys.path.insert(0, str(pathlib.Path(__file__).parent)) + +from estimators import ( # noqa: E402 + API_ROOT, + ESTIMATORS, + SECTIONS, + Section, + check_complete, + collect, + page_path, + summarize_text, ) -def summarize_text(text: str) -> str: - """Collapse text to a single line, keeping whole sentences within the limit. - - Text longer than the limit is cut at the last sentence that still fits, - rather than mid-word. - """ - line = " ".join(text.split()) - if len(line) <= DESCRIPTION_LIMIT: - return line - - kept = "" - for sentence in re.findall(r"[^.]*\.(?:\s|$)", line): - if len(kept) + len(sentence) > DESCRIPTION_LIMIT: - break - kept += sentence - # A first sentence that is itself over the limit leaves nothing to keep. - return kept.strip() or line[:DESCRIPTION_LIMIT].rstrip() - - -def summarize(obj: object) -> str: - """Return the first paragraph of an object's docstring as a single line.""" - doc = inspect.getdoc(obj) or "" - return summarize_text(doc.split("\n\n")[0]) - - -def page_path(module: str, name: str) -> str: - """Return the doc path for a class, mirroring its location in the package.""" - package = module.removeprefix("ikpykit.").replace(".", "/") - return f"{package}/{name.lower()}.md" - - def write_page(module: str, name: str, description: str) -> None: """Write the stub that mkdocstrings expands into the rendered class page.""" path = page_path(module, name) @@ -99,16 +45,6 @@ def write_page(module: str, name: str, description: str) -> None: page.write(f"::: {module}.{name}\n") -def collect(section: Section) -> list[tuple[str, str, str]]: - """Return (module, name, description) for every public class in a section.""" - module = importlib.import_module(section.module) - entries = [] - for name in module.__all__: - obj = getattr(module, name) - entries.append((section.module, name, summarize(obj))) - return entries - - def render_nav(section: Section, depth: int = 0) -> list[str]: """Render one section of the literate-nav SUMMARY, subsections first.""" indent = " " * depth @@ -121,15 +57,25 @@ def render_nav(section: Section, depth: int = 0) -> list[str]: def render_overview(section: Section, level: int = 2) -> list[str]: - """Render one section of the overview page as a table of its estimators.""" + """Render one section of the overview page as a table of its estimators. + + The publication column only exists for estimators the READMEs list, so + dataset loaders get the two-column form. + """ + entries = collect(section) + described = all(name in ESTIMATORS for _, name, _ in entries) + header = ["Estimator", "Description"] + (["Publication"] if described else []) lines = [ f"{'#' * level} {section.title}", "", - "| Estimator | Description |", - "| --- | --- |", + "| " + " | ".join(header) + " |", + "| " + " | ".join("---" for _ in header) + " |", ] - for module, name, description in collect(section): - lines.append(f"| [{name}]({page_path(module, name)}) | {description} |") + for module, name, description in entries: + row = [f"[{name}]({page_path(module, name)})", description] + if described: + row.append(", ".join(ESTIMATORS[name].publications)) + lines.append("| " + " | ".join(row) + " |") lines.append("") for subsection in section.subsections: lines += render_overview(subsection, level + 1) @@ -137,6 +83,8 @@ def render_overview(section: Section, level: int = 2) -> list[str]: def main() -> None: + check_complete() + for section in SECTIONS: for subsection in (section, *section.subsections): for module, name, description in collect(subsection): diff --git a/scripts/sync_readme_tables.py b/scripts/sync_readme_tables.py new file mode 100644 index 0000000..b72465c --- /dev/null +++ b/scripts/sync_readme_tables.py @@ -0,0 +1,99 @@ +"""Write the estimator tables into both READMEs, or check that they are current. + +The root README is read on GitHub and never passes through MkDocs, so a plugin +cannot reach it. Instead both files carry the tables between markers and this +script fills them in from scripts/estimators.py. Run with --check by pre-commit, +which fails if either file has fallen behind. + + python scripts/sync_readme_tables.py # rewrite both READMEs + python scripts/sync_readme_tables.py --check # exit 1 if out of date +""" + +from __future__ import annotations + +import argparse +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent)) + +from estimators import API_ROOT, check_complete, page_path, render_tables # noqa: E402 + +BEGIN = "" +END = "" + +SITE = "https://isolationkernel.github.io/ikpykit/latest" + + +def site_link(module: str, name: str) -> str: + """Absolute link for the root README, which is read outside the site.""" + return f"{SITE}/{API_ROOT}/{page_path(module, name)}".replace(".md", ".html") + + +def docs_link(module: str, name: str) -> str: + """Relative link for docs/README.md, which MkDocs resolves and checks.""" + return f"./{API_ROOT}/{page_path(module, name)}" + + +TARGETS = { + pathlib.Path("README.md"): site_link, + pathlib.Path("docs/README.md"): docs_link, +} + + +def bounds(text: str, path: pathlib.Path) -> tuple[int, int]: + start, end = text.find(BEGIN), text.find(END) + if start == -1 or end == -1: + raise SystemExit(f"{path}: missing the {BEGIN} / {END} markers.") + return start, end + + +def normalize(block: str) -> str: + """Collapse the whitespace used to align table pipes. + + Editors and Markdown formatters pad table cells so the pipes line up, which + is how these tables were kept before. Comparing normalized forms means a + padded table still counts as up to date, rather than this script and a + formatter overwriting each other on every commit. + """ + return "\n".join( + " ".join(line.split()) for line in block.splitlines() if line.strip() + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="do not write; exit non-zero if a README is out of date", + ) + args = parser.parse_args() + + check_complete() + + stale = [] + for path, link in TARGETS.items(): + current = path.read_text() + start, end = bounds(current, path) + body = render_tables(link) + if normalize(current[start + len(BEGIN) : end]) == normalize(body): + continue + if args.check: + stale.append(path) + else: + path.write_text(f"{current[:start]}{BEGIN}\n\n{body}\n\n{current[end:]}") + print(f"updated {path}") + + if stale: + names = ", ".join(str(p) for p in stale) + print( + f"{names} out of date. Run `python {__file__}` to regenerate.", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())