diff --git a/.github/workflows/offline-freeze.yml b/.github/workflows/offline-freeze.yml new file mode 100644 index 00000000..99d80dda --- /dev/null +++ b/.github/workflows/offline-freeze.yml @@ -0,0 +1,24 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +name: Offline wheels freeze + +on: + workflow_dispatch: + +jobs: + wheels: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + with: + python-version: "3.12" + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Freeze wheelhouse + run: bash bin/freeze_logstashui.sh --wheels + - name: Network-none install smoke + run: bash bin/test_freeze_wheels.sh diff --git a/.github/workflows/test-databases.yml b/.github/workflows/test-databases.yml new file mode 100644 index 00000000..599f31fe --- /dev/null +++ b/.github/workflows/test-databases.yml @@ -0,0 +1,24 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +name: Database matrix + +on: + pull_request: + push: + branches: [main, master] + +jobs: + db: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + with: + python-version: "3.12" + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Run database matrix + run: bash bin/test_databases.sh diff --git a/.gitignore b/.gitignore index 96ae9dd6..4ec32aae 100644 --- a/.gitignore +++ b/.gitignore @@ -223,7 +223,6 @@ __marimo__/ # LogstashUI /src/logstashui/staticfiles/ /src/logstashui/data/ -/.plans/ # Local config (use logstashui.example.yml) /src/logstashui/logstashui.yml @@ -233,6 +232,7 @@ __marimo__/ /docker/logstashui_data/ # Agentic / local tooling scratch (not product docs) +/.plans/ .grokignore /docs/superpowers/ .hermes.md @@ -242,6 +242,8 @@ CLAUDE.md .cursorrules IDEA.md graphify-out/ +.github/copilot-instructions.md +/scratch/ # LogstashAgent (cloned from GitHub in host mode) /LogstashAgent/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b960a19..ac717891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,109 @@ +## [0.5.2] - Multi-database + k8s - 09/06/2026 + +Package version is **0.5.2** (`pyproject.toml`). Preferred LogstashAgent version is **0.5.2** (lockstep). + +SQLite does not scale under gunicorn/gevent. Operators can now run LogstashUI on **SQLite** (default), **PostgreSQL 14+**, or **MariaDB 10.6+ / MySQL 8.0+** without changing the Django ORM data model. + +### Database engines + +- `LOGSTASHUI_DB_ENGINE=sqlite|postgresql|mysql` (MariaDB uses `mysql`). Aliases: `sqlite3`, `postgres`, `mariadb`, `my`. +- Discrete env only: `LOGSTASHUI_DB_HOST`, `PORT`, `NAME`, `USER`, `PASSWORD`, plus `LOGSTASHUI_DB_SSLMODE` / `LOGSTASHUI_DB_SSL_CA`, `LOGSTASHUI_DB_CONN_MAX_AGE` (default 60), `LOGSTASHUI_DB_CONN_HEALTH_CHECKS` (default true). No YAML. No `DATABASE_URL`. +- Unset engine is still SQLite at `$LOGSTASHUI_DATA_DIR/db.sqlite3` (WAL + `busy_timeout` unchanged). +- `logstashui serve` logs a warning when engine is SQLite and `LOGSTASHUI_WORKERS>1`; it does not refuse to start. +- Fail-fast on unknown engine, missing driver extra, missing HOST/USER, or server below version floors. `logstashui serve` checks the server version **before** `migrate` and still checks when `--skip-migrate` is set. +- `logstashui migrate-engine --to` accepts `postgresql`, `mysql`, and `mariadb` (`mariadb` is an alias of `mysql`). +- MySQL/MariaDB use `utf8mb4` / `utf8mb4_bin` so unique names match SQLite/Postgres case-sensitivity. Create the database with that collation. +- `LOGSTASHUI_DATA_DIR` is still required when the database is remote (TLS, Django secret, logs, staticfiles). +- gunicorn stays `--worker-class gevent`. PostgreSQL uses `psycopg[binary]`; MySQL/MariaDB use PyMySQL (not mysqlclient). Optional PgBouncer is documented, not required. + +### Packaging and Docker + +- Native extras: `LogstashUI[postgres]`, `LogstashUI[mysql]`, `LogstashUI[databases]`, `LogstashUI[otel]`. Default wheel stays SQLite-only. +- Optional air-gapped freeze: `bin/freeze_logstashui.sh` (`--wheels` / `--docker` / `--standalone`). Default `uv build` unchanged. Linux x86_64, CPython 3.12, `[databases]` and `[otel]` included, no Agent. Wheelhouse prefers manylinux2014 then manylinux_2_28; pure-Python sdists are wheeled on the builder. Standalone PyInstaller is experimental. See [Air-gapped freeze](docs/docs/logstashui/general/offline.md). +- Container image installs `LogstashUI[databases,otel]`. Tracing stays off until `LOGSTASHUI_OTEL=true`. Kubernetes only sets env. +- systemd generator prompts for engine/host/port/name/user; sample `/etc/default/logstashui` documents all `LOGSTASHUI_DB_*` keys. Set the password in the EnvironmentFile or a Secret (`chmod 640`). +- gunicorn writes `$LOGSTASHUI_DATA_DIR/gunicorn.pid`. + +### Migration off SQLite + +- **Supported offline path:** stop UI → back up `db.sqlite3` → `dumpdata` while still on SQLite → create the server database → set `LOGSTASHUI_DB_*` → `migrate` + `loaddata`. Keep `DATA_DIR` (same secret key) or encrypted keystore rows will not decrypt. Sessions are not copied; log in again. +- **BETA** `logstashui migrate-engine --to postgresql|mysql --i-have-a-backup` dumps the SQLite file, loads the target, and does **not** restart serve. It SIGTERMs gunicorn if a pidfile is live. Prefer `systemctl stop` first so `Restart=` does not race. Optional `--write-env` appends engine/host/name/user (never the password). +- BETA `migrate-engine` is **not atomic** on the target: `dumpdata` → `migrate` → `loaddata` is three steps. If `loaddata` fails, the target may be partially populated. Drop or recreate the target database (SQLite is only WAL-checkpointed) and re-run. A later production migrator could wrap `loaddata` and Postgres `sequence_reset_sql` in `transaction.atomic()` after `migrate`; `migrate` itself applies DDL and cannot be one atomic unit on MySQL/MariaDB. + +### Testing + +- Default `pytest` stays SQLite (no Docker, no extras). +- `bin/test_databases.sh` / `bin/test_databases.bat` start local Docker Postgres 16, MariaDB 11, and MySQL 8.0 and run the full suite on each engine, then run `tests/Database/` via testcontainers. CI workflow `.github/workflows/test-databases.yml` calls the same script. +- Self-contained database test suite at `tests/Database/integration/` uses `testcontainers` (postgres:16, mysql:8.0, mariadb:11 — no external Docker Compose). Parametrized over PostgreSQL and MySQL; MariaDB covered for `check_server_version` version-detection. Skips gracefully when Docker is unavailable. Covers DB config, migrations (clean, idempotent, no unapplied), ORM CRUD/JSON/uniqueness, and full SQLite → PG/MySQL/MariaDB `migrate-engine` round-trips. Run with `uv run pytest tests/Database/ -v --no-cov` after `uv sync --group dev --extra databases`. +- All tests migrated from `src/logstashui//tests/` to a dedicated `tests/` tree at the project root. Layout: `tests//unit/` per Django app, `tests/Database/unit|integration/` for database and migration tests. Shared fixtures moved from `Common/test_resources.py` to `tests/conftest.py` (auto-discovered by pytest). `testpaths` trimmed to `["tests"]`; `pythonpath = ["src/logstashui"]` unchanged so app imports still resolve. Run any app in isolation: `uv run pytest tests/SNMP -v`. +- `test_case_sensitive_unique` now asserts case-sensitive uniqueness at both layers. `Network.save()` calls `full_clean()`, so a duplicate name raises `ValidationError` from `validate_unique()` and the `INSERT` is never issued — the test previously expected an `IntegrityError` that the database could never raise, and failed identically on both engines. The collation check (`utf8mb4_bin` on MySQL, default on PostgreSQL) rides on the `validate_unique()` query; the database unique index is verified separately via `bulk_create()`, which bypasses `save()`, inside `transaction.atomic()` so PostgreSQL can roll back the aborted statement before cleanup. +- Container fixtures moved from the deprecated `testcontainers.postgres` / `testcontainers.mysql` shims to `testcontainers.community.*`. The dev-dependency floor is raised to `testcontainers>=4.15.0`, the first release containing that package. +- API token coverage in `tests/Common/unit/test_api_token_middleware.py` and `tests/Management/unit/test_api_tokens.py`. The load-bearing case is the inverse one: a logged-in session POSTing without a CSRF token must still be rejected, so the exemption cannot regress into a site-wide CSRF bypass. Also covers revoked/expired/inactive-owner tokens, readonly-owner denial, agent keys passing through untouched, and that re-saving a token does not double-hash it. +- Smoke compose is still SQLite (product CA / PUID unchanged). + +### Kubernetes and database docs + +- Kubernetes subsection: StatefulSet + one PVC at `/var/lib/logstashui`, TLS kept on `:8443`, Ingress-nginx skip backend verify, Envoy Gateway `Backend` `insecureSkipVerify` (enable Backend API), CloudNativePG Cluster in the app namespace. +- K8s probes send `Host: logstashui` (kubelet otherwise uses the pod IP and Django returns 400). Downward API `status.podIP` → `LOGSTASHUI_HOST_IPS` for the product leaf; those IPs are appended to `ALLOWED_HOSTS` unless the list is `*`. +- Example manifests under `docs/docs/logstashui/kubernetes/examples/{sqlite,postgresql,mysql}/`. +- Optional embedded simulation agent overlay: `docs/docs/logstashui/kubernetes/examples/embedded-agent.yaml` (compose `--profile embedded` analog). ClusterIP 9500 / 9560 / 9449. Uncomment `LOGSTASH_AGENT_URL` and `LOGSTASHUI_AGENT_CSR_SECRET` on the UI examples before apply. Agent ConfigMap comments `LOGSTASH_AGENT_TLS` (default true) and `LOGSTASH_UI_TLS_INSECURE` (default false). +- Database subsection: engines, every `LOGSTASHUI_DB_*` default, offline dump/load, BETA `migrate-engine`, CREATE DATABASE scripts (`utf8mb4_bin` for MySQL/MariaDB), schema snapshots from 0.5.2 `migrate`. + +### Insecure HTTP (escape hatch) + +- `LOGSTASHUI_INSECURE_HTTP=true` forces plain HTTP for the UI and every UI→agent URL, skips product CA and certificate generation, and overrides `LOGSTASHUI_TLS`. This is **not** best practice; automatic TLS remains the supported default. It is **not** the same as `LOGSTASHUI_TLS=false` (TLS-terminating ingress). See [Environment](docs/docs/logstashui/configuration/environment.md). +- Settings TLS **Upload** and **Revert** are hidden in this mode and return **409** before any filesystem change. Leftover `$DATA_DIR/tls/` custom leaves are left intact (`save_custom_ui_certificate` / `revert_ui_certificate_to_product_default` raise `ProductCADisabled` first). +- Default Compose and Kubernetes examples remain HTTPS. The hatch is native/`logstashui serve` (or a hand-edited EnvironmentFile). StatefulSet examples comment that probes stay `scheme: HTTPS`. + +### Fixes + +- `LOGSTASHUI_OTEL=true` without the `[otel]` extra now logs **ERROR** (was INFO) and continues. Docker/K8s and freeze artifacts install `[otel]`; uncomment the OTEL keys in the Kubernetes ConfigMap examples to enable OTLP/HTTP (port 4318, not gRPC 4317). +- `LOGSTASHUI_TLS=false` now suppresses the Django-level HTTP→HTTPS redirect (`SECURE_SSL_REDIRECT`) in addition to disabling the Gunicorn TLS certificate. Previously, running the container with `-e LOGSTASHUI_TLS=false` still returned a `301` because `SECURE_SSL_REDIRECT` was gated on `DEBUG` only. Both knobs are now independent. +- `ApiKey.save()` no longer re-hashes an already-hashed key. `make_password()` ran unconditionally on every save, so any update to an existing row silently rewrote the hash and invalidated the credential. Latent until now — nothing re-saved an `ApiKey` — but renaming or revoking a token does. +- The pipeline editor's simulation **Target** picker no longer hides the embedded agent on first page load. Making the probe non-blocking left `list_simulation_targets()` requiring a successful probe it never performed, so the sticky embedded row lost a race against the background thread and the dropdown rendered empty until a later refresh. The row is now dropped only when a probe has *explicitly* reported the agent offline — never-probed is treated as unknown, not offline. Target rows also carry a `discovered` flag so callers can distinguish a confirmed agent from an unconfirmed one. +- Connection Manager – Cloud ID field bleed-through: Fixed a bug where switching from Cloud ID to URL connection type with a Cloud ID already entered would attempt to use the Cloud ID value to establish the connection instead of the URL. +- SNMP – Graceful "no data yet" error: Resolved an unhelpful error message shown to users who had successfully created a device in SNMP but for whom no data had yet been written to Elasticsearch. The experience is now clean and friendly. +- SNMP – Cancel icon color in Safari: Fixed a cross-browser rendering issue where cancel/delete icons (which should be pink) were rendering as green in Safari. +- Windows – Logging file handler: Updated the logging file handler configuration so that it works correctly on Windows (file rotation and path handling were broken on Windows previously). +- Pre-commit – Windows line endings: Fixed the add_license_headers.py script to prevent CRLF line endings from being injected when adding the NOTICE header to source files on Windows. Also retroactively corrected headers added to files that were previously missing them. + +# UI +- Modal close confirmation: Added a confirmation dialogue to modal close handlers for modals with significant user input (e.g. Connection, SNMP Device, Network, Profile modals). This prevents accidental data loss when a user clicks outside the modal. +- SNMP Device – IPv6 support: Relaxed the IP address validation on SNMP devices to accept IPv6 addresses in addition to IPv4. + +### API access + +- **Admin API tokens** (`Management → API Tokens`) let scripts, CI, and provisioning tools call LogstashUI's JSON endpoints without a browser session. Send `Authorization: ApiKey lsui__`. The original motivation was registering a remote Elasticsearch cluster for Centralized Pipeline Management from `curl`, which previously failed CSRF verification; it now works against the existing `/ConnectionManager/AddConnection` URL. Every other JSON endpoint accepts a token too — no per-endpoint opt-in. +- A token acts as the user who created it and inherits that account's role, so audit lines stay attributable and a `readonly` user's token stays readonly. Tokens can carry an optional expiry and be revoked or deleted; revocation takes effect on the next request. Only a hash is stored, so the secret is displayed exactly once at creation. +- Implemented as a pair of middlewares rather than per-view decorators. `ApiTokenCsrfMiddleware` runs immediately before `CsrfViewMiddleware` and sets `_dont_enforce_csrf_checks` **only after a token verifies** — a forged or absent header cannot switch CSRF off, and cookie-authenticated browser requests are unaffected. `ApiTokenUserMiddleware` runs just after `AuthenticationMiddleware`, which would otherwise overwrite `request.user`; the split is forced by that ordering. Because `request.user` becomes a real user, `LoginRequiredMiddleware` and `require_admin_role` pass on their own — no `@csrf_exempt` and no `LOGIN_REQUIRED_IGNORE_PATHS` entries were added. +- `require_admin_role` now answers API-token callers with JSON instead of an `HX-Trigger` toast, which is unreadable to a script. +- Admin tokens reuse the agent `ApiKey` table and its `make_password`/`check_password` machinery. The one addition is an unhashed, indexed `prefix` column: agent keys are found via the `connection_id` in the request body before their hash is checked, but a token presents only a header, and without a lookup key resolving it would mean a PBKDF2 comparison against every row. Existing agent keys get `prefix=NULL` and are never matched by the middleware. +- See [API Access](docs/docs/logstashui/api_access.md). + +### Agent version display + +- ConnectionManager and Policy editor Agents tab show a cyan **LS X.Y.Z** pill for the Logstash version the agent reported (`logstash_version_resolved`, else Logstash API version). Hidden until known. +- Policy Source **VERSION** live-fills and persists Binary Path as `{download_dir}/logstash-{version}/bin` (default `/opt/logstash-agent/logstash-versions/logstash-X.Y.Z/bin`). Custom paths are kept. Switching back to SYSTEM restores `/usr/share/logstash/bin` when the field still looks derived. +- Agent newer than preferred (`__PREFERRED_LS_AGENT_VERSION__`, currently 0.5.2) shows **unreleased version** instead of a backwards Upgrade button. Older agents still get Upgrade. Unparseable versions still get Upgrade. +- **The LS pill now tracks the Logstash version actually running on the host.** It was frozen at whatever version happened to be recorded first, for two reasons. `resolve_running_logstash_version()` consulted the `Connection.logstash_version_resolved` column *before* the current check-in's `status_blob`, and that column is only ever written on a truthy value and never cleared — so one stored version permanently shadowed every later one. The check-in handler also never read `status_blob.logstash_api.version`, the version the running instance reports through its own API, so on hosts that report it only there the column was never refreshed at all. The blob now leads and the column is the fallback, which keeps the last known version on screen while Logstash is stopped or its API is unreachable. +- The pill also updates without a page reload. The Connections page adds `logstash_version` to the existing agent-status SSE payload — free, since the stream already selects `status_blob` — and the Policies → Agents table, which had no live channel at all, polls every 10s while that tab is visible and pauses when the browser tab is hidden. + +### Logstash tarball proxy + +- **LogstashUI can now cache Logstash release tarballs and serve them to agents.** A `MANAGED` or `SIMULATE` policy pinned to **VERSION** previously made every agent pull its own ~450 MB tarball from `artifacts.elastic.co`. Tick **Download the tarball from LogstashUI** on the policy and LogstashUI fetches each release once, verifies its SHA-512, and serves it to every agent. Required for air-gapped sites, where the direct download is impossible. See [Logstash Tarball Proxy](docs/docs/logstashui/configuration/logstash_proxy.md). +- **Compatibility — the agent artifact URL gained a `connection_id` segment.** `GET /ConnectionManager/LogstashArtifact/{filename}` is now `GET /ConnectionManager/LogstashArtifact/{connection_id}/{filename}`. A GET has no body, and an agent key is a bare hash with no lookup column, so the header alone cannot identify the caller — the path is what narrows the lookup to one row before `check_password` runs. Agents older than the paired release cannot use the proxy; the boolean simply stays off for them and they continue downloading from Elastic. +- The proxy is exposed to agents in three places: `logstash_via_ui` in the enrollment `policy_config`, `logstash_via_ui` in the check-in response, and `logstash_runtime.via_ui` in the config delta. `via_ui` participates in the `runtime_changed` comparison, so flipping the checkbox alone triggers a re-materialize — no separate Deploy for binary-only changes. +- **Management → Logstash Tarballs** manages the cache: download by version + architecture (or from a full URL for an internal mirror), delete, retry, and **Import from disk** for air-gapped operators who copy a tarball into the cache directory by hand. Import hashes the file, checks a supplied `.sha512` and fails on mismatch, and writes one when none was supplied. There is no browser upload — 450 MB through a form is not viable. Progress polls over htmx only while a row is actively fetching. +- **Single-flight is a conditional database `UPDATE`, not an in-process lock.** `settings.py` defines no `CACHES`, so Django falls back to per-process `LocMemCache`, which does not coordinate across gunicorn workers. The first agent to ask claims the row and gets a `503`; everyone else gets a `503` until the file lands, across all workers. A fetch killed by a restart leaves a stale claim that the next request reclaims after the heartbeat window, and the orphaned `.part` is swept at startup — crash recovery is the normal path, not an edge case, because a 450 MB fetch will always be killed by `graceful_timeout`. +- Nothing partial is ever served: downloads land in a `.part` file and are `os.replace()`d into place only after the SHA-512 verifies. Agents get `200`/`206` when the file is ready, `503` while it is being fetched, `429` at the concurrent-serve cap, `502` on an upstream failure, `404` for an unrecognized filename, and `401` for a bad key or mismatched `connection_id`. `503`/`429`/`502` all carry `Retry-After`; agents honour it, backing off to a 5-minute ceiling, and never fall back to `artifacts.elastic.co` while the proxy is enabled. +- Range requests are supported (single range only) so an interrupted transfer resumes instead of re-pulling the whole tarball. Files under 1 MiB — the `.sha512` companions — are exempt from the serve semaphore, so a burst of checksum fetches cannot consume download slots. +- New env vars: `LOGSTASHUI_LOGSTASH_DIR` (default `/logstashes`), `LOGSTASHUI_ARTIFACT_MAX_UPSTREAM` (default `2`, cluster-wide), and `LOGSTASHUI_ARTIFACT_MAX_SERVE_PER_WORKER` (default `4`; effective total is this × `LOGSTASHUI_WORKERS`). The serve cap is deliberately per-worker — dividing a global limit by the worker count truncates to zero and makes the knob lie. The upstream base URL is **Management → Settings → Logstash tarball source**, blank meaning `https://artifacts.elastic.co/downloads/logstash`. +- Tarballs are stored in `/logstashes`, **not** under `staticfiles/` — `STATIC_ROOT` is served by WhiteNoise at `/static/`, which is in `LOGIN_REQUIRED_IGNORE_PATHS`, so every tarball would have been an unauthenticated public download, and `collectstatic` would churn over them on every `serve`. +- **Optional OpenTelemetry export** (`LOGSTASHUI_OTEL=true` plus the new `LogstashUI[otel]` extra) sends traces and metrics over OTLP/HTTP. The Docker/K8s image and freeze artifacts install `[otel]` (the wheelhouse already downloaded those wheels); tracing stays off until the env flag is set. Native pip/uv still needs the extra. If `LOGSTASHUI_OTEL=true` and the extra is missing, LogstashUI logs ERROR and the worker keeps serving. Four custom instruments answer the capacity question Django spans cannot: `logstashui.gevent.hub.lag`, `logstashui.artifact.downloads.active`, `logstashui.artifact.requests`, and `logstashui.artifact.serve.bytes_per_second`. Only the HTTP/protobuf exporter is supported — the gRPC exporter's native threads are not gevent-patchable. +- Migrations: `PipelineManager/0029_logstash_artifacts` (the `LogstashArtifact` model and `Policy.logstash_via_ui`) and `Management/0004_settings_logstash_artifact_base_url`. +- New tests: `tests/PipelineManager/unit/test_logstash_artifacts.py` and `tests/Management/unit/test_logstash_artifact_page.py`. + + ## [0.5.1] - Agent control plane, SNMP NMS, dual HTTPS - 08/31/2026 Package version is **0.5.1** (`pyproject.toml`). Preferred LogstashAgent version is **0.5.1**. diff --git a/NOTICE.txt b/NOTICE.txt index 7d0a1692..4813e21d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2398,3 +2398,193 @@ Redistribution and use in source and binary forms, with or without modification, This software is provided by the copyright holders and contributors “as is” and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profits; or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. + +-------------- psycopg -------------- + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + + +-------------- PyMySQL -------------- +Copyright (c) 2010, 2013 PyMySQL contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/README.md b/README.md index e27f1408..28fe6a67 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ > > ⚠️ **Beta Release** - This project is under active development. Features may change. > -> **Current package version: 0.5.1** — see [CHANGELOG.md](CHANGELOG.md). Pair with **LogstashAgent 0.5.1**. Operator guide: [agent roles / ports / coexistence / VERSION](docs/docs/logstashagent/general/roles.md). +> **Current package version: 0.5.2** — see [CHANGELOG.md](CHANGELOG.md). Pair with **LogstashAgent 0.5.2**. Operator guide: [agent roles / ports / coexistence / VERSION](docs/docs/logstashagent/general/roles.md). diff --git a/bin/freeze_logstashui.sh b/bin/freeze_logstashui.sh new file mode 100755 index 00000000..8409afdc --- /dev/null +++ b/bin/freeze_logstashui.sh @@ -0,0 +1,282 @@ +#!/bin/bash +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Optional air-gapped freeze. Default `uv build` is unchanged. +# Usage: +# ./bin/freeze_logstashui.sh [--wheels] [--docker] [--standalone] [--all] +# [--output DIR] [--image NAME] +# No artifact flags → --all. + +set -euo pipefail + +ROOT=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) +TEMPLATES="$ROOT/packaging/offline" +DO_WHEELS=0 +DO_DOCKER=0 +DO_STANDALONE=0 +EXPLICIT_STANDALONE=0 +OUT="" +IMAGE_OVERRIDE="" + +usage() { + cat <<'EOF' +Usage: freeze_logstashui.sh [--wheels] [--docker] [--standalone] [--all] + [--output DIR] [--image NAME] + +Connected-builder freeze for air-gapped hosts. Default uv build is unchanged. + + --wheels CPython 3.12 manylinux x86_64 wheelhouse zip + --docker docker save of a local image (never docker pull) + --standalone experimental PyInstaller onedir (Linux x86_64 only) + --all all three (default if no artifact flags) + --output DIR default: /dist/offline + --image NAME docker save this local tag instead of building + +Isolated wheels host: CPython 3.12 x86_64 + python3.12-venv. No uv. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --wheels) DO_WHEELS=1 ;; + --docker) DO_DOCKER=1 ;; + --standalone) DO_STANDALONE=1; EXPLICIT_STANDALONE=1 ;; + --all) DO_WHEELS=1; DO_DOCKER=1; DO_STANDALONE=1 ;; + --output) + OUT="${2:?--output requires a directory}" + shift + ;; + --image) + IMAGE_OVERRIDE="${2:?--image requires a name}" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac + shift +done + +if [[ $DO_WHEELS -eq 0 && $DO_DOCKER -eq 0 && $DO_STANDALONE -eq 0 ]]; then + DO_WHEELS=1 + DO_DOCKER=1 + DO_STANDALONE=1 +fi + +OUT="${OUT:-$ROOT/dist/offline}" +mkdir -p "$OUT" + +die() { echo "ERROR: $*" >&2; exit 1; } + +version_from_pyproject() { + sed -n 's/^version = "\([^"]*\)"/\1/p' "$ROOT/pyproject.toml" | head -n 1 +} + +subst() { + # $1 src $2 dest + local git_sha="$GIT_SHA" + local version="$VERSION" + local image="$IMAGE_NAME" + sed \ + -e "s|__VERSION__|${version}|g" \ + -e "s|__GIT_SHA__|${git_sha}|g" \ + -e "s|__IMAGE_NAME__|${image}|g" \ + "$1" > "$2" +} + +sha256_tree() { + local dir="$1" + local out="$2" + ( + cd "$dir" + if command -v sha256sum >/dev/null 2>&1; then + find . -type f ! -name SHA256SUMS.txt -print0 | sort -z | xargs -0 sha256sum + else + find . -type f ! -name SHA256SUMS.txt -print0 | sort -z | xargs -0 shasum -a 256 + fi + ) > "$out" +} + +zip_dir() { + local src="$1" + local dest="$2" + rm -f "$dest" + ( + cd "$(dirname "$src")" + zip -r -q "$dest" "$(basename "$src")" + ) +} + +require_uv() { + command -v uv >/dev/null 2>&1 || die "uv is required. Install: https://docs.astral.sh/uv/getting-started/installation/" + uv python find 3.12 >/dev/null 2>&1 || die "Need CPython 3.12 (uv python install 3.12)" + command -v zip >/dev/null 2>&1 || die "zip is required to pack freeze artifacts" +} + +ensure_tailwind() { + local css="$ROOT/src/logstashui/theme/static/css/dist/styles.css" + if [[ -s "$css" ]]; then + return 0 + fi + command -v npm >/dev/null 2>&1 || die "Tailwind CSS missing and npm is not installed" + (cd "$ROOT/src/logstashui/theme/static_src" && npm install && npm run build) + [[ -s "$css" ]] || die "Tailwind CSS build did not produce $css" +} + +linux_x86_64() { + [[ "$(uname -s)" == Linux && "$(uname -m)" == x86_64 ]] +} + +freeze_wheels() { + local stage="$OUT/logstashui-${VERSION}-offline-wheels-linux-x86_64-cp312" + local wheels="$stage/wheels" + local req="$OUT/requirements-offline.txt" + local zip="$OUT/logstashui-${VERSION}-offline-wheels-linux-x86_64-cp312.zip" + + rm -rf "$stage" + mkdir -p "$wheels" + + echo "==> uv build (normal wheel)" + (cd "$ROOT" && uv build) + + local whl + whl=$(ls -1 "$ROOT/dist"/logstashui-"${VERSION}"-*.whl 2>/dev/null | head -n 1 || true) + [[ -n "$whl" && -f "$whl" ]] || die "uv build did not produce dist/logstashui-${VERSION}-*.whl" + cp "$whl" "$wheels/" + + # otel rides along so an air-gapped site can still enable tracing with + # LOGSTASHUI_OTEL=true. It is inert until then. + echo "==> uv export --frozen --extra databases --extra otel" + (cd "$ROOT" && uv export --frozen --no-dev --extra databases --extra otel --no-emit-project \ + -o "$req" >/dev/null) + local req_plain="$OUT/requirements-offline.nohash.txt" + (cd "$ROOT" && uv export --frozen --no-dev --extra databases --extra otel --no-emit-project --no-hashes \ + -o "$req_plain" >/dev/null) + + echo "==> download manylinux cp312 wheels (pure-python sdists → wheel on builder)" + (cd "$ROOT" && uv run --python 3.12 --with pip python \ + "$TEMPLATES/download_wheels.py" "$req_plain" "$wheels") + + subst "$TEMPLATES/wheels-install.sh" "$stage/install.sh" + subst "$TEMPLATES/wheels-README.md" "$stage/README.md" + chmod +x "$stage/install.sh" + cp "$ROOT/LICENSE.txt" "$stage/LICENSE.txt" + cp "$ROOT/NOTICE.txt" "$stage/NOTICE.txt" + cp "$req" "$stage/requirements-offline.txt" + + { + echo "LogstashUI ${VERSION}" + echo "git ${GIT_SHA}" + echo "python CPython 3.12" + echo "platform linux-x86_64" + echo "extras databases otel" + echo + echo "wheels:" + (cd "$wheels" && ls -1 *.whl | sort) + } > "$stage/MANIFEST.txt" + + sha256_tree "$stage" "$stage/SHA256SUMS.txt" + zip_dir "$stage" "$zip" + echo "Wrote $zip" +} + +freeze_docker() { + local tag="${IMAGE_OVERRIDE:-logstashui:offline-${VERSION}}" + IMAGE_NAME="$tag" + local stage="$OUT/logstashui-${VERSION}-offline-docker-linux-x86_64" + local zip="$OUT/logstashui-${VERSION}-offline-docker-linux-x86_64.zip" + + command -v docker >/dev/null 2>&1 || die "docker is required for --docker" + + if [[ -n "$IMAGE_OVERRIDE" ]]; then + docker image inspect "$IMAGE_OVERRIDE" >/dev/null 2>&1 \ + || die "image ${IMAGE_OVERRIDE} is not local (never docker pull). Build it or omit --image." + else + echo "==> docker build --platform linux/amd64 ${tag}" + docker build --platform linux/amd64 -f "$ROOT/docker/Dockerfile" -t "$tag" "$ROOT" + fi + + rm -rf "$stage" + mkdir -p "$stage" + echo "==> docker save ${tag}" + docker save "$tag" | gzip -c > "$stage/image.tar.gz" + + subst "$TEMPLATES/docker-load.sh" "$stage/load.sh" + subst "$TEMPLATES/docker-README.md" "$stage/README.md" + subst "$TEMPLATES/compose.offline.yml" "$stage/compose.offline.yml" + chmod +x "$stage/load.sh" + cp "$ROOT/LICENSE.txt" "$stage/LICENSE.txt" + cp "$ROOT/NOTICE.txt" "$stage/NOTICE.txt" + sha256_tree "$stage" "$stage/SHA256SUMS.txt" + zip_dir "$stage" "$zip" + echo "Wrote $zip" +} + +freeze_standalone() { + if ! linux_x86_64; then + if [[ "$EXPLICIT_STANDALONE" -eq 1 ]]; then + die "standalone freeze requires Linux x86_64 (PyInstaller binary is per-OS)" + fi + echo "WARNING: skipping --standalone (not Linux x86_64)" >&2 + return 0 + fi + + local venv="$OUT/.standalone-venv" + local work="$OUT/pyinstaller-work" + local dist="$OUT/pyinstaller-dist" + local stage="$OUT/logstashui-${VERSION}-offline-standalone-linux-x86_64" + local zip="$OUT/logstashui-${VERSION}-offline-standalone-linux-x86_64.zip" + echo "==> throwaway venv + PyInstaller (not a project dependency)" + rm -rf "$venv" "$work" "$dist" + uv venv --python 3.12 "$venv" + (cd "$ROOT" && uv pip install --python "$venv" ".[databases,otel]") + uv pip install --python "$venv" pyinstaller + "$venv/bin/pyinstaller" \ + --noconfirm \ + --clean \ + --workpath "$work" \ + --distpath "$dist" \ + "$TEMPLATES/logstashui.spec" + + [[ -x "$dist/logstashui/logstashui" ]] || die "PyInstaller did not produce $dist/logstashui/logstashui" + + rm -rf "$stage" + mkdir -p "$stage" + cp -a "$dist/logstashui" "$stage/logstashui" + subst "$TEMPLATES/standalone-run.sh" "$stage/run.sh" + subst "$TEMPLATES/standalone-README.md" "$stage/README.md" + chmod +x "$stage/run.sh" + cp "$ROOT/LICENSE.txt" "$stage/LICENSE.txt" + cp "$ROOT/NOTICE.txt" "$stage/NOTICE.txt" + sha256_tree "$stage" "$stage/SHA256SUMS.txt" + zip_dir "$stage" "$zip" + echo "Wrote $zip" +} + +VERSION=$(version_from_pyproject) +[[ -n "$VERSION" ]] || die "could not read version from pyproject.toml" +GIT_SHA=$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown) +IMAGE_NAME="${IMAGE_OVERRIDE:-logstashui:offline-${VERSION}}" + +require_uv +ensure_tailwind + +echo "LogstashUI ${VERSION} git ${GIT_SHA} output ${OUT}" + +if [[ $DO_WHEELS -eq 1 ]]; then + freeze_wheels +fi +if [[ $DO_DOCKER -eq 1 ]]; then + freeze_docker +fi +if [[ $DO_STANDALONE -eq 1 ]]; then + freeze_standalone +fi diff --git a/bin/test_databases.bat b/bin/test_databases.bat new file mode 100644 index 00000000..4e4281e6 --- /dev/null +++ b/bin/test_databases.bat @@ -0,0 +1,49 @@ +@echo off +setlocal +cd /d "%~dp0\.." +set FAIL=0 +where docker >nul 2>&1 +if errorlevel 1 ( + echo ERROR: Docker is required for bin\test_databases.bat + exit /b 1 +) +uv sync --extra databases --group dev +uv run pytest src\logstashui --no-cov +if errorlevel 1 exit /b 1 +docker compose -f docker\docker-compose.db.yml up -d --wait +if errorlevel 1 exit /b 1 + +set LOGSTASHUI_DB_ENGINE=postgresql +set LOGSTASHUI_DB_HOST=127.0.0.1 +set LOGSTASHUI_DB_PORT=55432 +set LOGSTASHUI_DB_NAME=logstashui +set LOGSTASHUI_DB_USER=logstashui +set LOGSTASHUI_DB_PASSWORD=logstashui +uv run pytest src\logstashui --no-cov +if errorlevel 1 ( + set FAIL=1 + goto :down +) + +set LOGSTASHUI_DB_ENGINE=mysql +set LOGSTASHUI_DB_PORT=53306 +set LOGSTASHUI_DB_USER=root +uv run pytest src\logstashui --no-cov +if errorlevel 1 ( + set FAIL=1 + goto :down +) + +set LOGSTASHUI_DB_PORT=53307 +uv run pytest src\logstashui --no-cov +if errorlevel 1 ( + set FAIL=1 + goto :down +) + +uv run pytest tests\Database\ -v --no-cov +if errorlevel 1 set FAIL=1 + +:down +if /I not "%1"=="--keep" docker compose -f docker\docker-compose.db.yml down -v +if "%FAIL%"=="1" exit /b 1 diff --git a/bin/test_databases.sh b/bin/test_databases.sh new file mode 100755 index 00000000..9dd98750 --- /dev/null +++ b/bin/test_databases.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +COMPOSE=(docker compose -f docker/docker-compose.db.yml) +KEEP=0 +if [[ "${1:-}" == "--keep" ]]; then KEEP=1; fi +if ! command -v docker >/dev/null; then + echo "ERROR: Docker is required for bin/test_databases.sh" >&2 + echo "Default pytest (SQLite) still works: uv run pytest" >&2 + exit 1 +fi +uv sync --extra databases --group dev +echo "==> SQLite pytest (no compose)" +uv run pytest src/logstashui --no-cov +echo "==> Starting Postgres / MariaDB / MySQL" +"${COMPOSE[@]}" up -d --wait +run_engine () { + local name="$1"; shift + echo "==> pytest on ${name}" + env "$@" uv run pytest src/logstashui --no-cov +} +run_engine postgresql LOGSTASHUI_DB_ENGINE=postgresql LOGSTASHUI_DB_HOST=127.0.0.1 LOGSTASHUI_DB_PORT=55432 LOGSTASHUI_DB_NAME=logstashui LOGSTASHUI_DB_USER=logstashui LOGSTASHUI_DB_PASSWORD=logstashui +run_engine mariadb LOGSTASHUI_DB_ENGINE=mysql LOGSTASHUI_DB_HOST=127.0.0.1 LOGSTASHUI_DB_PORT=53306 LOGSTASHUI_DB_NAME=logstashui LOGSTASHUI_DB_USER=root LOGSTASHUI_DB_PASSWORD=logstashui +run_engine mysql LOGSTASHUI_DB_ENGINE=mysql LOGSTASHUI_DB_HOST=127.0.0.1 LOGSTASHUI_DB_PORT=53307 LOGSTASHUI_DB_NAME=logstashui LOGSTASHUI_DB_USER=root LOGSTASHUI_DB_PASSWORD=logstashui +echo "==> Database test suite (testcontainers)" +uv run pytest tests/Database/ -v --no-cov +if [[ "$KEEP" -eq 0 ]]; then "${COMPOSE[@]}" down -v; fi diff --git a/bin/test_docker_otel.sh b/bin/test_docker_otel.sh new file mode 100755 index 00000000..50ddc72f --- /dev/null +++ b/bin/test_docker_otel.sh @@ -0,0 +1,36 @@ +#!/bin/bash +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Smoke that a pre-built LogstashUI image includes LogstashUI[otel]. +# Does not build the image (slow). Not a default PR check. +# +# IMAGE=logstashui:0.5.2-dev bin/test_docker_otel.sh +# bin/test_docker_otel.sh logstashui:0.5.2-dev + +set -euo pipefail + +IMAGE="${1:-${IMAGE:-}}" +if [[ -z "$IMAGE" ]]; then + echo "ERROR: pass an image tag or set IMAGE. Build first:" >&2 + echo " docker build -f docker/Dockerfile -t logstashui:0.5.2-dev ." >&2 + echo " # or: bin/start_logstashui.sh --rebuild" >&2 + exit 1 +fi + +command -v docker >/dev/null 2>&1 || { + echo "ERROR: docker required" >&2 + exit 1 +} + +docker image inspect "$IMAGE" >/dev/null 2>&1 || { + echo "ERROR: image ${IMAGE} is not local. Build it; this script never docker pull." >&2 + exit 1 +} + +echo "==> ${IMAGE}: import opentelemetry.sdk + instrumentation.django" +docker run --rm --entrypoint python "$IMAGE" -c \ + "import opentelemetry.sdk; import opentelemetry.instrumentation.django; print('otel extra ok')" + +echo "Docker OTEL smoke passed: $IMAGE" diff --git a/bin/test_freeze_wheels.sh b/bin/test_freeze_wheels.sh new file mode 100755 index 00000000..b5727707 --- /dev/null +++ b/bin/test_freeze_wheels.sh @@ -0,0 +1,59 @@ +#!/bin/bash +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Smoke the air-gapped wheelhouse: unzip, pip --no-index in a network-none +# linux/amd64 CPython 3.12 container. Pull python:3.12-slim first (needs net). + +set -euo pipefail + +ROOT=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) +OUT="${1:-$ROOT/dist/offline}" + +ZIP=$(ls -1 "$OUT"/logstashui-*-offline-wheels-linux-x86_64-cp312.zip 2>/dev/null | tail -n 1 || true) +[[ -n "$ZIP" && -f "$ZIP" ]] || { + echo "ERROR: no wheels zip in $OUT. Run: bin/freeze_logstashui.sh --wheels" >&2 + exit 1 +} + +WORKDIR=$(mktemp -d) +trap 'rm -rf "$WORKDIR"' EXIT +unzip -q "$ZIP" -d "$WORKDIR" +STAGE=$(find "$WORKDIR" -maxdepth 1 -type d -name 'logstashui-*-offline-wheels-*' | head -n 1) +[[ -n "$STAGE" ]] || STAGE=$WORKDIR + +shopt -s nullglob +sdists=("$STAGE"/wheels/*.tar.gz) +if (( ${#sdists[@]} )); then + echo "ERROR: sdist in wheelhouse" >&2 + printf '%s\n' "${sdists[@]}" + exit 1 +fi + +command -v docker >/dev/null 2>&1 || { + echo "ERROR: docker required for network-none install smoke" >&2 + exit 1 +} + +echo "==> docker pull python:3.12-slim (linux/amd64)" +docker pull --platform linux/amd64 python:3.12-slim + +echo "==> install.sh + logstashui --help / manage check (--network=none)" +docker run --rm \ + --platform linux/amd64 \ + --network=none \ + -v "$STAGE:/offline:ro" \ + -w /tmp \ + python:3.12-slim \ + bash -lc ' + set -euo pipefail + cp -a /offline /tmp/pkg + cd /tmp/pkg + PYTHON=python3 sh install.sh + .venv/bin/logstashui --help + .venv/bin/logstashui manage check + .venv/bin/python -c "import opentelemetry.sdk; import opentelemetry.instrumentation.django" + ' + +echo "Wheels freeze smoke passed: $ZIP" diff --git a/docker/Dockerfile b/docker/Dockerfile index 4ac81fa3..523b60e3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,7 @@ WORKDIR /app/src/logstashui/theme/static_src RUN rm -rf node_modules && npm install && npm run build && rm -rf node_modules WORKDIR /app -RUN uv pip install --system --no-cache /app +RUN uv pip install --system --no-cache "/app[databases,otel]" # Pinned uid so K8s runAsUser/fsGroup is stable. Entrypoint starts as root, # chowns only LOGSTASHUI_DATA_DIR when needed, then drops to this user (or PUID). diff --git a/docker/db-init/mysql-extra.sql b/docker/db-init/mysql-extra.sql new file mode 100644 index 00000000..a27fc626 --- /dev/null +++ b/docker/db-init/mysql-extra.sql @@ -0,0 +1,3 @@ +CREATE DATABASE IF NOT EXISTS logstashui_migrate CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; +GRANT ALL ON logstashui_migrate.* TO 'logstashui'@'%'; +GRANT ALL ON logstashui_migrate.* TO 'root'@'%'; diff --git a/docker/db-init/postgres-extra.sql b/docker/db-init/postgres-extra.sql new file mode 100644 index 00000000..fe18b44e --- /dev/null +++ b/docker/db-init/postgres-extra.sql @@ -0,0 +1 @@ +CREATE DATABASE logstashui_migrate OWNER logstashui; diff --git a/docker/docker-compose.db.yml b/docker/docker-compose.db.yml new file mode 100644 index 00000000..20c5c787 --- /dev/null +++ b/docker/docker-compose.db.yml @@ -0,0 +1,60 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Local/CI database matrix for LogstashUI. Not used by smoke compose. +# Host ports avoid clashing with a developer’s own 5432/3306. +name: logstashui-db-test + +services: + postgres: + image: postgres:16 + environment: + POSTGRES_USER: logstashui + POSTGRES_PASSWORD: logstashui + POSTGRES_DB: logstashui + ports: + - "55432:5432" + volumes: + - ./db-init/postgres-extra.sql:/docker-entrypoint-initdb.d/02-migrate.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U logstashui -d logstashui"] + interval: 2s + timeout: 5s + retries: 30 + + mariadb: + image: mariadb:11 + environment: + MARIADB_ROOT_PASSWORD: logstashui + MARIADB_DATABASE: logstashui + MARIADB_USER: logstashui + MARIADB_PASSWORD: logstashui + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin + ports: + - "53306:3306" + volumes: + - ./db-init/mysql-extra.sql:/docker-entrypoint-initdb.d/02-migrate.sql:ro + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 3s + timeout: 5s + retries: 40 + + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: logstashui + MYSQL_DATABASE: logstashui + MYSQL_USER: logstashui + MYSQL_PASSWORD: logstashui + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin + ports: + - "53307:3306" + volumes: + - ./db-init/mysql-extra.sql:/docker-entrypoint-initdb.d/02-migrate.sql:ro + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ulogstashui", "-plogstashui"] + interval: 3s + timeout: 5s + retries: 40 diff --git a/docs/docs/index.md b/docs/docs/index.md index 3bc46a67..50991e86 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -63,6 +63,17 @@ New to LogstashUI? Start here: - **[Simulation Modes](/docs/docs/logstashui/configuration/simulation.md)** - Embedded vs Host mode - **[Host Mode Setup](/docs/docs/logstashui/configuration/host_mode.md)** - High-performance simulation setup +#### Database +- **[Database](/docs/docs/logstashui/database/index.md)** - SQLite default, PostgreSQL, MySQL/MariaDB + - **[Migration](/docs/docs/logstashui/database/migration.md)** - Offline dump/load and BETA `migrate-engine` + - **[SQL examples](/docs/docs/logstashui/database/examples/README.md)** - CREATE DATABASE and schema snapshots + +#### Kubernetes +- **[Kubernetes](/docs/docs/logstashui/kubernetes/index.md)** - StatefulSet, PVC, TLS on :8443 + - **[Envoy Gateway](/docs/docs/logstashui/kubernetes/envoy-gateway.md)** - Enable Backend API, skip backend cert verify + - **[CloudNativePG](/docs/docs/logstashui/kubernetes/cnpg.md)** - Cluster in the app namespace + - **[Example manifests](/docs/docs/logstashui/kubernetes/examples/README.md)** - SQLite, PostgreSQL, MySQL/MariaDB + #### SNMP Monitoring - **[SNMP Overview](/docs/docs/logstashui/SNMP/index.md)** - Network monitoring introduction - **[Quickstart Guide](/docs/docs/logstashui/SNMP/Quickstart.md)** - From zero to metrics diff --git a/docs/docs/logstashagent/general/roles.md b/docs/docs/logstashagent/general/roles.md index 19e79354..32b3c96e 100644 --- a/docs/docs/logstashagent/general/roles.md +++ b/docs/docs/logstashagent/general/roles.md @@ -2,7 +2,7 @@ LogstashAgent and LogstashUI use **policy types** and matching **agent modes** so one Linux host can run production Logstash and one or more isolated multi-instance agents without sharing state or config. -> **Paired releases:** LogstashUI **0.5.1** ↔ LogstashAgent **0.5.1** (see [Compatibility](/docs/docs/logstashui/compatibility.md) and [CHANGELOG](https://github.com/elastic/LogstashUI/blob/main/CHANGELOG.md)). +> **Paired releases:** LogstashUI **0.5.2** ↔ LogstashAgent **0.5.2** (see [Compatibility](/docs/docs/logstashui/compatibility.md) and [CHANGELOG](https://github.com/elastic/LogstashUI/blob/main/CHANGELOG.md)). --- @@ -140,7 +140,9 @@ Policies can set **Logstash binary source**: 1. Save the policy in LogstashUI (Source = VERSION, pin e.g. `9.4.3`). 2. **No separate Deploy is required for binary-only changes** — the next agent check-in detects runtime drift. -3. Agent downloads (if needed) into `/opt/logstash-agent/logstash-versions//`. +3. Agent downloads (if needed) into `/opt/logstash-agent/logstash-versions//` — from Elastic + artifacts, or from LogstashUI when the policy enables + [the tarball proxy](/docs/docs/logstashui/configuration/logstash_proxy.md). 4. Writes `LOGSTASH_BINARY=` into the instance `env` file. 5. Restarts the Logstash unit when the binary path or pin changes. 6. Reports resolved version on check-in (`status_blob.logstash_version_resolved`). diff --git a/docs/docs/logstashui/api_access.md b/docs/docs/logstashui/api_access.md new file mode 100644 index 00000000..55666686 --- /dev/null +++ b/docs/docs/logstashui/api_access.md @@ -0,0 +1,106 @@ +# API Access + +LogstashUI's JSON endpoints are normally reached from a logged-in browser session. An **API +token** lets a script, CI job, or provisioning tool call the same endpoints with `curl`. + +## Creating a token + +**Management → API Tokens → Create Token.** + +Give the token a name, optionally set an expiry in days, and copy the value. It looks like: + +```plain +lsui_bfed5132382f_kmrqP9ETXxF0yKz1s--1SPrHl93yllxhq_kj2Hpz3i8 +``` + +Only a hash is stored, so **the token is shown exactly once**. If you lose it, revoke it and mint +a new one. + +A token acts as the user who created it and inherits that account's role. A token created by a +`readonly` user can read but not write. Revoking takes effect on the next request. + +## Making requests + +Send the token in an `Authorization` header: + +```bash +curl -k -X POST \ + -H "Authorization: ApiKey lsui__" \ + -d "connection_type=CENTRALIZED&name=my-cluster&host=https://es.example.com&port=443&api_key=" \ + https://localhost:8443/ConnectionManager/AddConnection +``` + +Token requests are exempt from CSRF, so no cookie jar or `X-CSRFToken` header is needed. CSRF +remains fully enforced for ordinary browser sessions — the exemption applies only after a token +has been verified. + +Every existing JSON endpoint accepts a token, not just the one below. + +## Adding a remote Elasticsearch cluster for Centralized Pipeline Management + +`POST /ConnectionManager/AddConnection` (note: **no trailing slash**), form-encoded. + +| Field | Required | Notes | +| --- | --- | --- | +| `connection_type` | yes | `CENTRALIZED` | +| `name` | yes | Display name, must be unique | +| `host` | one of | Full URL, e.g. `https://es.example.com` | +| `cloud_id` | one of | Elastic Cloud ID, as an alternative to `host` | +| `port` | no | Usually `9200`, or `443` for Cloud | +| `api_key` | no | Elasticsearch API key | +| `username` / `password` | no | Basic auth, as an alternative to `api_key` | +| `cloud_url` | no | Full cluster URL | + +Either `host` or `cloud_id` must be present. Supply either `api_key` or `username`+`password`; +missing credentials are not rejected by validation but will fail the connectivity test. + +On success: + +```json +{"success": true, "connection_id": 7, "message": "Connection created and tested successfully!"} +``` + +LogstashUI pings the cluster before saving. If the ping fails, the connection is rolled back and +the error is returned. + +## Two things that will bite you + +**Check the body, not the status code.** `AddConnection` returns **HTTP 200** even when the +request fails, with the reason in the body: + +```json +{"success": false, "error": "Connection error caused by: ..."} +``` + +So test `.success`: + +```bash +resp=$(curl -sk -X POST -H "Authorization: ApiKey $TOKEN" -d "$BODY" \ + https://localhost:8443/ConnectionManager/AddConnection) +echo "$resp" | grep -q '"success": true' || { echo "failed: $resp" >&2; exit 1; } +``` + +**`-k` is required against the built-in certificate.** LogstashUI serves TLS from its own product +CA, which curl does not trust by default. Either pass `-k`, or fetch the CA and trust it properly: + +```bash +curl -sk https://localhost:8443/.well-known/logstashui/ca.crt -o logstashui-ca.crt +curl --cacert logstashui-ca.crt -X POST -H "Authorization: ApiKey $TOKEN" ... +``` + +## Error responses + +| Status | Meaning | +| --- | --- | +| `401` | Token missing, malformed, unknown, revoked, expired, or owned by a disabled user | +| `403` | Token is valid but its owner lacks the required role | +| `405` | Wrong HTTP method | +| `200` with `"success": false` | Request reached the view and was rejected there | + +## Notes + +- Agent enrollment keys are a separate credential and are unaffected. They carry no `lsui_` + marker and continue to authenticate as before. +- Verifying a token costs one PBKDF2 comparison. That is deliberate, and fine at scripting rates, + but do not put a token request in a tight loop. +- Never commit a token. Treat it as equivalent to the password of the account that created it. diff --git a/docs/docs/logstashui/compatibility.md b/docs/docs/logstashui/compatibility.md index 910ad9f1..942890fe 100644 --- a/docs/docs/logstashui/compatibility.md +++ b/docs/docs/logstashui/compatibility.md @@ -27,6 +27,7 @@ Logstash Agent is versioned alongside LogstashUI. When updating LogstashUI, upda | LogstashUI | Preferred LogstashAgent | |------------|-------------------------| +| 0.5.2 | 0.5.2 (lockstep with LogstashUI 0.5.2) | | 0.5.1 | 0.5.1 (Packaged/Managed/Simulate, multi-instance, install registry, VERSION, dual HTTPS, NMS) | | 0.5.0 | 0.5.0 | diff --git a/docs/docs/logstashui/configuration/environment.md b/docs/docs/logstashui/configuration/environment.md index 7c9a18bc..46158fad 100644 --- a/docs/docs/logstashui/configuration/environment.md +++ b/docs/docs/logstashui/configuration/environment.md @@ -35,11 +35,53 @@ Relative values resolve from the process working directory. | `LOGSTASHUI_BIND` | `0.0.0.0:8443` | gunicorn bind | | `LOGSTASHUI_WORKERS` | `2` | gunicorn workers | | `LOGSTASHUI_TLS` | `true` | HTTPS with product CA under `$LOGSTASHUI_DATA_DIR/tls/` | +| `LOGSTASHUI_INSECURE_HTTP` | `false` | **Not recommended.** Plain HTTP for the UI and every agent connection. Overrides `LOGSTASHUI_TLS`. No product CA. See below. | | `ALLOWED_HOSTS` | `*` | Django allowed hosts | | `CSRF_TRUSTED_ORIGINS` | (dev localhost defaults) | comma-separated origins | | `SECRET_KEY` | auto in data dir | Django secret | -Set `LOGSTASHUI_TLS=false` when an ingress terminates TLS and the pod should speak HTTP. +Keep `LOGSTASHUI_TLS=true` (the default) in Kubernetes. Ingress/HTTPRoute should originate HTTPS to `:8443` and skip backend cert verify. See [Kubernetes](/docs/docs/logstashui/kubernetes/index.md). + +### Plain HTTP (not recommended) + +This is **not best practice.** Do not use it in production. + +LogstashUI already provisions TLS automatically: a product CA under `$LOGSTASHUI_DATA_DIR/tls/`, a UI leaf for gunicorn, and signed agent certificates at enroll/check-in. That path is supported and works out of the box. + +If you **absolutely must** run the UI and all agent connections over plain HTTP (no certificates at all), set: + +```bash +export LOGSTASHUI_INSECURE_HTTP=true +``` + +Effects: + +- gunicorn serves HTTP (no `--certfile`). Default bind is still `:8443` — the port is not a protocol. +- `LOGSTASHUI_TLS` is overridden (default or explicit `true`). Startup logs a WARNING and continues. +- Product CA and UI certificates are **not** generated or refreshed. Leftover files in `$DATA_DIR/tls/` are left on disk. +- The UI rewrites every agent URL and every `--logstash-ui-url` / callback it emits from `https://` to `http://`. +- Enrollment tokens omit the CA fingerprint. A CSR in enroll/check-in is ignored; the request still succeeds. +- `GET /.well-known/logstashui/ca.crt` returns 404. + +LogstashAgent needs a matching TLS-off flag on its side. That flag is not configured here. + +**This is not `LOGSTASHUI_TLS=false`.** That setting is only for a TLS-terminating ingress: gunicorn speaks HTTP, agents still use HTTPS, and the product CA is still issued. Kubernetes should keep `LOGSTASHUI_TLS=true` and skip-verify at the Gateway/Ingress. See [Kubernetes](/docs/docs/logstashui/kubernetes/index.md). + +**Standard Compose and Kubernetes examples stay HTTPS.** Setting `LOGSTASHUI_INSECURE_HTTP` in the host shell does not enter the UI container. Injecting it without changing the Compose healthcheck (today `https://127.0.0.1:8443/.well-known/logstashui/ca.crt`) leaves the service unhealthy — that endpoint is 404 in this mode. Kubernetes example probes use `scheme: HTTPS`; uncommenting the ConfigMap key without changing probe (and Ingress/HTTPRoute backend) scheme fails the pod. Use this hatch with native `logstashui serve` or a hand-edited systemd EnvironmentFile. + +--- + +## Observability + +| Variable | Default | Purpose | +|---|---|---| +| `LOGSTASHUI_OTEL` | `false` | Enable OTLP/HTTP traces + metrics. Native: install `LogstashUI[otel]`. | +| `OTEL_SERVICE_NAME` | `logstashui` | Resource `service.name` | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | SDK default | Collector base URL (HTTP/protobuf) | + +The Docker/K8s image and freeze artifacts already install `[otel]`. Set `LOGSTASHUI_OTEL=true` to turn tracing on. Native pip/uv still needs `pip install 'LogstashUI[otel]'` (or `uv pip install 'LogstashUI[otel]'`). If the extra is missing, LogstashUI logs **ERROR** and continues without tracing. + +Only the HTTP/protobuf OTLP exporter is supported. The gRPC exporter is incompatible with the gevent worker (native threads cannot be monkey-patched). Use collector port **4318**, not 4317. --- @@ -50,22 +92,66 @@ Set `LOGSTASHUI_TLS=false` when an ingress terminates TLS and the pod should spe | `LOGSTASHUI_NO_AUTH` | `false` | Bypass login (**sandbox only**) | | `LOGSTASHUI_AGENT_UI_URL` | empty | Prefill `--logstash-ui-url` (DB Settings wins if set) | | `LOGSTASHUI_INCLUDE_CA_FINGERPRINT` | `true` | Embed product CA fingerprint in enrollment tokens | -| `LOGSTASH_AGENT_URL` | debug: `http://127.0.0.1:9500`; else `https://logstashagent:9500` | Embedded/compose agent API | -| `LOGSTASHUI_HOST_HOSTNAME` / `LOGSTASHUI_HOST_IPS` / `LOGSTASHUI_TLS_SANS` | empty | Extra SANs on the product UI cert | -| `LOGSTASHUI_AGENT_CSR_SECRET` | empty | Compose/embedded agent CSR without enroll | +| `LOGSTASH_AGENT_URL` | debug: `http://127.0.0.1:9500`; else `https://logstashagent:9500` | Embedded/compose agent API. Kubernetes examples comment this; apply [embedded-agent.yaml](/docs/docs/logstashui/kubernetes/examples/embedded-agent.yaml) after uncommenting. | +| `LOGSTASHUI_HOST_HOSTNAME` / `LOGSTASHUI_HOST_IPS` / `LOGSTASHUI_TLS_SANS` | empty | Extra SANs on the product UI cert. Kubernetes: set `LOGSTASHUI_HOST_IPS` from `status.podIP` (Downward API). IPs are also appended to `ALLOWED_HOSTS` unless that list is `*`. | +| `LOGSTASHUI_AGENT_CSR_SECRET` | empty | Compose/embedded agent CSR without enroll. Kubernetes examples comment this on Secret `logstashui`; the overlay `secretKeyRef` requires it uncommented. | | `LOGSTASHUI_DOCS_DIR` | checkout `docs/` or packaged copy | In-app documentation root | +Agent-side (not UI ConfigMap): `LOGSTASH_AGENT_TLS` (default `true`) and `LOGSTASH_UI_TLS_INSECURE` (default `false`) are LogstashAgent env vars. The Kubernetes overlay comments both on ConfigMap `logstashagent`. See the LogstashAgent README TLS table. Do not enable without cause. + Booleans accept `true`/`false`, `1`/`0`, `yes`/`no`, `on`/`off`. --- -## Database (sqlite only) +## Database | Variable | Default | Purpose | |---|---|---| -| `LOGSTASHUI_DB_ENGINE` | `sqlite` | **Only `sqlite` is implemented** | +| `LOGSTASHUI_DB_ENGINE` | `sqlite` | `sqlite`, `postgresql`, or `mysql` (MariaDB uses `mysql`). Aliases: `sqlite3`, `postgres`, `mariadb`, `my` | +| `LOGSTASHUI_DB_NAME` | sqlite: `$LOGSTASHUI_DATA_DIR/db.sqlite3`; else `logstashui` | Database name / sqlite path | +| `LOGSTASHUI_DB_HOST` | empty | **Required** for postgresql/mysql | +| `LOGSTASHUI_DB_PORT` | `5432` / `3306` | | +| `LOGSTASHUI_DB_USER` | empty | **Required** for postgresql/mysql | +| `LOGSTASHUI_DB_PASSWORD` | empty | Put in a Secret / `chmod 640` EnvironmentFile | +| `LOGSTASHUI_DB_SSLMODE` | postgres: `prefer` | `disable` `allow` `prefer` `require` `verify-ca` `verify-full` | +| `LOGSTASHUI_DB_SSL_CA` | empty | CA file for mysql TLS and postgres `verify-*` | +| `LOGSTASHUI_DB_CONN_MAX_AGE` | `60` | Persistent connections (seconds); `0` closes per request | +| `LOGSTASHUI_DB_CONN_HEALTH_CHECKS` | `true` | Django `CONN_HEALTH_CHECKS` | + +Floors: PostgreSQL 14+, MariaDB 10.6+, MySQL 8.0+. Create MySQL/MariaDB as `utf8mb4` / `utf8mb4_bin` so unique names match SQLite/Postgres case-sensitivity. Full engine docs, env defaults, and SQL examples: [Database](/docs/docs/logstashui/database/index.md). Migration (offline + BETA CLI): [Migration](/docs/docs/logstashui/database/migration.md). + +**Install extras (native pip/uv):** `uv pip install 'LogstashUI[postgres]'`, `'LogstashUI[mysql]'`, or `'LogstashUI[databases]'`. The Docker/K8s image already installs `[databases]` and `[otel]`. Missing driver fails at startup with that extra name. Tracing stays off until `LOGSTASHUI_OTEL=true`. + +`LOGSTASHUI_DATA_DIR` is still required when the database is remote (TLS, `.django_secret_key`, logs, staticfiles). + +**SQLite scale:** `logstashui serve` logs a warning when engine is sqlite and `LOGSTASHUI_WORKERS` > 1. Use PostgreSQL or MySQL/MariaDB for concurrent agents. Startup still succeeds. + +**Connections:** gunicorn remains gevent (`--worker-connections 1000`). Keep `LOGSTASHUI_WORKERS` × in-flight requests under the server `max_connections`. PgBouncer (or equivalent) is optional, not required. + +No `DATABASE_URL`. No YAML. + +### Offline migration (supported) + +1. `systemctl stop logstashui` (or stop the container). +2. Copy `$LOGSTASHUI_DATA_DIR/db.sqlite3` somewhere safe. Keep the rest of `DATA_DIR` (same Django secret key). +3. Dump **from sqlite** while `LOGSTASHUI_DB_ENGINE` is still sqlite (or unset): `logstashui manage dumpdata --natural-foreign --natural-primary -e contenttypes -e auth.permission -e sessions -o dump.json`. Do not set target `LOGSTASHUI_DB_*` yet, or dumpdata will dump the empty server database. +4. Create the server database (`utf8mb4_bin` on MySQL/MariaDB). +5. **Then** set `LOGSTASHUI_DB_*` for the target. Native installs need the matching extra. +6. `logstashui manage migrate --noinput && logstashui manage loaddata dump.json` +7. Postgres sequences: the BETA CLI (`logstashui migrate-engine`) resets them through Django (no `psql`). The dump/load path above does not; use `migrate-engine` when you need sequence reset without a Postgres client. +8. Start LogstashUI. Log in again (sessions were not copied). + +### BETA CLI + +```bash +# env already points at the empty target server; sqlite file still in DATA_DIR +sudo systemctl stop logstashui # avoid Restart= racing SIGTERM +logstashui migrate-engine --to postgresql --i-have-a-backup +# optional: --write-env /etc/default/logstashui +sudo systemctl start logstashui +``` -`postgresql` and `mysql` are reserved names: setting them **fails at startup** until those backends land. Keep a PVC on `LOGSTASHUI_DATA_DIR` even after that work (TLS material and secrets still live there). +`--to mysql` covers MariaDB and MySQL. The command SIGTERMs gunicorn if `$LOGSTASHUI_DATA_DIR/gunicorn.pid` is live, checkpoints WAL, dump/load from the sqlite file in `DATA_DIR` (regardless of target `LOGSTASHUI_DB_*`), and **does not** restart serve. --- @@ -73,12 +159,12 @@ Booleans accept `true`/`false`, `1`/`0`, `yes`/`no`, `on`/`off`. Minimum: -1. Deployment env from a ConfigMap + Secret -2. PVC mounted at `/var/lib/logstashui` +1. StatefulSet `replicas: 1`, env from a ConfigMap (`LOGSTASHUI_DB_ENGINE` / `HOST` / `NAME` / `USER`) + Secret (`SECRET_KEY`, `LOGSTASHUI_DB_PASSWORD`) +2. PVC mounted at `/var/lib/logstashui` (still required when the database is external) 3. Container image `CMD` is `logstashui serve` (already the Docker default) -4. Optional ingress: `LOGSTASHUI_TLS=false` and `CSRF_TRUSTED_ORIGINS=https://` +4. Ingress or HTTPRoute: keep `LOGSTASHUI_TLS=true`, originate HTTPS to `:8443`, skip backend cert verify, set `CSRF_TRUSTED_ORIGINS=https://` -No ConfigMap file mount is required. +No ConfigMap file mount is required. Manifests and CloudNativePG: [Kubernetes](/docs/docs/logstashui/kubernetes/index.md). --- diff --git a/docs/docs/logstashui/configuration/host_mode.md b/docs/docs/logstashui/configuration/host_mode.md index 1a08e23c..1d6eb7ef 100644 --- a/docs/docs/logstashui/configuration/host_mode.md +++ b/docs/docs/logstashui/configuration/host_mode.md @@ -22,7 +22,9 @@ - Root for `logstash-agent install --enroll …` (or non-root enroll + `sudo logstash-agent setup-simulate`) - Logstash binary available either: - **SYSTEM** — package or tarball already on the host, or - - **VERSION** — agent downloads from Elastic artifacts into `/opt/logstash-agent/logstash-versions/` + - **VERSION** — agent downloads into `/opt/logstash-agent/logstash-versions/`, from Elastic + artifacts or, with [the tarball proxy](/docs/docs/logstashui/configuration/logstash_proxy.md) + enabled, from LogstashUI - Reachable LogstashUI URL from the agent host ## Install a simulate agent diff --git a/docs/docs/logstashui/configuration/index.md b/docs/docs/logstashui/configuration/index.md index 24720c57..772add71 100644 --- a/docs/docs/logstashui/configuration/index.md +++ b/docs/docs/logstashui/configuration/index.md @@ -12,8 +12,17 @@ Data dir, TLS, bind address, `LOGSTASHUI_NO_AUTH`, agent URL, systemd, and Kuber **📖 [View environment configuration →](/docs/docs/logstashui/configuration/environment.md)** +Database engines and `LOGSTASHUI_DB_*`: **[Database](/docs/docs/logstashui/database/index.md)**. Kubernetes manifests: **[Kubernetes](/docs/docs/logstashui/kubernetes/index.md)**. + `logstashui.yml` is [removed](/docs/docs/logstashui/configuration/logstashui.yml.md). +### **[Logstash tarball proxy](/docs/docs/logstashui/configuration/logstash_proxy.md)** + +Cache each Logstash release once and serve it to every agent, instead of each agent pulling ~450 MB +from Elastic. Required for air-gapped sites. + +**📖 [View the tarball proxy guide →](/docs/docs/logstashui/configuration/logstash_proxy.md)** + --- ## Simulation Configuration diff --git a/docs/docs/logstashui/configuration/logstash_proxy.md b/docs/docs/logstashui/configuration/logstash_proxy.md new file mode 100644 index 00000000..a5bf1849 --- /dev/null +++ b/docs/docs/logstashui/configuration/logstash_proxy.md @@ -0,0 +1,138 @@ +# Logstash Tarball Proxy + +A `MANAGED` or `SIMULATE` policy pinned to **VERSION** makes every agent download its own copy of +the Logstash release tarball — roughly 450 MB each. At two agents that is wasteful; at fifty it is a +bandwidth event, and in an air-gapped site it is impossible. + +With the proxy enabled, LogstashUI fetches each release **once**, verifies its SHA-512, and serves it +to every agent that needs it. + +## Enabling it + +1. **Cache the tarball.** Management → **Logstash Tarballs** → *Download Tarball*. Pick a version and + architecture, or paste a full URL for an internal mirror. +2. **Point the policy at LogstashUI.** In the policy editor, set the Logstash source to **VERSION**, + enter the version, and tick **Download the tarball from LogstashUI**. + +The checkbox only appears for **Managed** and **Simulate** policies using **VERSION**. Packaged +policies use the OS package and Embedded runs Logstash in-process, so neither ever downloads a +tarball. + +Agents pick up the change on their next check-in. No separate Deploy is needed for binary-only +changes. + +## Managing the cache + +**Management → Logstash Tarballs** lists everything cached, with live progress while a download is +running. + +| Action | What it does | +| --- | --- | +| Download Tarball | Fetch by version + architecture, or from a full URL | +| Import from disk | Register tarballs copied into the cache directory by hand | +| Delete | Remove the tarball, its checksum, and the row | +| Retry download | Re-attempt a failed fetch | + +Tarballs live in `LOGSTASHUI_LOGSTASH_DIR`, default `/logstashes`. There is no upload +action — 450 MB through a browser is not viable. + +**Served** counts full tarball downloads. An agent fetches the `.sha512` sidecar as well, and may +resume an interrupted transfer with a range request; neither adds to the count, so the number tracks +how many agents actually pulled the release. + +### Air-gapped sites + +Copy the tarball (and its `.sha512`, if you have it) into the cache directory, then click **Import +from disk**. LogstashUI hashes the file and publishes it. If you supply a `.sha512` it is checked and +a mismatch fails the import; if you do not, LogstashUI writes one from what it computed so the +agent's own verification step still has something to check. + +## Configuration + +The upstream source is **Management → Settings → Logstash tarball source**. Blank means +`https://artifacts.elastic.co/downloads/logstash`. Point it at an internal mirror to keep fetches +inside your network. + +Everything else is environment, because the concurrency limits interact with `LOGSTASHUI_WORKERS` +and take effect at startup: + +| Variable | Default | Notes | +| --- | --- | --- | +| `LOGSTASHUI_LOGSTASH_DIR` | `/logstashes` | Cache root | +| `LOGSTASHUI_ARTIFACT_MAX_UPSTREAM` | `2` | Concurrent fetches from upstream, across all workers | +| `LOGSTASHUI_ARTIFACT_MAX_SERVE_PER_WORKER` | `4` | Concurrent agent downloads **per worker**; effective total is this × `LOGSTASHUI_WORKERS` | + +The serve limit is a fairness knob, not a throughput knob. Capping does not reduce the total bytes +moved — it decides whether twenty agents each crawl at a twentieth of your bandwidth, or eight finish +quickly and twelve retry shortly after. For a rollout, the second is better. + +## What agents see + +`GET /ConnectionManager/LogstashArtifact//` with +`Authorization: ApiKey `, over the product CA. + +The `connection_id` is in the path because a GET has no body to carry it, and an agent key is a bare +hash with no lookup column — the header alone cannot identify which agent is calling. + +| Status | Meaning | Agent behaviour | +| --- | --- | --- | +| `200` | The file, with `Content-Length` and `Accept-Ranges: bytes` | Verify the SHA-512 as usual | +| `206` | Partial content, if the agent sent a `Range` header | Append and continue | +| `503` | Not cached yet; a fetch has started | Sleep for `Retry-After`, retry | +| `429` | At the concurrent-serve cap | Sleep for `Retry-After`, retry | +| `502` | The upstream fetch failed | Retry, and check the Logstash Tarballs page | +| `404` | Filename not recognized | Fail the deployment | +| `401` | Bad key or wrong `connection_id` | Re-enroll | + +Agents honour `Retry-After`, backing off exponentially to a 5-minute ceiling. When the proxy is +enabled they **never** fall back to `artifacts.elastic.co` — silently reaching the internet would +defeat both the bandwidth saving and air-gapped operation. + +Range requests let an agent resume an interrupted transfer instead of re-pulling 450 MB, which is +exactly the load the cache exists to prevent. + +## How it behaves under load + +- **One download per release, ever.** The first agent to ask triggers the fetch and gets a 503; + everyone else gets a 503 until it lands. This holds across gunicorn workers — the claim is a + conditional database update, not an in-process lock. +- **Nothing partial is ever served.** Downloads land in a `.part` file and are moved into place only + after the SHA-512 verifies. +- **Restarts are safe.** A download killed by a restart leaves a stale claim, which the next request + reclaims; the orphaned `.part` is swept at startup. +- **Check-ins do not starve.** Transfers are chunked so they yield frequently. With the default caps + the practical ceiling is your network, not the server. + +## Measuring it + +Set `LOGSTASHUI_OTEL=true` to export traces and metrics over OTLP/HTTP to +`OTEL_EXPORTER_OTLP_ENDPOINT`. The Docker/K8s image and freeze artifacts already +install `LogstashUI[otel]`. Native pip/uv: `pip install 'LogstashUI[otel]'`. +If the extra is missing, LogstashUI logs ERROR and continues without tracing. + +Four instruments answer the capacity question that request traces cannot: + +| Instrument | What it tells you | +| --- | --- | +| `logstashui.gevent.hub.lag` | The important one. Flat under load means you are network-bound and more workers will not help; rising means requests are starving and more workers or cores will | +| `logstashui.artifact.downloads.active` | If this never reaches the cap, the cap is not your constraint | +| `logstashui.artifact.requests` | The 429/503 rate — the direct "raise the cap" signal | +| `logstashui.artifact.serve.bytes_per_second` | Per-stream throughput. Falling while the aggregate stays flat means you are at the network ceiling | + +Only the HTTP/protobuf OTLP exporter is supported. The gRPC exporter is incompatible with the gevent +worker LogstashUI runs under. + +## Troubleshooting + +**Agents keep getting 503.** Check the Logstash Tarballs page. A row stuck on *Downloading* with no +progress usually means the upstream is unreachable; a *Failed* row shows the reason. + +**"SHA-512 mismatch".** The upstream published a checksum that does not match what was downloaded. +Nothing is published, so agents are unaffected. Retry — this is usually a truncated transfer. + +**Agents still hit artifacts.elastic.co.** The checkbox only takes effect for **Managed**/**Simulate** +policies with source **VERSION**. Confirm the policy saved, then confirm the agent has checked in +since. + +**A version disappeared.** Deleting a tarball a policy still pins sends those agents into a retry +loop until it is cached again. The table marks those rows **in use**. diff --git a/docs/docs/logstashui/configuration/simulation.md b/docs/docs/logstashui/configuration/simulation.md index 89f251c0..a82359b0 100644 --- a/docs/docs/logstashui/configuration/simulation.md +++ b/docs/docs/logstashui/configuration/simulation.md @@ -41,7 +41,7 @@ You can run **default** and **simulate** agents on the same host; simulate never #### Multi-version testing -Clone the Simulate Policy, set `logstash_source=VERSION` and a version (e.g. `9.4.3`). The agent downloads that release into `/opt/logstash-agent/logstash-versions/` when materializing. Pick the instance in the editor to compare pipeline behavior across releases. +Clone the Simulate Policy, set `logstash_source=VERSION` and a version (e.g. `9.4.3`). The agent downloads that release into `/opt/logstash-agent/logstash-versions/` when materializing — from Elastic artifacts, or from LogstashUI if the policy also enables [the tarball proxy](/docs/docs/logstashui/configuration/logstash_proxy.md). Pick the instance in the editor to compare pipeline behavior across releases. #### Keystore variables diff --git a/docs/docs/logstashui/database/examples/README.md b/docs/docs/logstashui/database/examples/README.md new file mode 100644 index 00000000..9211a8a3 --- /dev/null +++ b/docs/docs/logstashui/database/examples/README.md @@ -0,0 +1,15 @@ +# Database examples + +Apply a **create-*.sql** script to an empty server, then start LogstashUI (`migrate` creates tables). + +| File | Use | +|---|---| +| [create-postgresql.sql](create-postgresql.sql) | Role + database (UTF8) | +| [create-mysql.sql](create-mysql.sql) | MySQL 8.0+ `utf8mb4` / `utf8mb4_bin` | +| [create-mariadb.sql](create-mariadb.sql) | MariaDB 10.6+ same collation | +| [schema-postgresql.sql](schema-postgresql.sql) | Snapshot of `migrate` DDL (reference) | +| [schema-mysql.sql](schema-mysql.sql) | Snapshot of `migrate` DDL (reference) | + +`schema-*.sql` is generated from LogstashUI **0.5.2** migrations. It **will go stale**. Do not apply it instead of `logstashui manage migrate --noinput`. It exists so you can see tables, indexes, and the MySQL collation before you run migrate. + +CloudNativePG: bootstrap `database` / `owner` already creates the empty database — skip `create-postgresql.sql`, still run migrate. diff --git a/docs/docs/logstashui/database/examples/create-mariadb.sql b/docs/docs/logstashui/database/examples/create-mariadb.sql new file mode 100644 index 00000000..2685eff4 --- /dev/null +++ b/docs/docs/logstashui/database/examples/create-mariadb.sql @@ -0,0 +1,13 @@ +-- Empty MariaDB 10.6+ database for LogstashUI. +-- Engine is still LOGSTASHUI_DB_ENGINE=mysql. +-- utf8mb4_bin keeps unique names case-sensitive like SQLite/Postgres. +-- Tables are created by: logstashui manage migrate --noinput +-- Replace the password before running. + +CREATE DATABASE IF NOT EXISTS logstashui + CHARACTER SET utf8mb4 + COLLATE utf8mb4_bin; + +CREATE USER IF NOT EXISTS 'logstashui'@'%' IDENTIFIED BY 'change-me'; +GRANT ALL PRIVILEGES ON logstashui.* TO 'logstashui'@'%'; +FLUSH PRIVILEGES; diff --git a/docs/docs/logstashui/database/examples/create-mysql.sql b/docs/docs/logstashui/database/examples/create-mysql.sql new file mode 100644 index 00000000..fff304ae --- /dev/null +++ b/docs/docs/logstashui/database/examples/create-mysql.sql @@ -0,0 +1,12 @@ +-- Empty MySQL 8.0+ database for LogstashUI. +-- utf8mb4_bin keeps unique names case-sensitive like SQLite/Postgres. +-- Tables are created by: logstashui manage migrate --noinput +-- Replace the password before running. + +CREATE DATABASE IF NOT EXISTS logstashui + CHARACTER SET utf8mb4 + COLLATE utf8mb4_bin; + +CREATE USER IF NOT EXISTS 'logstashui'@'%' IDENTIFIED BY 'change-me'; +GRANT ALL PRIVILEGES ON logstashui.* TO 'logstashui'@'%'; +FLUSH PRIVILEGES; diff --git a/docs/docs/logstashui/database/examples/create-postgresql.sql b/docs/docs/logstashui/database/examples/create-postgresql.sql new file mode 100644 index 00000000..b95f4e57 --- /dev/null +++ b/docs/docs/logstashui/database/examples/create-postgresql.sql @@ -0,0 +1,13 @@ +-- Empty PostgreSQL 14+ database for LogstashUI. +-- Tables are created by: logstashui manage migrate --noinput +-- Replace the password before running. + +CREATE USER logstashui WITH LOGIN PASSWORD 'change-me'; +CREATE DATABASE logstashui + OWNER logstashui + ENCODING 'UTF8' + TEMPLATE template0; + +\connect logstashui +GRANT ALL ON SCHEMA public TO logstashui; +ALTER DATABASE logstashui OWNER TO logstashui; diff --git a/docs/docs/logstashui/database/examples/schema-mysql.sql b/docs/docs/logstashui/database/examples/schema-mysql.sql new file mode 100644 index 00000000..833ea675 --- /dev/null +++ b/docs/docs/logstashui/database/examples/schema-mysql.sql @@ -0,0 +1,502 @@ +-- Snapshot of LogstashUI 0.5.2 `migrate` DDL on MySQL 8.0 (utf8mb4_bin). +-- Generated with mysqldump --no-data. Do NOT apply this instead of +-- `logstashui manage migrate --noinput`. It will go stale with new migrations. +-- Create the empty database first (see create-mysql.sql / create-mariadb.sql). + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!50503 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; +DROP TABLE IF EXISTS `Management_userprofile`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `Management_userprofile` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `role` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `user_id` int NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `user_id` (`user_id`), + CONSTRAINT `Management_userprofile_user_id_70f1a900_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `PipelineManager_apikey`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `PipelineManager_apikey` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `api_key` varchar(512) COLLATE utf8mb4_bin NOT NULL, + `connection_id` bigint NOT NULL, + PRIMARY KEY (`id`), + KEY `PipelineManager_apik_connection_id_27156847_fk_PipelineM` (`connection_id`), + CONSTRAINT `PipelineManager_apik_connection_id_27156847_fk_PipelineM` FOREIGN KEY (`connection_id`) REFERENCES `PipelineManager_connection` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `PipelineManager_connection`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `PipelineManager_connection` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `connection_type` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `host` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + `port` int unsigned DEFAULT NULL, + `username` varchar(100) COLLATE utf8mb4_bin DEFAULT NULL, + `password` varchar(512) COLLATE utf8mb4_bin DEFAULT NULL, + `ssh_key` longtext COLLATE utf8mb4_bin, + `cloud_id` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + `cloud_url` varchar(200) COLLATE utf8mb4_bin DEFAULT NULL, + `api_key` varchar(512) COLLATE utf8mb4_bin DEFAULT NULL, + `created_at` datetime(6) NOT NULL, + `updated_at` datetime(6) NOT NULL, + `is_active` tinyint(1) NOT NULL, + `policy_id` bigint DEFAULT NULL, + `agent_id` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + `last_check_in` datetime(6) DEFAULT NULL, + `status_blob` json DEFAULT NULL, + `restart_on_next_checkin` tinyint(1) NOT NULL, + `desired_agent_version` varchar(20) COLLATE utf8mb4_bin DEFAULT NULL, + `agent_api_port` int unsigned DEFAULT NULL, + `instance_id` int unsigned DEFAULT NULL, + `last_selected_at` datetime(6) DEFAULT NULL, + `logstash_api_port` int unsigned DEFAULT NULL, + `logstash_version_resolved` varchar(64) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `agent_id` (`agent_id`), + KEY `PipelineManager_conn_policy_id_819d316d_fk_PipelineM` (`policy_id`), + CONSTRAINT `PipelineManager_conn_policy_id_819d316d_fk_PipelineM` FOREIGN KEY (`policy_id`) REFERENCES `PipelineManager_policy` (`id`), + CONSTRAINT `PipelineManager_connection_chk_1` CHECK ((`port` >= 0)), + CONSTRAINT `PipelineManager_connection_chk_2` CHECK ((`agent_api_port` >= 0)), + CONSTRAINT `PipelineManager_connection_chk_3` CHECK ((`instance_id` >= 0)), + CONSTRAINT `PipelineManager_connection_chk_4` CHECK ((`logstash_api_port` >= 0)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `PipelineManager_enrollmenttoken`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `PipelineManager_enrollmenttoken` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `token` varchar(512) COLLATE utf8mb4_bin NOT NULL, + `policy_id` bigint NOT NULL, + `name` varchar(100) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `unique_token_name_per_policy` (`policy_id`,`name`), + CONSTRAINT `PipelineManager_enro_policy_id_355d6dca_fk_PipelineM` FOREIGN KEY (`policy_id`) REFERENCES `PipelineManager_policy` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `PipelineManager_keystore`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `PipelineManager_keystore` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `key_name` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `key_value` varchar(512) COLLATE utf8mb4_bin NOT NULL, + `last_updated` datetime(6) NOT NULL, + `policy_id` bigint NOT NULL, + `revision_number` int NOT NULL, + `kv_hash` varchar(64) COLLATE utf8mb4_bin NOT NULL, + `managed_by` varchar(20) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `unique_key_per_policy` (`policy_id`,`key_name`), + CONSTRAINT `PipelineManager_keys_policy_id_6309b699_fk_PipelineM` FOREIGN KEY (`policy_id`) REFERENCES `PipelineManager_policy` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `PipelineManager_pipeline`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `PipelineManager_pipeline` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `description` longtext COLLATE utf8mb4_bin, + `lscl` longtext COLLATE utf8mb4_bin NOT NULL, + `lscl_hash` varchar(64) COLLATE utf8mb4_bin DEFAULT NULL, + `last_updated` datetime(6) NOT NULL, + `policy_id` bigint NOT NULL, + `revision_number` int NOT NULL, + `pipeline_batch_delay` int NOT NULL, + `pipeline_batch_size` int NOT NULL, + `pipeline_workers` int NOT NULL, + `queue_checkpoint_writes` int NOT NULL, + `queue_max_bytes` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `queue_type` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `pipeline_hash` varchar(64) COLLATE utf8mb4_bin NOT NULL, + `no_input` tinyint(1) NOT NULL, + `non_reloadable` tinyint(1) NOT NULL, + `managed_by` varchar(20) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `unique_pipeline_per_policy` (`policy_id`,`name`), + CONSTRAINT `PipelineManager_pipe_policy_id_7c3a6fd9_fk_PipelineM` FOREIGN KEY (`policy_id`) REFERENCES `PipelineManager_policy` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `PipelineManager_policy`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `PipelineManager_policy` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `settings_path` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `logs_path` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `logstash_yml` longtext COLLATE utf8mb4_bin NOT NULL, + `jvm_options` longtext COLLATE utf8mb4_bin NOT NULL, + `log4j2_properties` longtext COLLATE utf8mb4_bin NOT NULL, + `created_at` datetime(6) NOT NULL, + `updated_at` datetime(6) NOT NULL, + `current_revision_number` int NOT NULL, + `has_undeployed_changes` tinyint(1) NOT NULL, + `jvm_options_hash` varchar(64) COLLATE utf8mb4_bin NOT NULL, + `log4j2_properties_hash` varchar(64) COLLATE utf8mb4_bin NOT NULL, + `logstash_yml_hash` varchar(64) COLLATE utf8mb4_bin NOT NULL, + `binary_path` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `keystore_password` varchar(512) COLLATE utf8mb4_bin DEFAULT NULL, + `keystore_password_hash` varchar(64) COLLATE utf8mb4_bin NOT NULL, + `last_deployed_at` datetime(6) DEFAULT NULL, + `agent_api_port` int unsigned NOT NULL, + `cloned_from_id` bigint DEFAULT NULL, + `data_path` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `is_system` tinyint(1) NOT NULL, + `keystore_env_file` varchar(512) COLLATE utf8mb4_bin NOT NULL, + `logstash_api_port` int unsigned NOT NULL, + `logstash_download_dir` varchar(512) COLLATE utf8mb4_bin NOT NULL, + `logstash_source` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `logstash_version` varchar(32) COLLATE utf8mb4_bin NOT NULL, + `policy_type` varchar(20) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`), + KEY `PipelineManager_poli_cloned_from_id_4177e74a_fk_PipelineM` (`cloned_from_id`), + CONSTRAINT `PipelineManager_poli_cloned_from_id_4177e74a_fk_PipelineM` FOREIGN KEY (`cloned_from_id`) REFERENCES `PipelineManager_policy` (`id`), + CONSTRAINT `PipelineManager_policy_chk_1` CHECK ((`agent_api_port` >= 0)), + CONSTRAINT `PipelineManager_policy_chk_2` CHECK ((`logstash_api_port` >= 0)) +) ENGINE=InnoDB AUTO_INCREMENT=5 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `PipelineManager_revision`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `PipelineManager_revision` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `revision_number` int NOT NULL, + `snapshot_json` json NOT NULL, + `created_at` datetime(6) NOT NULL, + `created_by` varchar(150) COLLATE utf8mb4_bin NOT NULL, + `policy_id` bigint NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `unique_revision_per_policy` (`policy_id`,`revision_number`), + CONSTRAINT `PipelineManager_revi_policy_id_99e7d146_fk_PipelineM` FOREIGN KEY (`policy_id`) REFERENCES `PipelineManager_policy` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `SNMP_credential`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `SNMP_credential` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `description` longtext COLLATE utf8mb4_bin NOT NULL, + `version` varchar(2) COLLATE utf8mb4_bin NOT NULL, + `community` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `security_name` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `security_level` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `auth_protocol` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `auth_pass` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `priv_protocol` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `priv_pass` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `created_at` datetime(6) NOT NULL, + `updated_at` datetime(6) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `SNMP_device`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `SNMP_device` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `ip_address` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + `port` int NOT NULL, + `retries` int NOT NULL, + `timeout` int NOT NULL, + `created_at` datetime(6) NOT NULL, + `updated_at` datetime(6) NOT NULL, + `credential_id` bigint DEFAULT NULL, + `network_id` bigint DEFAULT NULL, + `device_template_id` bigint DEFAULT NULL, + `hostname` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + `building` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + `latitude` decimal(15,10) DEFAULT NULL, + `longitude` decimal(15,10) DEFAULT NULL, + `metadata` json NOT NULL DEFAULT (_utf8mb4'{}'), + `room` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + `site` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`), + KEY `SNMP_device_credential_id_f229ff4e_fk_SNMP_credential_id` (`credential_id`), + KEY `SNMP_device_name_087021_idx` (`name`), + KEY `SNMP_device_ip_addr_6a90e5_idx` (`ip_address`), + KEY `SNMP_device_created_becb56_idx` (`created_at` DESC), + KEY `SNMP_device_network_889f1c_idx` (`network_id`,`name`), + KEY `SNMP_device_device_template_id_ca143332_fk_SNMP_devi` (`device_template_id`), + KEY `SNMP_device_hostnam_fae88a_idx` (`hostname`), + CONSTRAINT `SNMP_device_credential_id_f229ff4e_fk_SNMP_credential_id` FOREIGN KEY (`credential_id`) REFERENCES `SNMP_credential` (`id`), + CONSTRAINT `SNMP_device_device_template_id_ca143332_fk_SNMP_devi` FOREIGN KEY (`device_template_id`) REFERENCES `SNMP_devicetemplate` (`id`), + CONSTRAINT `SNMP_device_network_id_4dea94fa_fk_SNMP_network_id` FOREIGN KEY (`network_id`) REFERENCES `SNMP_network` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `SNMP_devicetemplate`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `SNMP_devicetemplate` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `description` longtext COLLATE utf8mb4_bin NOT NULL, + `vendor` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `model` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `matching_rules` json NOT NULL, + `official` tinyint(1) NOT NULL, + `created_at` datetime(6) NOT NULL, + `updated_at` datetime(6) NOT NULL, + `product` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `type` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `official_key` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`), + UNIQUE KEY `official_key` (`official_key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `SNMP_devicetemplate_profiles`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `SNMP_devicetemplate_profiles` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `devicetemplate_id` bigint NOT NULL, + `profile_id` bigint NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `SNMP_devicetemplate_prof_devicetemplate_id_profil_fb6396a5_uniq` (`devicetemplate_id`,`profile_id`), + KEY `SNMP_devicetemplate__profile_id_4cb23513_fk_SNMP_prof` (`profile_id`), + CONSTRAINT `SNMP_devicetemplate__devicetemplate_id_da70425d_fk_SNMP_devi` FOREIGN KEY (`devicetemplate_id`) REFERENCES `SNMP_devicetemplate` (`id`), + CONSTRAINT `SNMP_devicetemplate__profile_id_4cb23513_fk_SNMP_prof` FOREIGN KEY (`profile_id`) REFERENCES `SNMP_profile` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `SNMP_network`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `SNMP_network` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `network_range` varchar(50) COLLATE utf8mb4_bin NOT NULL, + `discovery_enabled` tinyint(1) NOT NULL, + `traps_enabled` tinyint(1) NOT NULL, + `interval` int unsigned NOT NULL, + `created_at` datetime(6) NOT NULL, + `updated_at` datetime(6) NOT NULL, + `connection_id` bigint DEFAULT NULL, + `credential_id` bigint DEFAULT NULL, + `discovery_credential_id` bigint DEFAULT NULL, + `namespace` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `namespace_from_device_template` tinyint(1) NOT NULL, + `agent_connection_id` bigint DEFAULT NULL, + `deployment_mode` varchar(20) COLLATE utf8mb4_bin NOT NULL, + `credential_mode` varchar(20) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`), + KEY `SNMP_network_connection_id_432a2f88_fk_PipelineM` (`connection_id`), + KEY `SNMP_network_credential_id_3b15dfc1_fk_SNMP_credential_id` (`credential_id`), + KEY `SNMP_network_discovery_credential_d9229589_fk_SNMP_cred` (`discovery_credential_id`), + KEY `SNMP_network_agent_connection_id_cf646bb5_fk_PipelineM` (`agent_connection_id`), + CONSTRAINT `SNMP_network_agent_connection_id_cf646bb5_fk_PipelineM` FOREIGN KEY (`agent_connection_id`) REFERENCES `PipelineManager_connection` (`id`), + CONSTRAINT `SNMP_network_connection_id_432a2f88_fk_PipelineM` FOREIGN KEY (`connection_id`) REFERENCES `PipelineManager_connection` (`id`), + CONSTRAINT `SNMP_network_credential_id_3b15dfc1_fk_SNMP_credential_id` FOREIGN KEY (`credential_id`) REFERENCES `SNMP_credential` (`id`), + CONSTRAINT `SNMP_network_discovery_credential_d9229589_fk_SNMP_cred` FOREIGN KEY (`discovery_credential_id`) REFERENCES `SNMP_credential` (`id`), + CONSTRAINT `SNMP_network_chk_1` CHECK ((`interval` >= 0)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `SNMP_profile`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `SNMP_profile` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `profile_data` json NOT NULL, + `description` longtext COLLATE utf8mb4_bin NOT NULL, + `vendor` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `created_at` datetime(6) NOT NULL, + `updated_at` datetime(6) NOT NULL, + `product` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `normalizers` json NOT NULL DEFAULT (_utf8mb4'[]'), + `official_key` varchar(255) COLLATE utf8mb4_bin DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`), + UNIQUE KEY `official_key` (`official_key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `auth_group`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `auth_group` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(150) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `auth_group_permissions`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `auth_group_permissions` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `group_id` int NOT NULL, + `permission_id` int NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `auth_group_permissions_group_id_permission_id_0cd325b0_uniq` (`group_id`,`permission_id`), + KEY `auth_group_permissio_permission_id_84c5c92e_fk_auth_perm` (`permission_id`), + CONSTRAINT `auth_group_permissio_permission_id_84c5c92e_fk_auth_perm` FOREIGN KEY (`permission_id`) REFERENCES `auth_permission` (`id`), + CONSTRAINT `auth_group_permissions_group_id_b120cbf9_fk_auth_group_id` FOREIGN KEY (`group_id`) REFERENCES `auth_group` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `auth_permission`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `auth_permission` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `content_type_id` int NOT NULL, + `codename` varchar(100) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `auth_permission_content_type_id_codename_01ab375a_uniq` (`content_type_id`,`codename`), + CONSTRAINT `auth_permission_content_type_id_2f476e4b_fk_django_co` FOREIGN KEY (`content_type_id`) REFERENCES `django_content_type` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=85 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `auth_user`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `auth_user` ( + `id` int NOT NULL AUTO_INCREMENT, + `password` varchar(128) COLLATE utf8mb4_bin NOT NULL, + `last_login` datetime(6) DEFAULT NULL, + `is_superuser` tinyint(1) NOT NULL, + `username` varchar(150) COLLATE utf8mb4_bin NOT NULL, + `first_name` varchar(150) COLLATE utf8mb4_bin NOT NULL, + `last_name` varchar(150) COLLATE utf8mb4_bin NOT NULL, + `email` varchar(254) COLLATE utf8mb4_bin NOT NULL, + `is_staff` tinyint(1) NOT NULL, + `is_active` tinyint(1) NOT NULL, + `date_joined` datetime(6) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `username` (`username`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `auth_user_groups`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `auth_user_groups` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `user_id` int NOT NULL, + `group_id` int NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `auth_user_groups_user_id_group_id_94350c0c_uniq` (`user_id`,`group_id`), + KEY `auth_user_groups_group_id_97559544_fk_auth_group_id` (`group_id`), + CONSTRAINT `auth_user_groups_group_id_97559544_fk_auth_group_id` FOREIGN KEY (`group_id`) REFERENCES `auth_group` (`id`), + CONSTRAINT `auth_user_groups_user_id_6a12ed8b_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `auth_user_user_permissions`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `auth_user_user_permissions` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `user_id` int NOT NULL, + `permission_id` int NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `auth_user_user_permissions_user_id_permission_id_14a6b632_uniq` (`user_id`,`permission_id`), + KEY `auth_user_user_permi_permission_id_1fbb5f2c_fk_auth_perm` (`permission_id`), + CONSTRAINT `auth_user_user_permi_permission_id_1fbb5f2c_fk_auth_perm` FOREIGN KEY (`permission_id`) REFERENCES `auth_permission` (`id`), + CONSTRAINT `auth_user_user_permissions_user_id_a95ead1b_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `django_admin_log`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `django_admin_log` ( + `id` int NOT NULL AUTO_INCREMENT, + `action_time` datetime(6) NOT NULL, + `object_id` longtext COLLATE utf8mb4_bin, + `object_repr` varchar(200) COLLATE utf8mb4_bin NOT NULL, + `action_flag` smallint unsigned NOT NULL, + `change_message` longtext COLLATE utf8mb4_bin NOT NULL, + `content_type_id` int DEFAULT NULL, + `user_id` int NOT NULL, + PRIMARY KEY (`id`), + KEY `django_admin_log_content_type_id_c4bce8eb_fk_django_co` (`content_type_id`), + KEY `django_admin_log_user_id_c564eba6_fk_auth_user_id` (`user_id`), + CONSTRAINT `django_admin_log_content_type_id_c4bce8eb_fk_django_co` FOREIGN KEY (`content_type_id`) REFERENCES `django_content_type` (`id`), + CONSTRAINT `django_admin_log_user_id_c564eba6_fk_auth_user_id` FOREIGN KEY (`user_id`) REFERENCES `auth_user` (`id`), + CONSTRAINT `django_admin_log_chk_1` CHECK ((`action_flag` >= 0)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `django_content_type`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `django_content_type` ( + `id` int NOT NULL AUTO_INCREMENT, + `app_label` varchar(100) COLLATE utf8mb4_bin NOT NULL, + `model` varchar(100) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `django_content_type_app_label_model_76bd3d3b_uniq` (`app_label`,`model`) +) ENGINE=InnoDB AUTO_INCREMENT=22 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `django_migrations`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `django_migrations` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `app` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `name` varchar(255) COLLATE utf8mb4_bin NOT NULL, + `applied` datetime(6) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=76 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `django_session`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `django_session` ( + `session_key` varchar(40) COLLATE utf8mb4_bin NOT NULL, + `session_data` longtext COLLATE utf8mb4_bin NOT NULL, + `expire_date` datetime(6) NOT NULL, + PRIMARY KEY (`session_key`), + KEY `django_session_expire_date_a5c62663` (`expire_date`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `settings`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `settings` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `experimental_mode` tinyint(1) NOT NULL, + `agent_ui_url` varchar(512) COLLATE utf8mb4_bin NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +DROP TABLE IF EXISTS `snmp_deployment_state`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `snmp_deployment_state` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `last_deployment` datetime(6) DEFAULT NULL, + `last_config_change` datetime(6) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + diff --git a/docs/docs/logstashui/database/examples/schema-postgresql.sql b/docs/docs/logstashui/database/examples/schema-postgresql.sql new file mode 100644 index 00000000..b168a7e5 --- /dev/null +++ b/docs/docs/logstashui/database/examples/schema-postgresql.sql @@ -0,0 +1,1709 @@ +-- Snapshot of LogstashUI 0.5.2 `migrate` DDL on PostgreSQL 16. +-- Generated with pg_dump --schema-only. Do NOT apply this instead of +-- `logstashui manage migrate --noinput`. It will go stale with new migrations. +-- Create the empty database first (see create-postgresql.sql). + +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 16.15 (Debian 16.15-1.pgdg13+2) +-- Dumped by pg_dump version 16.15 (Debian 16.15-1.pgdg13+2) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: Management_userprofile; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."Management_userprofile" ( + id bigint NOT NULL, + role character varying(20) NOT NULL, + user_id integer NOT NULL +); + + +-- +-- Name: Management_userprofile_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."Management_userprofile" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."Management_userprofile_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: PipelineManager_apikey; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."PipelineManager_apikey" ( + id bigint NOT NULL, + api_key character varying(512) NOT NULL, + connection_id bigint NOT NULL +); + + +-- +-- Name: PipelineManager_apikey_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."PipelineManager_apikey" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."PipelineManager_apikey_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: PipelineManager_connection; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."PipelineManager_connection" ( + id bigint NOT NULL, + name character varying(100) NOT NULL, + connection_type character varying(20) NOT NULL, + host character varying(255), + port integer, + username character varying(100), + password character varying(512), + ssh_key text, + cloud_id character varying(255), + cloud_url character varying(200), + api_key character varying(512), + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + is_active boolean NOT NULL, + policy_id bigint, + agent_id character varying(255), + last_check_in timestamp with time zone, + status_blob jsonb, + restart_on_next_checkin boolean NOT NULL, + desired_agent_version character varying(20), + agent_api_port integer, + instance_id integer, + last_selected_at timestamp with time zone, + logstash_api_port integer, + logstash_version_resolved character varying(64) NOT NULL, + CONSTRAINT "PipelineManager_connection_agent_api_port_check" CHECK ((agent_api_port >= 0)), + CONSTRAINT "PipelineManager_connection_instance_id_check" CHECK ((instance_id >= 0)), + CONSTRAINT "PipelineManager_connection_logstash_api_port_check" CHECK ((logstash_api_port >= 0)), + CONSTRAINT "PipelineManager_connection_port_check" CHECK ((port >= 0)) +); + + +-- +-- Name: PipelineManager_connection_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."PipelineManager_connection" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."PipelineManager_connection_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: PipelineManager_enrollmenttoken; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."PipelineManager_enrollmenttoken" ( + id bigint NOT NULL, + token character varying(512) NOT NULL, + policy_id bigint NOT NULL, + name character varying(100) NOT NULL +); + + +-- +-- Name: PipelineManager_enrollmenttoken_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."PipelineManager_enrollmenttoken" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."PipelineManager_enrollmenttoken_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: PipelineManager_keystore; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."PipelineManager_keystore" ( + id bigint NOT NULL, + key_name character varying(100) NOT NULL, + key_value character varying(512) NOT NULL, + last_updated timestamp with time zone NOT NULL, + policy_id bigint NOT NULL, + revision_number integer NOT NULL, + kv_hash character varying(64) NOT NULL, + managed_by character varying(20) NOT NULL +); + + +-- +-- Name: PipelineManager_keystore_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."PipelineManager_keystore" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."PipelineManager_keystore_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: PipelineManager_pipeline; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."PipelineManager_pipeline" ( + id bigint NOT NULL, + name character varying(100) NOT NULL, + description text, + lscl text NOT NULL, + lscl_hash character varying(64), + last_updated timestamp with time zone NOT NULL, + policy_id bigint NOT NULL, + revision_number integer NOT NULL, + pipeline_batch_delay integer NOT NULL, + pipeline_batch_size integer NOT NULL, + pipeline_workers integer NOT NULL, + queue_checkpoint_writes integer NOT NULL, + queue_max_bytes character varying(20) NOT NULL, + queue_type character varying(20) NOT NULL, + pipeline_hash character varying(64) NOT NULL, + no_input boolean NOT NULL, + non_reloadable boolean NOT NULL, + managed_by character varying(20) NOT NULL +); + + +-- +-- Name: PipelineManager_pipeline_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."PipelineManager_pipeline" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."PipelineManager_pipeline_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: PipelineManager_policy; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."PipelineManager_policy" ( + id bigint NOT NULL, + name character varying(100) NOT NULL, + settings_path character varying(255) NOT NULL, + logs_path character varying(255) NOT NULL, + logstash_yml text NOT NULL, + jvm_options text NOT NULL, + log4j2_properties text NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + current_revision_number integer NOT NULL, + has_undeployed_changes boolean NOT NULL, + jvm_options_hash character varying(64) NOT NULL, + log4j2_properties_hash character varying(64) NOT NULL, + logstash_yml_hash character varying(64) NOT NULL, + binary_path character varying(255) NOT NULL, + keystore_password character varying(512), + keystore_password_hash character varying(64) NOT NULL, + last_deployed_at timestamp with time zone, + agent_api_port integer NOT NULL, + cloned_from_id bigint, + data_path character varying(255) NOT NULL, + is_system boolean NOT NULL, + keystore_env_file character varying(512) NOT NULL, + logstash_api_port integer NOT NULL, + logstash_download_dir character varying(512) NOT NULL, + logstash_source character varying(20) NOT NULL, + logstash_version character varying(32) NOT NULL, + policy_type character varying(20) NOT NULL, + CONSTRAINT "PipelineManager_policy_agent_api_port_check" CHECK ((agent_api_port >= 0)), + CONSTRAINT "PipelineManager_policy_logstash_api_port_check" CHECK ((logstash_api_port >= 0)) +); + + +-- +-- Name: PipelineManager_policy_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."PipelineManager_policy" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."PipelineManager_policy_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: PipelineManager_revision; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."PipelineManager_revision" ( + id bigint NOT NULL, + revision_number integer NOT NULL, + snapshot_json jsonb NOT NULL, + created_at timestamp with time zone NOT NULL, + created_by character varying(150) NOT NULL, + policy_id bigint NOT NULL +); + + +-- +-- Name: PipelineManager_revision_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."PipelineManager_revision" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."PipelineManager_revision_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: SNMP_credential; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."SNMP_credential" ( + id bigint NOT NULL, + name character varying(255) NOT NULL, + description text NOT NULL, + version character varying(2) NOT NULL, + community character varying(255) NOT NULL, + security_name character varying(255) NOT NULL, + security_level character varying(20) NOT NULL, + auth_protocol character varying(20) NOT NULL, + auth_pass character varying(255) NOT NULL, + priv_protocol character varying(20) NOT NULL, + priv_pass character varying(255) NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL +); + + +-- +-- Name: SNMP_credential_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."SNMP_credential" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."SNMP_credential_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: SNMP_device; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."SNMP_device" ( + id bigint NOT NULL, + name character varying(255) NOT NULL, + ip_address character varying(255), + port integer NOT NULL, + retries integer NOT NULL, + timeout integer NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + credential_id bigint, + network_id bigint, + device_template_id bigint, + hostname character varying(255), + building character varying(255), + latitude numeric(15,10), + longitude numeric(15,10), + metadata jsonb NOT NULL, + room character varying(255), + site character varying(255) +); + + +-- +-- Name: SNMP_device_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."SNMP_device" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."SNMP_device_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: SNMP_devicetemplate; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."SNMP_devicetemplate" ( + id bigint NOT NULL, + name character varying(255) NOT NULL, + description text NOT NULL, + vendor character varying(100) NOT NULL, + model character varying(100) NOT NULL, + matching_rules jsonb NOT NULL, + official boolean NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + product character varying(100) NOT NULL, + type character varying(100) NOT NULL, + official_key character varying(255) +); + + +-- +-- Name: SNMP_devicetemplate_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."SNMP_devicetemplate" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."SNMP_devicetemplate_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: SNMP_devicetemplate_profiles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."SNMP_devicetemplate_profiles" ( + id bigint NOT NULL, + devicetemplate_id bigint NOT NULL, + profile_id bigint NOT NULL +); + + +-- +-- Name: SNMP_devicetemplate_profiles_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."SNMP_devicetemplate_profiles" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."SNMP_devicetemplate_profiles_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: SNMP_network; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."SNMP_network" ( + id bigint NOT NULL, + name character varying(255) NOT NULL, + network_range character varying(50) NOT NULL, + discovery_enabled boolean NOT NULL, + traps_enabled boolean NOT NULL, + "interval" integer NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + connection_id bigint, + credential_id bigint, + discovery_credential_id bigint, + namespace character varying(100) NOT NULL, + namespace_from_device_template boolean NOT NULL, + agent_connection_id bigint, + deployment_mode character varying(20) NOT NULL, + credential_mode character varying(20) NOT NULL, + CONSTRAINT "SNMP_network_interval_check" CHECK (("interval" >= 0)) +); + + +-- +-- Name: SNMP_network_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."SNMP_network" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."SNMP_network_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: SNMP_profile; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public."SNMP_profile" ( + id bigint NOT NULL, + name character varying(255) NOT NULL, + profile_data jsonb NOT NULL, + description text NOT NULL, + vendor character varying(100) NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + product character varying(100) NOT NULL, + normalizers jsonb NOT NULL, + official_key character varying(255) +); + + +-- +-- Name: SNMP_profile_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public."SNMP_profile" ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public."SNMP_profile_id_seq" + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: auth_group; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.auth_group ( + id integer NOT NULL, + name character varying(150) NOT NULL +); + + +-- +-- Name: auth_group_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.auth_group ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.auth_group_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: auth_group_permissions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.auth_group_permissions ( + id bigint NOT NULL, + group_id integer NOT NULL, + permission_id integer NOT NULL +); + + +-- +-- Name: auth_group_permissions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.auth_group_permissions ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.auth_group_permissions_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: auth_permission; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.auth_permission ( + id integer NOT NULL, + name character varying(255) NOT NULL, + content_type_id integer NOT NULL, + codename character varying(100) NOT NULL +); + + +-- +-- Name: auth_permission_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.auth_permission ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.auth_permission_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: auth_user; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.auth_user ( + id integer NOT NULL, + password character varying(128) NOT NULL, + last_login timestamp with time zone, + is_superuser boolean NOT NULL, + username character varying(150) NOT NULL, + first_name character varying(150) NOT NULL, + last_name character varying(150) NOT NULL, + email character varying(254) NOT NULL, + is_staff boolean NOT NULL, + is_active boolean NOT NULL, + date_joined timestamp with time zone NOT NULL +); + + +-- +-- Name: auth_user_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.auth_user_groups ( + id bigint NOT NULL, + user_id integer NOT NULL, + group_id integer NOT NULL +); + + +-- +-- Name: auth_user_groups_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.auth_user_groups ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.auth_user_groups_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: auth_user_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.auth_user ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.auth_user_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: auth_user_user_permissions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.auth_user_user_permissions ( + id bigint NOT NULL, + user_id integer NOT NULL, + permission_id integer NOT NULL +); + + +-- +-- Name: auth_user_user_permissions_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.auth_user_user_permissions ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.auth_user_user_permissions_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: django_admin_log; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.django_admin_log ( + id integer NOT NULL, + action_time timestamp with time zone NOT NULL, + object_id text, + object_repr character varying(200) NOT NULL, + action_flag smallint NOT NULL, + change_message text NOT NULL, + content_type_id integer, + user_id integer NOT NULL, + CONSTRAINT django_admin_log_action_flag_check CHECK ((action_flag >= 0)) +); + + +-- +-- Name: django_admin_log_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.django_admin_log ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.django_admin_log_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: django_content_type; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.django_content_type ( + id integer NOT NULL, + app_label character varying(100) NOT NULL, + model character varying(100) NOT NULL +); + + +-- +-- Name: django_content_type_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.django_content_type ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.django_content_type_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: django_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.django_migrations ( + id bigint NOT NULL, + app character varying(255) NOT NULL, + name character varying(255) NOT NULL, + applied timestamp with time zone NOT NULL +); + + +-- +-- Name: django_migrations_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.django_migrations ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.django_migrations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: django_session; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.django_session ( + session_key character varying(40) NOT NULL, + session_data text NOT NULL, + expire_date timestamp with time zone NOT NULL +); + + +-- +-- Name: settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.settings ( + id bigint NOT NULL, + experimental_mode boolean NOT NULL, + agent_ui_url character varying(512) NOT NULL +); + + +-- +-- Name: settings_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.settings ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.settings_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: snmp_deployment_state; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.snmp_deployment_state ( + id bigint NOT NULL, + last_deployment timestamp with time zone, + last_config_change timestamp with time zone +); + + +-- +-- Name: snmp_deployment_state_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +ALTER TABLE public.snmp_deployment_state ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME public.snmp_deployment_state_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + + +-- +-- Name: Management_userprofile Management_userprofile_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."Management_userprofile" + ADD CONSTRAINT "Management_userprofile_pkey" PRIMARY KEY (id); + + +-- +-- Name: Management_userprofile Management_userprofile_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."Management_userprofile" + ADD CONSTRAINT "Management_userprofile_user_id_key" UNIQUE (user_id); + + +-- +-- Name: PipelineManager_apikey PipelineManager_apikey_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_apikey" + ADD CONSTRAINT "PipelineManager_apikey_pkey" PRIMARY KEY (id); + + +-- +-- Name: PipelineManager_connection PipelineManager_connection_agent_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_connection" + ADD CONSTRAINT "PipelineManager_connection_agent_id_key" UNIQUE (agent_id); + + +-- +-- Name: PipelineManager_connection PipelineManager_connection_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_connection" + ADD CONSTRAINT "PipelineManager_connection_pkey" PRIMARY KEY (id); + + +-- +-- Name: PipelineManager_enrollmenttoken PipelineManager_enrollmenttoken_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_enrollmenttoken" + ADD CONSTRAINT "PipelineManager_enrollmenttoken_pkey" PRIMARY KEY (id); + + +-- +-- Name: PipelineManager_keystore PipelineManager_keystore_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_keystore" + ADD CONSTRAINT "PipelineManager_keystore_pkey" PRIMARY KEY (id); + + +-- +-- Name: PipelineManager_pipeline PipelineManager_pipeline_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_pipeline" + ADD CONSTRAINT "PipelineManager_pipeline_pkey" PRIMARY KEY (id); + + +-- +-- Name: PipelineManager_policy PipelineManager_policy_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_policy" + ADD CONSTRAINT "PipelineManager_policy_name_key" UNIQUE (name); + + +-- +-- Name: PipelineManager_policy PipelineManager_policy_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_policy" + ADD CONSTRAINT "PipelineManager_policy_pkey" PRIMARY KEY (id); + + +-- +-- Name: PipelineManager_revision PipelineManager_revision_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_revision" + ADD CONSTRAINT "PipelineManager_revision_pkey" PRIMARY KEY (id); + + +-- +-- Name: SNMP_credential SNMP_credential_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_credential" + ADD CONSTRAINT "SNMP_credential_name_key" UNIQUE (name); + + +-- +-- Name: SNMP_credential SNMP_credential_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_credential" + ADD CONSTRAINT "SNMP_credential_pkey" PRIMARY KEY (id); + + +-- +-- Name: SNMP_device SNMP_device_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_device" + ADD CONSTRAINT "SNMP_device_name_key" UNIQUE (name); + + +-- +-- Name: SNMP_device SNMP_device_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_device" + ADD CONSTRAINT "SNMP_device_pkey" PRIMARY KEY (id); + + +-- +-- Name: SNMP_devicetemplate SNMP_devicetemplate_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_devicetemplate" + ADD CONSTRAINT "SNMP_devicetemplate_name_key" UNIQUE (name); + + +-- +-- Name: SNMP_devicetemplate SNMP_devicetemplate_official_key_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_devicetemplate" + ADD CONSTRAINT "SNMP_devicetemplate_official_key_key" UNIQUE (official_key); + + +-- +-- Name: SNMP_devicetemplate SNMP_devicetemplate_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_devicetemplate" + ADD CONSTRAINT "SNMP_devicetemplate_pkey" PRIMARY KEY (id); + + +-- +-- Name: SNMP_devicetemplate_profiles SNMP_devicetemplate_prof_devicetemplate_id_profil_fb6396a5_uniq; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_devicetemplate_profiles" + ADD CONSTRAINT "SNMP_devicetemplate_prof_devicetemplate_id_profil_fb6396a5_uniq" UNIQUE (devicetemplate_id, profile_id); + + +-- +-- Name: SNMP_devicetemplate_profiles SNMP_devicetemplate_profiles_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_devicetemplate_profiles" + ADD CONSTRAINT "SNMP_devicetemplate_profiles_pkey" PRIMARY KEY (id); + + +-- +-- Name: SNMP_network SNMP_network_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_network" + ADD CONSTRAINT "SNMP_network_name_key" UNIQUE (name); + + +-- +-- Name: SNMP_network SNMP_network_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_network" + ADD CONSTRAINT "SNMP_network_pkey" PRIMARY KEY (id); + + +-- +-- Name: SNMP_profile SNMP_profile_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_profile" + ADD CONSTRAINT "SNMP_profile_name_key" UNIQUE (name); + + +-- +-- Name: SNMP_profile SNMP_profile_official_key_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_profile" + ADD CONSTRAINT "SNMP_profile_official_key_key" UNIQUE (official_key); + + +-- +-- Name: SNMP_profile SNMP_profile_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_profile" + ADD CONSTRAINT "SNMP_profile_pkey" PRIMARY KEY (id); + + +-- +-- Name: auth_group auth_group_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_group + ADD CONSTRAINT auth_group_name_key UNIQUE (name); + + +-- +-- Name: auth_group_permissions auth_group_permissions_group_id_permission_id_0cd325b0_uniq; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_group_permissions + ADD CONSTRAINT auth_group_permissions_group_id_permission_id_0cd325b0_uniq UNIQUE (group_id, permission_id); + + +-- +-- Name: auth_group_permissions auth_group_permissions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_group_permissions + ADD CONSTRAINT auth_group_permissions_pkey PRIMARY KEY (id); + + +-- +-- Name: auth_group auth_group_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_group + ADD CONSTRAINT auth_group_pkey PRIMARY KEY (id); + + +-- +-- Name: auth_permission auth_permission_content_type_id_codename_01ab375a_uniq; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_permission + ADD CONSTRAINT auth_permission_content_type_id_codename_01ab375a_uniq UNIQUE (content_type_id, codename); + + +-- +-- Name: auth_permission auth_permission_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_permission + ADD CONSTRAINT auth_permission_pkey PRIMARY KEY (id); + + +-- +-- Name: auth_user_groups auth_user_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user_groups + ADD CONSTRAINT auth_user_groups_pkey PRIMARY KEY (id); + + +-- +-- Name: auth_user_groups auth_user_groups_user_id_group_id_94350c0c_uniq; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user_groups + ADD CONSTRAINT auth_user_groups_user_id_group_id_94350c0c_uniq UNIQUE (user_id, group_id); + + +-- +-- Name: auth_user auth_user_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user + ADD CONSTRAINT auth_user_pkey PRIMARY KEY (id); + + +-- +-- Name: auth_user_user_permissions auth_user_user_permissions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user_user_permissions + ADD CONSTRAINT auth_user_user_permissions_pkey PRIMARY KEY (id); + + +-- +-- Name: auth_user_user_permissions auth_user_user_permissions_user_id_permission_id_14a6b632_uniq; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user_user_permissions + ADD CONSTRAINT auth_user_user_permissions_user_id_permission_id_14a6b632_uniq UNIQUE (user_id, permission_id); + + +-- +-- Name: auth_user auth_user_username_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user + ADD CONSTRAINT auth_user_username_key UNIQUE (username); + + +-- +-- Name: django_admin_log django_admin_log_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.django_admin_log + ADD CONSTRAINT django_admin_log_pkey PRIMARY KEY (id); + + +-- +-- Name: django_content_type django_content_type_app_label_model_76bd3d3b_uniq; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.django_content_type + ADD CONSTRAINT django_content_type_app_label_model_76bd3d3b_uniq UNIQUE (app_label, model); + + +-- +-- Name: django_content_type django_content_type_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.django_content_type + ADD CONSTRAINT django_content_type_pkey PRIMARY KEY (id); + + +-- +-- Name: django_migrations django_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.django_migrations + ADD CONSTRAINT django_migrations_pkey PRIMARY KEY (id); + + +-- +-- Name: django_session django_session_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.django_session + ADD CONSTRAINT django_session_pkey PRIMARY KEY (session_key); + + +-- +-- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.settings + ADD CONSTRAINT settings_pkey PRIMARY KEY (id); + + +-- +-- Name: snmp_deployment_state snmp_deployment_state_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.snmp_deployment_state + ADD CONSTRAINT snmp_deployment_state_pkey PRIMARY KEY (id); + + +-- +-- Name: PipelineManager_keystore unique_key_per_policy; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_keystore" + ADD CONSTRAINT unique_key_per_policy UNIQUE (policy_id, key_name); + + +-- +-- Name: PipelineManager_pipeline unique_pipeline_per_policy; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_pipeline" + ADD CONSTRAINT unique_pipeline_per_policy UNIQUE (policy_id, name); + + +-- +-- Name: PipelineManager_revision unique_revision_per_policy; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_revision" + ADD CONSTRAINT unique_revision_per_policy UNIQUE (policy_id, revision_number); + + +-- +-- Name: PipelineManager_enrollmenttoken unique_token_name_per_policy; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_enrollmenttoken" + ADD CONSTRAINT unique_token_name_per_policy UNIQUE (policy_id, name); + + +-- +-- Name: PipelineManager_apikey_connection_id_27156847; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_apikey_connection_id_27156847" ON public."PipelineManager_apikey" USING btree (connection_id); + + +-- +-- Name: PipelineManager_connection_agent_id_d5ec5cd5_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_connection_agent_id_d5ec5cd5_like" ON public."PipelineManager_connection" USING btree (agent_id varchar_pattern_ops); + + +-- +-- Name: PipelineManager_connection_policy_id_819d316d; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_connection_policy_id_819d316d" ON public."PipelineManager_connection" USING btree (policy_id); + + +-- +-- Name: PipelineManager_enrollmenttoken_policy_id_355d6dca; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_enrollmenttoken_policy_id_355d6dca" ON public."PipelineManager_enrollmenttoken" USING btree (policy_id); + + +-- +-- Name: PipelineManager_keystore_policy_id_6309b699; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_keystore_policy_id_6309b699" ON public."PipelineManager_keystore" USING btree (policy_id); + + +-- +-- Name: PipelineManager_pipeline_policy_id_7c3a6fd9; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_pipeline_policy_id_7c3a6fd9" ON public."PipelineManager_pipeline" USING btree (policy_id); + + +-- +-- Name: PipelineManager_policy_cloned_from_id_4177e74a; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_policy_cloned_from_id_4177e74a" ON public."PipelineManager_policy" USING btree (cloned_from_id); + + +-- +-- Name: PipelineManager_policy_name_ec6c8a56_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_policy_name_ec6c8a56_like" ON public."PipelineManager_policy" USING btree (name varchar_pattern_ops); + + +-- +-- Name: PipelineManager_revision_policy_id_99e7d146; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "PipelineManager_revision_policy_id_99e7d146" ON public."PipelineManager_revision" USING btree (policy_id); + + +-- +-- Name: SNMP_credential_name_2e8e0977_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_credential_name_2e8e0977_like" ON public."SNMP_credential" USING btree (name varchar_pattern_ops); + + +-- +-- Name: SNMP_device_created_becb56_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_created_becb56_idx" ON public."SNMP_device" USING btree (created_at DESC); + + +-- +-- Name: SNMP_device_credential_id_f229ff4e; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_credential_id_f229ff4e" ON public."SNMP_device" USING btree (credential_id); + + +-- +-- Name: SNMP_device_device_template_id_ca143332; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_device_template_id_ca143332" ON public."SNMP_device" USING btree (device_template_id); + + +-- +-- Name: SNMP_device_hostnam_fae88a_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_hostnam_fae88a_idx" ON public."SNMP_device" USING btree (hostname); + + +-- +-- Name: SNMP_device_ip_addr_6a90e5_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_ip_addr_6a90e5_idx" ON public."SNMP_device" USING btree (ip_address); + + +-- +-- Name: SNMP_device_name_087021_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_name_087021_idx" ON public."SNMP_device" USING btree (name); + + +-- +-- Name: SNMP_device_name_e0c9483b_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_name_e0c9483b_like" ON public."SNMP_device" USING btree (name varchar_pattern_ops); + + +-- +-- Name: SNMP_device_network_889f1c_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_network_889f1c_idx" ON public."SNMP_device" USING btree (network_id, name); + + +-- +-- Name: SNMP_device_network_id_4dea94fa; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_device_network_id_4dea94fa" ON public."SNMP_device" USING btree (network_id); + + +-- +-- Name: SNMP_devicetemplate_name_9c02299b_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_devicetemplate_name_9c02299b_like" ON public."SNMP_devicetemplate" USING btree (name varchar_pattern_ops); + + +-- +-- Name: SNMP_devicetemplate_official_key_65a0ae02_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_devicetemplate_official_key_65a0ae02_like" ON public."SNMP_devicetemplate" USING btree (official_key varchar_pattern_ops); + + +-- +-- Name: SNMP_devicetemplate_profiles_devicetemplate_id_da70425d; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_devicetemplate_profiles_devicetemplate_id_da70425d" ON public."SNMP_devicetemplate_profiles" USING btree (devicetemplate_id); + + +-- +-- Name: SNMP_devicetemplate_profiles_profile_id_4cb23513; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_devicetemplate_profiles_profile_id_4cb23513" ON public."SNMP_devicetemplate_profiles" USING btree (profile_id); + + +-- +-- Name: SNMP_network_agent_connection_id_cf646bb5; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_network_agent_connection_id_cf646bb5" ON public."SNMP_network" USING btree (agent_connection_id); + + +-- +-- Name: SNMP_network_connection_id_432a2f88; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_network_connection_id_432a2f88" ON public."SNMP_network" USING btree (connection_id); + + +-- +-- Name: SNMP_network_credential_id_3b15dfc1; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_network_credential_id_3b15dfc1" ON public."SNMP_network" USING btree (credential_id); + + +-- +-- Name: SNMP_network_discovery_credential_id_d9229589; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_network_discovery_credential_id_d9229589" ON public."SNMP_network" USING btree (discovery_credential_id); + + +-- +-- Name: SNMP_network_name_83546472_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_network_name_83546472_like" ON public."SNMP_network" USING btree (name varchar_pattern_ops); + + +-- +-- Name: SNMP_profile_name_8a9d3c9d_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_profile_name_8a9d3c9d_like" ON public."SNMP_profile" USING btree (name varchar_pattern_ops); + + +-- +-- Name: SNMP_profile_official_key_de922dd3_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX "SNMP_profile_official_key_de922dd3_like" ON public."SNMP_profile" USING btree (official_key varchar_pattern_ops); + + +-- +-- Name: auth_group_name_a6ea08ec_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_group_name_a6ea08ec_like ON public.auth_group USING btree (name varchar_pattern_ops); + + +-- +-- Name: auth_group_permissions_group_id_b120cbf9; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_group_permissions_group_id_b120cbf9 ON public.auth_group_permissions USING btree (group_id); + + +-- +-- Name: auth_group_permissions_permission_id_84c5c92e; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_group_permissions_permission_id_84c5c92e ON public.auth_group_permissions USING btree (permission_id); + + +-- +-- Name: auth_permission_content_type_id_2f476e4b; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_permission_content_type_id_2f476e4b ON public.auth_permission USING btree (content_type_id); + + +-- +-- Name: auth_user_groups_group_id_97559544; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_user_groups_group_id_97559544 ON public.auth_user_groups USING btree (group_id); + + +-- +-- Name: auth_user_groups_user_id_6a12ed8b; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_user_groups_user_id_6a12ed8b ON public.auth_user_groups USING btree (user_id); + + +-- +-- Name: auth_user_user_permissions_permission_id_1fbb5f2c; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_user_user_permissions_permission_id_1fbb5f2c ON public.auth_user_user_permissions USING btree (permission_id); + + +-- +-- Name: auth_user_user_permissions_user_id_a95ead1b; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_user_user_permissions_user_id_a95ead1b ON public.auth_user_user_permissions USING btree (user_id); + + +-- +-- Name: auth_user_username_6821ab7c_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_user_username_6821ab7c_like ON public.auth_user USING btree (username varchar_pattern_ops); + + +-- +-- Name: django_admin_log_content_type_id_c4bce8eb; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX django_admin_log_content_type_id_c4bce8eb ON public.django_admin_log USING btree (content_type_id); + + +-- +-- Name: django_admin_log_user_id_c564eba6; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX django_admin_log_user_id_c564eba6 ON public.django_admin_log USING btree (user_id); + + +-- +-- Name: django_session_expire_date_a5c62663; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX django_session_expire_date_a5c62663 ON public.django_session USING btree (expire_date); + + +-- +-- Name: django_session_session_key_c0390e0f_like; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX django_session_session_key_c0390e0f_like ON public.django_session USING btree (session_key varchar_pattern_ops); + + +-- +-- Name: Management_userprofile Management_userprofile_user_id_70f1a900_fk_auth_user_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."Management_userprofile" + ADD CONSTRAINT "Management_userprofile_user_id_70f1a900_fk_auth_user_id" FOREIGN KEY (user_id) REFERENCES public.auth_user(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: PipelineManager_apikey PipelineManager_apik_connection_id_27156847_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_apikey" + ADD CONSTRAINT "PipelineManager_apik_connection_id_27156847_fk_PipelineM" FOREIGN KEY (connection_id) REFERENCES public."PipelineManager_connection"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: PipelineManager_connection PipelineManager_conn_policy_id_819d316d_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_connection" + ADD CONSTRAINT "PipelineManager_conn_policy_id_819d316d_fk_PipelineM" FOREIGN KEY (policy_id) REFERENCES public."PipelineManager_policy"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: PipelineManager_enrollmenttoken PipelineManager_enro_policy_id_355d6dca_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_enrollmenttoken" + ADD CONSTRAINT "PipelineManager_enro_policy_id_355d6dca_fk_PipelineM" FOREIGN KEY (policy_id) REFERENCES public."PipelineManager_policy"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: PipelineManager_keystore PipelineManager_keys_policy_id_6309b699_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_keystore" + ADD CONSTRAINT "PipelineManager_keys_policy_id_6309b699_fk_PipelineM" FOREIGN KEY (policy_id) REFERENCES public."PipelineManager_policy"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: PipelineManager_pipeline PipelineManager_pipe_policy_id_7c3a6fd9_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_pipeline" + ADD CONSTRAINT "PipelineManager_pipe_policy_id_7c3a6fd9_fk_PipelineM" FOREIGN KEY (policy_id) REFERENCES public."PipelineManager_policy"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: PipelineManager_policy PipelineManager_poli_cloned_from_id_4177e74a_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_policy" + ADD CONSTRAINT "PipelineManager_poli_cloned_from_id_4177e74a_fk_PipelineM" FOREIGN KEY (cloned_from_id) REFERENCES public."PipelineManager_policy"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: PipelineManager_revision PipelineManager_revi_policy_id_99e7d146_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."PipelineManager_revision" + ADD CONSTRAINT "PipelineManager_revi_policy_id_99e7d146_fk_PipelineM" FOREIGN KEY (policy_id) REFERENCES public."PipelineManager_policy"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_device SNMP_device_credential_id_f229ff4e_fk_SNMP_credential_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_device" + ADD CONSTRAINT "SNMP_device_credential_id_f229ff4e_fk_SNMP_credential_id" FOREIGN KEY (credential_id) REFERENCES public."SNMP_credential"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_device SNMP_device_device_template_id_ca143332_fk_SNMP_devi; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_device" + ADD CONSTRAINT "SNMP_device_device_template_id_ca143332_fk_SNMP_devi" FOREIGN KEY (device_template_id) REFERENCES public."SNMP_devicetemplate"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_device SNMP_device_network_id_4dea94fa_fk_SNMP_network_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_device" + ADD CONSTRAINT "SNMP_device_network_id_4dea94fa_fk_SNMP_network_id" FOREIGN KEY (network_id) REFERENCES public."SNMP_network"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_devicetemplate_profiles SNMP_devicetemplate__devicetemplate_id_da70425d_fk_SNMP_devi; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_devicetemplate_profiles" + ADD CONSTRAINT "SNMP_devicetemplate__devicetemplate_id_da70425d_fk_SNMP_devi" FOREIGN KEY (devicetemplate_id) REFERENCES public."SNMP_devicetemplate"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_devicetemplate_profiles SNMP_devicetemplate__profile_id_4cb23513_fk_SNMP_prof; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_devicetemplate_profiles" + ADD CONSTRAINT "SNMP_devicetemplate__profile_id_4cb23513_fk_SNMP_prof" FOREIGN KEY (profile_id) REFERENCES public."SNMP_profile"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_network SNMP_network_agent_connection_id_cf646bb5_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_network" + ADD CONSTRAINT "SNMP_network_agent_connection_id_cf646bb5_fk_PipelineM" FOREIGN KEY (agent_connection_id) REFERENCES public."PipelineManager_connection"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_network SNMP_network_connection_id_432a2f88_fk_PipelineM; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_network" + ADD CONSTRAINT "SNMP_network_connection_id_432a2f88_fk_PipelineM" FOREIGN KEY (connection_id) REFERENCES public."PipelineManager_connection"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_network SNMP_network_credential_id_3b15dfc1_fk_SNMP_credential_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_network" + ADD CONSTRAINT "SNMP_network_credential_id_3b15dfc1_fk_SNMP_credential_id" FOREIGN KEY (credential_id) REFERENCES public."SNMP_credential"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: SNMP_network SNMP_network_discovery_credential_d9229589_fk_SNMP_cred; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public."SNMP_network" + ADD CONSTRAINT "SNMP_network_discovery_credential_d9229589_fk_SNMP_cred" FOREIGN KEY (discovery_credential_id) REFERENCES public."SNMP_credential"(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: auth_group_permissions auth_group_permissio_permission_id_84c5c92e_fk_auth_perm; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_group_permissions + ADD CONSTRAINT auth_group_permissio_permission_id_84c5c92e_fk_auth_perm FOREIGN KEY (permission_id) REFERENCES public.auth_permission(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: auth_group_permissions auth_group_permissions_group_id_b120cbf9_fk_auth_group_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_group_permissions + ADD CONSTRAINT auth_group_permissions_group_id_b120cbf9_fk_auth_group_id FOREIGN KEY (group_id) REFERENCES public.auth_group(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: auth_permission auth_permission_content_type_id_2f476e4b_fk_django_co; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_permission + ADD CONSTRAINT auth_permission_content_type_id_2f476e4b_fk_django_co FOREIGN KEY (content_type_id) REFERENCES public.django_content_type(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: auth_user_groups auth_user_groups_group_id_97559544_fk_auth_group_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user_groups + ADD CONSTRAINT auth_user_groups_group_id_97559544_fk_auth_group_id FOREIGN KEY (group_id) REFERENCES public.auth_group(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: auth_user_groups auth_user_groups_user_id_6a12ed8b_fk_auth_user_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user_groups + ADD CONSTRAINT auth_user_groups_user_id_6a12ed8b_fk_auth_user_id FOREIGN KEY (user_id) REFERENCES public.auth_user(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: auth_user_user_permissions auth_user_user_permi_permission_id_1fbb5f2c_fk_auth_perm; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user_user_permissions + ADD CONSTRAINT auth_user_user_permi_permission_id_1fbb5f2c_fk_auth_perm FOREIGN KEY (permission_id) REFERENCES public.auth_permission(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: auth_user_user_permissions auth_user_user_permissions_user_id_a95ead1b_fk_auth_user_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_user_user_permissions + ADD CONSTRAINT auth_user_user_permissions_user_id_a95ead1b_fk_auth_user_id FOREIGN KEY (user_id) REFERENCES public.auth_user(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: django_admin_log django_admin_log_content_type_id_c4bce8eb_fk_django_co; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.django_admin_log + ADD CONSTRAINT django_admin_log_content_type_id_c4bce8eb_fk_django_co FOREIGN KEY (content_type_id) REFERENCES public.django_content_type(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- Name: django_admin_log django_admin_log_user_id_c564eba6_fk_auth_user_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.django_admin_log + ADD CONSTRAINT django_admin_log_user_id_c564eba6_fk_auth_user_id FOREIGN KEY (user_id) REFERENCES public.auth_user(id) DEFERRABLE INITIALLY DEFERRED; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/docs/docs/logstashui/database/index.md b/docs/docs/logstashui/database/index.md new file mode 100644 index 00000000..2c59cfbd --- /dev/null +++ b/docs/docs/logstashui/database/index.md @@ -0,0 +1,79 @@ +# Database + +LogstashUI stores operator data (policies, connections, pipelines, SNMP, users) in a SQL database. The Django ORM is the only CRUD layer. Choose the engine with discrete environment variables. There is no `DATABASE_URL` and no YAML. + +| Engine | `LOGSTASHUI_DB_ENGINE` | Server floor | When to use | +|---|---|---|---| +| **SQLite** (default) | `sqlite` (aliases: `sqlite3`, empty) | — | Single replica, lab, default Docker | +| **PostgreSQL** | `postgresql` (alias: `postgres`) | 14+ | Concurrent agents, Kubernetes | +| **MariaDB / MySQL** | `mysql` (aliases: `mariadb`, `my`) | MariaDB 10.6+ / MySQL 8.0+ | Same as Postgres; one engine covers both | + +`logstashui serve` logs a **warning** when the engine is SQLite and `LOGSTASHUI_WORKERS` > 1. It still starts. SQLite does not scale under gunicorn/gevent with concurrent agents. + +The container image already installs `LogstashUI[databases]` (both drivers) and `LogstashUI[otel]`. Tracing stays off until `LOGSTASHUI_OTEL=true`. Native pip/uv: + +```bash +uv pip install 'LogstashUI[postgres]' +uv pip install 'LogstashUI[mysql]' +uv pip install 'LogstashUI[databases]' # both +``` + +Missing driver fails at startup with the extra name. Unknown engine, missing `HOST`/`USER` on server engines, or a server below the floor fails **before** `migrate`. + +Create the empty server database yourself ([examples](examples/)). Tables come from `logstashui manage migrate` (also run by `logstashui serve`). Do not apply the schema snapshots instead of migrate. + +MySQL/MariaDB **must** be `utf8mb4` / `utf8mb4_bin` so unique names match SQLite/Postgres case-sensitivity. The Django connection also sets `NAMES utf8mb4 COLLATE utf8mb4_bin`. + +--- + +## Environment variables + +Unset keys use the default in the **Default** column. Empty string after strip is treated as unset for name/port/sslmode. + +### Data directory (still required when the database is remote) + +| Variable | Default | Notes | +|---|---|---| +| `LOGSTASHUI_DATA_DIR` | `$(pwd)/logstashui_data` | Docker/K8s/systemd **must** set `/var/lib/logstashui`. TLS, `.django_secret_key`, logs, `staticfiles/`, SQLite file. | +| `LOGSTASHUI_LOGS_DIR` | `$LOGSTASHUI_DATA_DIR/logs` | | + +### Engine and connection + +| Variable | Default | Notes | +|---|---|---| +| `LOGSTASHUI_DB_ENGINE` | `sqlite` | `sqlite`, `postgresql`, `mysql`. Aliases: `sqlite3`, `postgres`, `mariadb`, `my`. | +| `LOGSTASHUI_DB_NAME` | sqlite: `$LOGSTASHUI_DATA_DIR/db.sqlite3`; else `logstashui` | Database name, or SQLite file path. | +| `LOGSTASHUI_DB_HOST` | empty | **Required** for postgresql/mysql. | +| `LOGSTASHUI_DB_PORT` | postgresql: `5432`; mysql: `3306` | | +| `LOGSTASHUI_DB_USER` | empty | **Required** for postgresql/mysql. | +| `LOGSTASHUI_DB_PASSWORD` | empty | Secret / `chmod 640` EnvironmentFile. Never in a ConfigMap. | +| `LOGSTASHUI_DB_SSLMODE` | postgresql: `prefer` | `disable` `allow` `prefer` `require` `verify-ca` `verify-full`. Ignored for mysql/sqlite. | +| `LOGSTASHUI_DB_SSL_CA` | empty | CA file. Postgres `sslrootcert` when `verify-*`; MySQL `ssl.ca`. | +| `LOGSTASHUI_DB_CONN_MAX_AGE` | `60` | Seconds. `0` closes per request. Non-integer → startup error. | +| `LOGSTASHUI_DB_CONN_HEALTH_CHECKS` | `true` | Django `CONN_HEALTH_CHECKS`. | + +SQLite opens WAL (`PRAGMA journal_mode=WAL`) and `busy_timeout=20000`. + +gunicorn stays `--worker-class gevent` (`--worker-connections 1000`). Keep `LOGSTASHUI_WORKERS` × in-flight requests under the server `max_connections`. PgBouncer is optional. + +Related server knobs (not DB-specific, but they interact): + +| Variable | Default | Notes | +|---|---|---| +| `LOGSTASHUI_WORKERS` | `2` | SQLite + workers > 1 logs a warning. | +| `LOGSTASHUI_TLS` | `true` | Keep `true` in Kubernetes. | +| `SECRET_KEY` | auto in `DATA_DIR/.django_secret_key` | Keep `DATA_DIR` across engine switches or encrypted rows will not decrypt. | + +--- + +## Kubernetes + +PVC at `/var/lib/logstashui` for every engine. ConfigMap: `LOGSTASHUI_DB_ENGINE` / `HOST` / `PORT` / `NAME` / `USER` (and `SSLMODE` for Postgres). Secret: `LOGSTASHUI_DB_PASSWORD`. CloudNativePG: [CNPG](/docs/docs/logstashui/kubernetes/cnpg.md). + +Manifests: [kubernetes/examples](/docs/docs/logstashui/kubernetes/examples/). + +--- + +## Moving data + +[Migration](migration.md) — offline `dumpdata`/`loaddata` (supported) and BETA `logstashui migrate-engine`. diff --git a/docs/docs/logstashui/database/migration.md b/docs/docs/logstashui/database/migration.md new file mode 100644 index 00000000..e5f4e2ce --- /dev/null +++ b/docs/docs/logstashui/database/migration.md @@ -0,0 +1,75 @@ +# Migrating the database + +Two paths copy **SQLite** data onto PostgreSQL or MySQL/MariaDB. Both require a **stopped** UI so nothing writes during dump. Keep `$LOGSTASHUI_DATA_DIR` (same Django secret key) or encrypted keystore rows will not decrypt. Sessions are not copied; log in again. + +Create the empty target first ([examples](examples/)). MySQL/MariaDB: `utf8mb4` / `utf8mb4_bin`. Native installs need `LogstashUI[postgres]` or `LogstashUI[mysql]`. The container image already has both drivers. + +BETA `migrate-engine` is **not atomic** on the target: `dumpdata` → `migrate` → `loaddata` is three steps. If `loaddata` fails, drop or recreate the target and re-run. SQLite is only WAL-checkpointed. + +--- + +## Offline dump / load (supported) + +Do this while `LOGSTASHUI_DB_ENGINE` is still sqlite (or unset). If you point env at the empty server first, `dumpdata` dumps the empty server. + +1. Stop the UI. + - systemd: `sudo systemctl stop logstashui` (avoid `Restart=` racing SIGTERM). + - Docker: stop the container (`docker compose stop logstashui`). + - Kubernetes: `kubectl -n logstashui scale statefulset/logstashui --replicas=0` and wait for the pod to disappear. +2. Copy `$LOGSTASHUI_DATA_DIR/db.sqlite3` (and `-wal` / `-shm` if present) somewhere safe. Keep the rest of `DATA_DIR`. +3. Dump from SQLite: + + ```bash + logstashui manage dumpdata --natural-foreign --natural-primary \ + -e contenttypes -e auth.permission -e sessions \ + -o dump.json + ``` + + In a container: `docker exec -it logstashui manage dumpdata ...` **before** changing env, or exec with `LOGSTASHUI_DB_ENGINE=sqlite` and `LOGSTASHUI_DB_NAME` pointing at the sqlite file. + +4. Create the server database (`utf8mb4_bin` on MySQL/MariaDB). +5. Set `LOGSTASHUI_DB_*` for the target (`ENGINE`, `HOST`, `PORT`, `NAME`, `USER`, `PASSWORD`, plus `SSLMODE` / `SSL_CA` as needed). +6. Schema + load: + + ```bash + logstashui manage migrate --noinput + logstashui manage loaddata dump.json + ``` + +7. Postgres sequences: this path does **not** reset them. Use BETA `migrate-engine` if you need `sequence_reset_sql` without a `psql` client. Django will still insert; sequences can drift until reset. +8. Start LogstashUI. Log in again. + +Kubernetes after env change: patch ConfigMap/Secret, then `scale --replicas=1` (or delete the pod). First start runs `migrate` again (no-op if already applied) — **loaddata is not automatic**. Run loaddata from a one-shot `kubectl exec` after migrate, or run steps 6–7 from a job with the same env and a copy of `dump.json`. + +--- + +## BETA CLI + +```bash +# Target LOGSTASHUI_DB_* already set; sqlite file still in DATA_DIR +sudo systemctl stop logstashui +logstashui migrate-engine --to postgresql --i-have-a-backup +# --to mysql | --to mariadb (mariadb is an alias of mysql) +# --write-env /etc/default/logstashui # upserts engine/host/name/user; never password +sudo systemctl start logstashui +``` + +`--i-have-a-backup` is required. + +The command: + +1. SIGTERMs gunicorn if `$LOGSTASHUI_DATA_DIR/gunicorn.pid` is live (does not restart serve). +2. WAL checkpoint on the sqlite file in `DATA_DIR`. +3. `dumpdata` from that sqlite file (ignores target `LOGSTASHUI_DB_*` for the dump). +4. `migrate --noinput` on the target, then `loaddata`. +5. PostgreSQL: `sequence_reset_sql` through the Django connection (no `psql`). + +Kubernetes: scale to 0, `kubectl exec` is the wrong place if the pod is gone. Run `migrate-engine` from a debug pod / CI job that mounts the same PVC (`DATA_DIR`) and has target `LOGSTASHUI_DB_*`. Then scale to 1. + +If `loaddata` fails: drop or recreate the **target** database and re-run. Do not restore SQLite unless you also lost the backup. + +--- + +## What is not copied + +`contenttypes`, `auth.permission`, and `sessions` are excluded from dump/load. Permissions recreate on migrate. Sessions mean everyone logs in again. diff --git a/docs/docs/logstashui/general/build.md b/docs/docs/logstashui/general/build.md index ee408147..153ae16a 100644 --- a/docs/docs/logstashui/general/build.md +++ b/docs/docs/logstashui/general/build.md @@ -69,6 +69,31 @@ uv run logstashui --- +## Contributing Setup (Pre-commit Hooks) + +After your first `uv sync`, install the git pre-commit hooks once: + +```bash +uv run pre-commit install +``` + +The hooks run automatically on every `git commit` and handle two things: + +- **License headers** — adds the Elastic license notice to any source file that doesn't have one yet +- **Dependency notices** — regenerates `NOTICE.txt` to reflect any new third-party packages + +### Windows developers + +Set git's line-ending mode to `input` **before your first commit**. This prevents git from converting LF to CRLF on checkout, which fights the repo's `.gitattributes` settings and produces phantom "modified" files: + +```powershell +git config --global core.autocrlf input +``` + +You only need to run this once — it applies to all your repos globally. + +--- + ## Building an sdist and wheel Compile Tailwind first (the `dist/` CSS path is not `node_modules`; repo-root `/dist/` is the packaging output): @@ -78,12 +103,14 @@ cd src/logstashui/theme/static_src npm install && npm run build cd ../../../.. uv build -# dist/logstashui-0.5.1.tar.gz -# dist/logstashui-0.5.1-py3-none-any.whl +# dist/logstashui-0.5.2.tar.gz +# dist/logstashui-0.5.2-py3-none-any.whl ``` The wheel includes systemd templates (`LogstashUI/packaging/`) and the `logstashui` console script. +Air-gapped hosts that cannot reach PyPI or a registry: optional `bin/freeze_logstashui.sh` (not default packaging). See [Air-gapped freeze](/docs/docs/logstashui/general/offline.md). + --- ## Building the Docker Image Locally diff --git a/docs/docs/logstashui/general/deploy.md b/docs/docs/logstashui/general/deploy.md index 219b7c5a..ffaf02ae 100644 --- a/docs/docs/logstashui/general/deploy.md +++ b/docs/docs/logstashui/general/deploy.md @@ -9,6 +9,8 @@ The ways to deploy LogstashUI, from the standard Docker install to running from - [Option 2: Host-backed Simulation](#option-2-host-backed-simulation) - [Option 3: pip / uv + systemd](#option-3-pip--uv--systemd) - [Option 4: Source Development Setup](#option-4-source-development-setup) +- [Option 5: Kubernetes](#option-5-kubernetes) +- [Option 6: Air-gapped freeze (optional)](#option-6-air-gapped-freeze-optional) --- @@ -33,11 +35,11 @@ Then browse to `https://:8443`. This is **embedded mode** — two containers: **LogstashUI** (gunicorn HTTPS on **8443**) and **LogstashAgent** (uvicorn HTTPS on **9500**). There is **no nginx**. Configuration is [environment variables](/docs/docs/logstashui/configuration/environment.md). -**Data directory:** Runtime state (sqlite, TLS, secrets, logs) lives **outside** `src/`. From a git checkout, Docker Compose bind-mounts `/logstashui_data` to `/var/lib/logstashui` and sets `LOGSTASHUI_DATA_DIR`. Native CLI default (no env) is `$(pwd)/logstashui_data`. Do not store this under `src/logstashui/data`. +**Data directory:** Runtime state (sqlite, TLS, secrets, logs) lives **outside** `src/`. From a git checkout, Docker Compose bind-mounts `/logstashui_data` to `/var/lib/logstashui` and sets `LOGSTASHUI_DATA_DIR`. Native CLI default (no env) is `$(pwd)/logstashui_data`. Do not store this under `src/logstashui/data`. The database may be external Postgres/MySQL; the PVC/bind-mount is still required for TLS and secrets. On **Linux Docker**, that bind-mount keeps host file ownership. The image entrypoint starts as root, chowns **only** the data directory if it is not writable, then drops to `PUID`/`PGID` (from `start_logstashui.sh`: your uid/gid) or image user **appuser (10001)**. Gunicorn never stays root. Docker Desktop (macOS/Windows) usually maps UIDs already; the same path is a no-op chown. If the directory is still unwritable, startup exits before migrate so sqlite/TLS are not created as the wrong user. -**Kubernetes:** use a PVC at `/var/lib/logstashui`, `runAsUser: 10001`, `runAsNonRoot: true`, and `fsGroup: 10001`. The entrypoint skips chown when it is not root. +**Kubernetes:** see [Option 5](#option-5-kubernetes) and the [Kubernetes subsection](/docs/docs/logstashui/kubernetes/index.md). PVC at `/var/lib/logstashui`, `runAsUser: 10001`, `runAsNonRoot: true`, `fsGroup: 10001`. Keep `LOGSTASHUI_TLS` on; the Ingress/HTTPRoute originates HTTPS to `:8443` and skips backend cert verify. **HTTPS / product CA:** On first start, LogstashUI writes a product CA and a UI server certificate under `$LOGSTASHUI_DATA_DIR/tls/` (`ui-server.crt` / `ui-server.key`). Gunicorn presents that cert on port **8443** (ports under 1000 would need root). The product leaf SANs include `localhost`, `logstashui`, **all non-loopback host IPs**, and **PTR reverse-DNS FQDNs** for those IPs when available (injected by `start_logstashui.sh` as `LOGSTASHUI_HOST_*` / `LOGSTASHUI_TLS_SANS`, because the container cannot see the host LAN addresses by itself). Bare short hostnames (common on macOS) are replaced by reverse-lookup FQDNs when PTR records exist. Changing the Agent callback URL or those env SANs **re-issues** the product leaf on next startup (or Settings save); restart the UI container so gunicorn reloads the file. Agents: @@ -64,7 +66,7 @@ For frequent or heavy simulation, enroll one or more **Simulate** policy agents ## Option 3: pip / uv + systemd ```bash -pip install logstashui-0.5.1-py3-none-any.whl # or: uv pip install … +pip install logstashui-0.5.2-py3-none-any.whl # or: uv pip install … logstashui # HTTPS :8443, data in $(pwd)/logstashui_data sudo logstashui systemd # writes /etc/default/logstashui + unit; does not enable sudo systemctl enable --now logstashui @@ -72,7 +74,7 @@ sudo systemctl enable --now logstashui Set `LOGSTASHUI_DATA_DIR=/var/lib/logstashui` in `/etc/default/logstashui` (the generator does this). See [environment configuration](/docs/docs/logstashui/configuration/environment.md). -Kubernetes: same image as Option 1, env/ConfigMap only, PVC at `/var/lib/logstashui`. No YAML mount. +Kubernetes: [Option 5](#option-5-kubernetes). Same image as Option 1; env/ConfigMap plus Secret; PVC at `/var/lib/logstashui`. Keep TLS on. No YAML mount. --- @@ -86,9 +88,32 @@ Use this when you want to run LogstashUI directly from source for development or --- +## Option 5: Kubernetes + +One-replica StatefulSet, PVC at `/var/lib/logstashui`, image `codyjackson032/logstashui:latest` (or a tag you built). Gunicorn keeps HTTPS on **8443** (`LOGSTASHUI_TLS` stays true). Ingress-nginx or Envoy Gateway originates HTTPS to the pod and skips verification of the product self-signed leaf. + +- Guide: [Kubernetes](/docs/docs/logstashui/kubernetes/index.md) +- Manifests: [examples](/docs/docs/logstashui/kubernetes/examples/README.md) (SQLite, PostgreSQL, MySQL/MariaDB) +- CloudNativePG: [cnpg.md](/docs/docs/logstashui/kubernetes/cnpg.md) +- Envoy Gateway Backend API: [envoy-gateway.md](/docs/docs/logstashui/kubernetes/envoy-gateway.md) +- Database env and migration: [Database](/docs/docs/logstashui/database/index.md) + +--- + +## Option 6: Air-gapped freeze (optional) + +Not the default packaging path. When the install host has **no PyPI and no registry**, a connected maintainer runs `bin/freeze_logstashui.sh` and copies a zip. Isolated host: CPython **3.12** x86_64 (wheels), Docker (image zip), or experimental PyInstaller (Linux x86_64). `[databases]` and `[otel]` are included; LogstashAgent is not. Tracing stays off until `LOGSTASHUI_OTEL=true`. + +**📖 Full instructions: [Air-gapped freeze](/docs/docs/logstashui/general/offline.md)** + +--- + ## Related Documentation - **[Getting Started](/docs/docs/getting_started.md)** - Step-by-step standard install +- **[Kubernetes](/docs/docs/logstashui/kubernetes/index.md)** - StatefulSet, PVC, Ingress, Envoy Gateway, CNPG +- **[Database](/docs/docs/logstashui/database/index.md)** - SQLite, PostgreSQL, MySQL/MariaDB +- **[Air-gapped freeze](/docs/docs/logstashui/general/offline.md)** - Optional offline zip (wheels / Docker / experimental standalone) - **[Building LogstashUI from Source](/docs/docs/logstashui/general/build.md)** - Source builds and local development - **[Host Mode Setup](/docs/docs/logstashui/configuration/host_mode.md)** - High-performance simulation setup - **[Updating LogstashUI](/docs/docs/logstashui/general/updating.md)** - Keeping your deployment current diff --git a/docs/docs/logstashui/general/index.md b/docs/docs/logstashui/general/index.md index e227fc2e..48746ba6 100644 --- a/docs/docs/logstashui/general/index.md +++ b/docs/docs/logstashui/general/index.md @@ -11,12 +11,23 @@ All the ways to deploy LogstashUI. **Covers:** - Standard Docker deployment (recommended) - Host-backed simulation +- pip / uv + systemd +- Kubernetes (see also the [Kubernetes](/docs/docs/logstashui/kubernetes/index.md) subsection) +- Air-gapped freeze (optional offline zips) - Source development setup **📖 [View deployment guide →](/docs/docs/logstashui/general/deploy.md)** --- +## **[Air-gapped freeze](/docs/docs/logstashui/general/offline.md)** + +Optional connected-builder script that freezes LogstashUI plus `[databases]` and `[otel]` into zips for hosts with no PyPI or registry. Default `uv build` is unchanged. + +**📖 [View air-gapped freeze →](/docs/docs/logstashui/general/offline.md)** + +--- + ## **[Building from Source](/docs/docs/logstashui/general/build.md)** Instructions for building and running LogstashUI from source. diff --git a/docs/docs/logstashui/general/offline.md b/docs/docs/logstashui/general/offline.md new file mode 100644 index 00000000..e1b5b629 --- /dev/null +++ b/docs/docs/logstashui/general/offline.md @@ -0,0 +1,57 @@ +# Air-gapped freeze (optional) + +Build zip files on a **connected** Linux x86_64 machine, copy them to a host with **no PyPI and no container registry**, and run LogstashUI. + +This is **not** the default packaging path and **not** the recommended install when the network can reach GitHub or a registry. Prefer [Option 1: Docker](deploy.md#option-1-standard-docker-deployment-recommended) or [Option 3: pip / uv](deploy.md#option-3-pip--uv--systemd). Default `uv build` (sdist + `py3-none-any` wheel) is unchanged. + +## What you get + +`bin/freeze_logstashui.sh` emits up to three zips under `dist/offline/` (gitignored): + +| Zip | Isolated host needs | Run | +|---|---|---| +| `logstashui-*-offline-wheels-linux-x86_64-cp312.zip` | CPython **3.12** x86_64 + venv | `./install.sh` then `.venv/bin/logstashui serve` | +| `logstashui-*-offline-docker-linux-x86_64.zip` | Docker Engine | `./load.sh` then `docker compose -f compose.offline.yml up -d` | +| `logstashui-*-offline-standalone-linux-x86_64.zip` | glibc Linux x86_64 | `./run.sh` (**experimental**) | + +All three include `LogstashUI[databases]` (psycopg + PyMySQL) and `LogstashUI[otel]` (inert until `LOGSTASHUI_OTEL=true`). SQLite remains the runtime default. **LogstashAgent is not bundled.** arm64 and Windows are later freeze invocations, not this zip. + +## Builder (connected) + +Requirements: Linux x86_64 (wheels can also be downloaded from another OS via pip's `--platform`), CPython 3.12 (`uv python install 3.12`), [uv](https://docs.astral.sh/uv/), Docker for `--docker`, and (on Linux x86_64 only) a throwaway venv for PyInstaller. + +```bash +./bin/freeze_logstashui.sh --wheels +./bin/freeze_logstashui.sh --docker +./bin/freeze_logstashui.sh --standalone # Linux x86_64 only +./bin/freeze_logstashui.sh --all # default if you pass no artifact flags +./bin/freeze_logstashui.sh --docker --image logstashui:offline-0.5.2 +``` + +`--image` saves a **local** tag. The script never `docker pull`. `--standalone` on macOS/Windows/ARM **fails** if you passed that flag; `--all` **skips** it with a warning. + +Wheel policy: **zip contains only `.whl` files**. The builder prefers `manylinux2014` then `manylinux_2_28` cp312 wheels (isolated host needs glibc **2.28+**, e.g. RHEL 8 / Ubuntu 20.04). Pure-Python sdists (no manylinux wheel) are converted to `py3-none-any` on the **connected** builder. A native package with no manylinux wheel fails the freeze — the isolated host has no compiler. Pins come from `uv.lock`. + +Smoke the wheelhouse (pulls `python:3.12-slim` first, then installs with `--network=none`): + +```bash +./bin/test_freeze_wheels.sh +``` + +Optional CI: `.github/workflows/offline-freeze.yml` is `workflow_dispatch` only (not a required PR check). + +## Isolated host + +Same env as a normal install: `LOGSTASHUI_DATA_DIR` (default `$(pwd)/logstashui_data`), `LOGSTASHUI_*`, `LOGSTASHUI_DB_*`. HTTPS on **:8443**. Product CA is created on first start under the data dir — do not expect CA files inside the zip. + +**Wheels:** Debian/Ubuntu need `python3.12` and `python3.12-venv`. `install.sh` uses `pip install --no-index --find-links ./wheels 'LogstashUI[databases,otel]'`. It does **not** upgrade pip (that would hit PyPI). uv is not required. + +**Docker:** UI-only compose (no Agent, no `embedded` profile). Set `ALLOWED_HOSTS` / `LOGSTASHUI_HOST_*` / `LOGSTASHUI_DB_*` as needed. After a local image build, optional extra check: `IMAGE= bin/test_docker_otel.sh` (imports the OTEL packages; does not start serve). Standalone has no automated smoke. + +**Standalone:** experimental PyInstaller onedir. Treat as a trial until `serve` completes migrate, SNMP official sync, collectstatic, and HTTPS :8443 with no network. Gunicorn stays gevent; do not switch workers as a workaround. No automated smoke. Run on a Linux x86_64 builder if you ship this zip. + +After a wheelhouse install, `logstashui systemd` still writes the unit and `/etc/default/logstashui` and does **not** enable it. + +## Later + +linux/arm64 and Windows x86_64 as extra freeze tags; a sibling LogstashAgent freeze; promoting standalone off experimental after the serve smoke exists. diff --git a/docs/docs/logstashui/general/updating.md b/docs/docs/logstashui/general/updating.md index 1a6be761..e8808484 100644 --- a/docs/docs/logstashui/general/updating.md +++ b/docs/docs/logstashui/general/updating.md @@ -23,14 +23,14 @@ start_logstashui.bat --update ## LogstashAgent pairing -When LogstashUI is updated, upgrade enrolled agents to the **preferred agent version** shown in the UI (banner / Settings). For **0.5.1**: +When LogstashUI is updated, upgrade enrolled agents to the **preferred agent version** shown in the UI (banner / Settings). For **0.5.2**: 1. Install the matching LogstashAgent package on each host. 2. Restart the agent unit for that role (`logstash-agent`, `logstash-agent@N`, or `lsagent-simulate@N`). 3. **Production Packaged/Default agents do not need to re-enroll.** 4. Apply DB migrations (compose entrypoint runs them; bare metal: `python manage.py migrate`). -See [CHANGELOG 0.5.1](https://github.com/elastic/LogstashUI/blob/main/CHANGELOG.md) and [agent roles](/docs/docs/logstashagent/general/roles.md) for Packaged/Managed coexistence and VERSION notes. +See [CHANGELOG](https://github.com/elastic/LogstashUI/blob/main/CHANGELOG.md) and [agent roles](/docs/docs/logstashagent/general/roles.md) for Packaged/Managed coexistence and VERSION notes. ### Smoke after upgrade (optional) diff --git a/docs/docs/logstashui/index.md b/docs/docs/logstashui/index.md index a3d8df3e..d5502645 100644 --- a/docs/docs/logstashui/index.md +++ b/docs/docs/logstashui/index.md @@ -39,9 +39,12 @@ Configure polling, traps, and discovery through a web interface. ## Documentation +- **[API Access](/docs/docs/logstashui/api_access.md)** - Scripting LogstashUI with API tokens and curl - **[Architecture](/docs/docs/logstashui/architecture.md)** - System architecture - **[Compatibility](/docs/docs/logstashui/compatibility.md)** - Logstash version compatibility and requirements - **[Configuration](/docs/docs/logstashui/configuration/index.md)** - Configuration options and settings for LogstashUI +- **[Database](/docs/docs/logstashui/database/index.md)** - SQLite, PostgreSQL, MySQL/MariaDB, and migration +- **[Kubernetes](/docs/docs/logstashui/kubernetes/index.md)** - StatefulSet, PVC, Ingress, Envoy Gateway, CloudNativePG - **[SNMP Monitoring](/docs/docs/logstashui/SNMP/index.md)** - Network monitoring with SNMP polling, traps, and discovery - **[General](/docs/docs/logstashui/general/index.md)** - Build, update, and deployment guides diff --git a/docs/docs/logstashui/kubernetes/cnpg.md b/docs/docs/logstashui/kubernetes/cnpg.md new file mode 100644 index 00000000..cebf61ff --- /dev/null +++ b/docs/docs/logstashui/kubernetes/cnpg.md @@ -0,0 +1,98 @@ +# CloudNativePG + +Use this when a [CloudNativePG](https://cloudnative-pg.io/) operator is already installed (commonly in namespace `cnpg-system`). Create the Postgres **Cluster in the LogstashUI namespace**. LogstashUI talks to the Cluster's read-write Service; it does not talk to the operator. + +PostgreSQL **14+** is required (16 is a good default). The container image already has `psycopg`. + +A sample Cluster is in [examples/postgresql/cnpg.yaml](examples/postgresql/cnpg.yaml). + +--- + +## Cluster + +```yaml +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: logstashui-pg + namespace: logstashui +spec: + instances: 1 # 3 for HA + imageName: ghcr.io/cloudnative-pg/postgresql:16 + bootstrap: + initdb: + database: logstashui + owner: logstashui + encoding: UTF8 + storage: + size: 10Gi +``` + +Do **not** put Django `CREATE TABLE` in `postInitApplicationSQL`. `logstashui serve` runs `migrate` on first start. + +Wait until Ready: + +```bash +kubectl -n logstashui wait --for=condition=Ready cluster/logstashui-pg --timeout=300s +``` + +--- + +## Connection (what CNPG creates) + +| Object | Purpose | +|---|---| +| Service `logstashui-pg-rw` | Primary, port 5432 | +| Secret `logstashui-pg-app` | `host`, `port`, `dbname`, `username`, `password`, `uri` | +| Secret `logstashui-pg-ca` | Cluster CA (`ca.crt`) if you want `verify-full` | + +Wire LogstashUI from that Secret (do not copy the password into your own Secret): + +```yaml +env: + - name: LOGSTASHUI_DB_ENGINE + value: postgresql + - name: LOGSTASHUI_DB_SSLMODE + value: require + - name: LOGSTASHUI_DB_HOST + valueFrom: + secretKeyRef: + name: logstashui-pg-app + key: host + - name: LOGSTASHUI_DB_PORT + valueFrom: + secretKeyRef: + name: logstashui-pg-app + key: port + - name: LOGSTASHUI_DB_NAME + valueFrom: + secretKeyRef: + name: logstashui-pg-app + key: dbname + - name: LOGSTASHUI_DB_USER + valueFrom: + secretKeyRef: + name: logstashui-pg-app + key: username + - name: LOGSTASHUI_DB_PASSWORD + valueFrom: + secretKeyRef: + name: logstashui-pg-app + key: password +``` + +`require` is enough for CNPG's server TLS. For `verify-full`, mount `logstashui-pg-ca` `ca.crt` and set `LOGSTASHUI_DB_SSL_CA` to that path. + +Keep `LOGSTASHUI_DATA_DIR` on a PVC (TLS and secrets). An init container can `pg_isready -h logstashui-pg-rw` before the UI starts. + +--- + +## Existing Cluster in another namespace + +Point `LOGSTASHUI_DB_HOST` at that Cluster's `-rw` Service FQDN (`-rw..svc.cluster.local`). Copy or ExternalSecret the app password into `logstashui`. NetworkPolicy must allow 5432 from the UI pods. + +--- + +## SQLite → CNPG + +Create the Cluster and empty database, set `LOGSTASHUI_DB_*`, then follow [migration](/docs/docs/logstashui/database/migration.md). Do not `loaddata` until `migrate` has created tables. diff --git a/docs/docs/logstashui/kubernetes/envoy-gateway.md b/docs/docs/logstashui/kubernetes/envoy-gateway.md new file mode 100644 index 00000000..5e874040 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/envoy-gateway.md @@ -0,0 +1,112 @@ +# Envoy Gateway + +Keep LogstashUI on HTTPS `:8443`. The HTTPRoute must send **HTTPS** to the pod and **skip verification** of the product self-signed leaf. + +Envoy Gateway documents that skip on the **`Backend` CR**, not on `BackendTrafficPolicy`. `BackendTrafficPolicy` is retries, timeouts, and circuit breaking. `tls.insecureSkipVerify` exists only on `Backend`. + +The Backend API is **disabled by default**. Enable it, then apply a `Backend` + `HTTPRoute`. + +--- + +## 1. Enable the Backend API + +Helm values: + +```yaml +config: + envoyGateway: + extensionApis: + enableBackend: true +``` + +Upgrade: + +```bash +helm upgrade eg oci://docker.io/envoyproxy/gateway-helm \ + -n envoy-gateway-system \ + --reuse-values \ + --set config.envoyGateway.extensionApis.enableBackend=true +``` + +If you manage an `EnvoyGateway` config ConfigMap instead of Helm values, merge: + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +extensionApis: + enableBackend: true +``` + +Restart the Envoy Gateway controller after the change. Confirm the `Backend` CRD is served: + +```bash +kubectl api-resources | grep -i backend +``` + +--- + +## 2. HTTPRoute + Backend + +The HTTPRoute `backendRefs` must use `group: gateway.envoyproxy.io` and `kind: Backend`. Point the `Backend` at the LogstashUI Service FQDN on port **8443**. + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: logstashui + namespace: logstashui +spec: + endpoints: + - fqdn: + hostname: logstashui.logstashui.svc.cluster.local + port: 8443 + tls: + insecureSkipVerify: true +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: logstashui + namespace: logstashui +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: public # your Gateway + namespace: envoygw # Gateway namespace + hostnames: + - logstashui.example.com + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: logstashui + matches: + - path: + type: PathPrefix + value: / +``` + +Skip-verify is for **testing / product-CA backends**. Prefer a `BackendTLSPolicy` with the product CA in production. + +--- + +## 3. Gateway `allowedRoutes` + +If the Gateway is in another namespace, its listener must allow HTTPRoutes from `logstashui`: + +```yaml +allowedRoutes: + namespaces: + from: All +``` + +or a label selector that includes the `logstashui` namespace. + +--- + +## 4. LogstashUI env + +Do **not** set `LOGSTASHUI_TLS=false`. Set `CSRF_TRUSTED_ORIGINS=https://` and `ALLOWED_HOSTS` to the same host. See [Kubernetes](index.md). diff --git a/docs/docs/logstashui/kubernetes/examples/README.md b/docs/docs/logstashui/kubernetes/examples/README.md new file mode 100644 index 00000000..a226707b --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/README.md @@ -0,0 +1,42 @@ +# Example manifests + +Apply **one** tree. All of them keep LogstashUI TLS on (`:8443`) and use Ingress-nginx with `backend-protocol: HTTPS` and `proxy-ssl-verify: "off"`. Image: `codyjackson032/logstashui:latest`. Namespace: `logstashui`. Replicas: **1**. One PVC at `/var/lib/logstashui`. + +| Directory | Database | +|---|---| +| [sqlite/](sqlite/) | SQLite file on the data PVC. Simplest. Not for concurrent agents. | +| [postgresql/](postgresql/) | External PostgreSQL 14+. Optional [cnpg.yaml](postgresql/cnpg.yaml) if you run CloudNativePG. | +| [mysql/](mysql/) | MariaDB 10.6+ or MySQL 8.0+. Engine is `mysql`. Create the schema as `utf8mb4` / `utf8mb4_bin`. | + +Replace `logstashui.example.com` and `SECRET_KEY` before apply. For Postgres/MySQL, create the empty database first — [SQL examples](/docs/docs/logstashui/database/examples/). + +Envoy Gateway instead of Ingress: [envoy-gateway.md](../envoy-gateway.md) (enable the Backend API, then `Backend` + `HTTPRoute`). + +```bash +kubectl apply -f docs/docs/logstashui/kubernetes/examples/sqlite/ +``` + +## Embedded agent (optional) + +Same role as compose `--profile embedded`: an in-cluster LogstashAgent for the pipeline editor Sim target (`embedded · docker`). No enroll. Lab only — prefer an enrolled Simulate agent for serious work. + +1. Apply **one** DB tree above. +2. Set `LOGSTASHUI_AGENT_CSR_SECRET` in its Secret. DO NOT use the secret default. +3. Apply the overlay: + +```bash +kubectl apply -f docs/docs/logstashui/kubernetes/examples/embedded-agent.yaml +``` + +`Service/logstashagent` is ClusterIP **9500** (agent API), **9560** (Logstash API), **9449** (HTTP input). Nothing is published outside the cluster. Image: `codyjackson032/logstash-agent:latest`. + +If the CSR secret key is still commented, the agent pod is `CreateContainerConfigError` until you uncomment it. + +Agent ConfigMap `logstashagent` already sets `LOGSTASH_UI_URL` / `LOGSTASH_URL` to `https://logstashui:8443`. Two extra keys are **commented** (LogstashAgent env, not UI): + +| Env | Default | Do not enable without cause | +|---|---|---| +| `LOGSTASH_AGENT_TLS` | `true` | Uncomment `"false"` only with `LOGSTASHUI_INSECURE_HTTP` and `http://` URLs. Serves the agent API over HTTP. | +| `LOGSTASH_UI_TLS_INSECURE` | `false` | Uncomment `"true"` to skip verifying the UI HTTPS cert. Not the same as plain HTTP. Product CA pinning works out of the box. | + +See the LogstashAgent README TLS table. Standard examples stay HTTPS. diff --git a/docs/docs/logstashui/kubernetes/examples/embedded-agent.yaml b/docs/docs/logstashui/kubernetes/examples/embedded-agent.yaml new file mode 100644 index 00000000..f2ba14ce --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/embedded-agent.yaml @@ -0,0 +1,117 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Optional embedded simulation agent (compose --profile embedded analog). +# Apply a DB tree first (sqlite/ postgresql/ or mysql/), uncomment +# LOGSTASH_AGENT_URL in ConfigMap/logstashui and LOGSTASHUI_AGENT_CSR_SECRET +# in Secret/logstashui, then: +# kubectl apply -f docs/docs/logstashui/kubernetes/examples/embedded-agent.yaml +# ClusterIP only. Lab node. Prefer enrolled Simulate agents for serious work. +# Missing CSR secret → CreateContainerConfigError (intentional). +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstashagent + namespace: logstashui +data: + LOGSTASH_UI_URL: https://logstashui:8443 + LOGSTASH_URL: https://logstashui:8443 + # + # Agent API TLS (default true). Automatic product TLS works out of the box. + # Uncomment false ONLY if the UI is also plain HTTP (LOGSTASHUI_INSECURE_HTTP) + # and you have rewritten LOGSTASH_UI_URL / LOGSTASH_URL / LOGSTASH_AGENT_URL + # to http://. Do not enable without cause. + # LOGSTASH_AGENT_TLS: "false" + # + # Skip verifying the UI HTTPS certificate (default false). + # Not the same as plain HTTP. Product CA pinning works OOTB; leave commented. + # LOGSTASH_UI_TLS_INSECURE: "true" +--- +apiVersion: v1 +kind: Service +metadata: + name: logstashagent + namespace: logstashui + labels: + app: logstashagent +spec: + selector: + app: logstashagent + ports: + - name: agent + port: 9500 + targetPort: agent + - name: logstash-api + port: 9560 + targetPort: logstash-api + - name: http-input + port: 9449 + targetPort: http-input +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: logstashagent + namespace: logstashui +spec: + replicas: 1 + selector: + matchLabels: + app: logstashagent + template: + metadata: + labels: + app: logstashagent + spec: + initContainers: + - name: wait-ui-ca + image: codyjackson032/logstashui:latest + imagePullPolicy: IfNotPresent + command: + - python + - -c + - | + import ssl, time, urllib.request + ctx = ssl._create_unverified_context() + url = "https://logstashui:8443/.well-known/logstashui/ca.crt" + while True: + try: + urllib.request.urlopen(url, context=ctx, timeout=5) + break + except Exception: + time.sleep(2) + containers: + - name: logstashagent + image: codyjackson032/logstash-agent:latest + imagePullPolicy: IfNotPresent + ports: + - name: agent + containerPort: 9500 + - name: logstash-api + containerPort: 9560 + - name: http-input + containerPort: 9449 + envFrom: + - configMapRef: + name: logstashagent + env: + - name: LOGSTASHUI_AGENT_CSR_SECRET + valueFrom: + secretKeyRef: + name: logstashui + key: LOGSTASHUI_AGENT_CSR_SECRET + startupProbe: + tcpSocket: + port: agent + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + tcpSocket: + port: agent + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: agent + periodSeconds: 30 + failureThreshold: 6 diff --git a/docs/docs/logstashui/kubernetes/examples/mysql/configmap.yaml b/docs/docs/logstashui/kubernetes/examples/mysql/configmap.yaml new file mode 100644 index 00000000..9e8f92a4 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/mysql/configmap.yaml @@ -0,0 +1,41 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstashui + namespace: logstashui +data: + LOGSTASHUI_DATA_DIR: /var/lib/logstashui + LOGSTASHUI_BIND: 0.0.0.0:8443 + LOGSTASHUI_TLS: "true" + LOGSTASHUI_WORKERS: "2" + DEBUG: "false" + ALLOWED_HOSTS: logstashui.example.com,logstashui,logstashui.logstashui.svc,logstashui.logstashui.svc.cluster.local + CSRF_TRUSTED_ORIGINS: https://logstashui.example.com + LOGSTASHUI_TLS_SANS: logstashui.example.com,logstashui.logstashui.svc.cluster.local + LOGSTASHUI_HOST_HOSTNAME: logstashui.example.com + LOGSTASHUI_AGENT_UI_URL: https://logstashui.example.com + # MariaDB uses the same engine. Create the database as utf8mb4 / utf8mb4_bin. + LOGSTASHUI_DB_ENGINE: mysql + LOGSTASHUI_DB_HOST: mysql.example.svc.cluster.local + LOGSTASHUI_DB_PORT: "3306" + LOGSTASHUI_DB_NAME: logstashui + LOGSTASHUI_DB_USER: logstashui + # Optional CA file path if the server requires TLS: + # LOGSTASHUI_DB_SSL_CA: /etc/ssl/certs/mysql-ca.pem + # + # NOT recommended. Automatic product TLS works out of the box. + # If you must run the UI and all agent connections over plain HTTP + # (no CA, no certs), uncomment. Do not enable without cause. + # LOGSTASHUI_INSECURE_HTTP: "true" + # + # OpenTelemetry (optional). The image already installs LogstashUI[otel]. + # Uncomment to export traces/metrics over OTLP/HTTP (not gRPC — gevent + # cannot patch grpcio). Point at a collector that accepts HTTP/protobuf + # on 4318. Leave commented to keep tracing off. + # LOGSTASHUI_OTEL: "true" + # OTEL_SERVICE_NAME: logstashui + # OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector.observability.svc:4318 diff --git a/docs/docs/logstashui/kubernetes/examples/mysql/ingress.yaml b/docs/docs/logstashui/kubernetes/examples/mysql/ingress.yaml new file mode 100644 index 00000000..a1bae431 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/mysql/ingress.yaml @@ -0,0 +1,31 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Ingress-nginx: originate HTTPS to the pod and skip verify of the product leaf. +# Do not set LOGSTASHUI_TLS=false on the container. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: logstashui + namespace: logstashui + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + nginx.ingress.kubernetes.io/proxy-ssl-verify: "off" +spec: + ingressClassName: nginx + # tls: + # - hosts: + # - logstashui.example.com + # secretName: logstashui-tls + rules: + - host: logstashui.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: logstashui + port: + number: 8443 diff --git a/src/logstashui/AI/tests.py b/docs/docs/logstashui/kubernetes/examples/mysql/namespace.yaml similarity index 79% rename from src/logstashui/AI/tests.py rename to docs/docs/logstashui/kubernetes/examples/mysql/namespace.yaml index 9b26a3af..d2f1dc8a 100644 --- a/src/logstashui/AI/tests.py +++ b/docs/docs/logstashui/kubernetes/examples/mysql/namespace.yaml @@ -2,6 +2,7 @@ #or more contributor license agreements. Licensed under the Elastic License; #you may not use this file except in compliance with the Elastic License. -from django.test import TestCase - -# Create your tests here. +apiVersion: v1 +kind: Namespace +metadata: + name: logstashui diff --git a/docs/docs/logstashui/kubernetes/examples/mysql/secret.yaml b/docs/docs/logstashui/kubernetes/examples/mysql/secret.yaml new file mode 100644 index 00000000..defc77c7 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/mysql/secret.yaml @@ -0,0 +1,16 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: Secret +metadata: + name: logstashui + namespace: logstashui +type: Opaque +stringData: + SECRET_KEY: CHANGE-ME-generate-a-django-secret-key + LOGSTASHUI_DB_PASSWORD: CHANGE-ME + # Shared with Deployment/logstashagent so the embedded node can CSR without enroll. + # Must match what the agent reads. Do not use the default in a real cluster. + LOGSTASHUI_AGENT_CSR_SECRET: CHANGE-ME-shared-with-the-agent diff --git a/docs/docs/logstashui/kubernetes/examples/mysql/statefulset.yaml b/docs/docs/logstashui/kubernetes/examples/mysql/statefulset.yaml new file mode 100644 index 00000000..32bed1bc --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/mysql/statefulset.yaml @@ -0,0 +1,106 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: Service +metadata: + name: logstashui + namespace: logstashui + labels: + app: logstashui +spec: + selector: + app: logstashui + ports: + - name: https + port: 8443 + targetPort: https +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: logstashui + namespace: logstashui +spec: + serviceName: logstashui + replicas: 1 + selector: + matchLabels: + app: logstashui + template: + metadata: + labels: + app: logstashui + spec: + securityContext: + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: logstashui + image: codyjackson032/logstashui:latest + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + envFrom: + - configMapRef: + name: logstashui + - secretRef: + name: logstashui + env: + - name: LOGSTASHUI_HOST_IPS + valueFrom: + fieldRef: + fieldPath: status.podIP + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /var/lib/logstashui + # Probes are HTTPS. If you uncomment LOGSTASHUI_INSECURE_HTTP in the + # ConfigMap, gunicorn speaks HTTP and these probes will fail until + # scheme is HTTP (and Ingress/HTTPRoute backend protocol too). + # Do not enable that flag without cause. Automatic TLS is supported. + startupProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 30 + failureThreshold: 6 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/docs/docs/logstashui/kubernetes/examples/postgresql/cnpg.yaml b/docs/docs/logstashui/kubernetes/examples/postgresql/cnpg.yaml new file mode 100644 index 00000000..671c5ecb --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/postgresql/cnpg.yaml @@ -0,0 +1,25 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Optional. Requires the CloudNativePG operator (often in cnpg-system). +# See ../../cnpg.md. After Ready, point LogstashUI at Secret logstashui-pg-app +# instead of the HOST/USER/PASSWORD in configmap.yaml / secret.yaml. +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: logstashui-pg + namespace: logstashui +spec: + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:16 + bootstrap: + initdb: + database: logstashui + owner: logstashui + encoding: UTF8 + storage: + size: 10Gi + postgresql: + parameters: + max_connections: "200" diff --git a/docs/docs/logstashui/kubernetes/examples/postgresql/configmap.yaml b/docs/docs/logstashui/kubernetes/examples/postgresql/configmap.yaml new file mode 100644 index 00000000..da3db7e7 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/postgresql/configmap.yaml @@ -0,0 +1,42 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstashui + namespace: logstashui +data: + LOGSTASHUI_DATA_DIR: /var/lib/logstashui + LOGSTASHUI_BIND: 0.0.0.0:8443 + LOGSTASHUI_TLS: "true" + LOGSTASHUI_WORKERS: "2" + DEBUG: "false" + ALLOWED_HOSTS: logstashui.example.com,logstashui,logstashui.logstashui.svc,logstashui.logstashui.svc.cluster.local + CSRF_TRUSTED_ORIGINS: https://logstashui.example.com + LOGSTASHUI_TLS_SANS: logstashui.example.com,logstashui.logstashui.svc.cluster.local + LOGSTASHUI_HOST_HOSTNAME: logstashui.example.com + LOGSTASHUI_AGENT_UI_URL: https://logstashui.example.com + LOGSTASHUI_DB_ENGINE: postgresql + LOGSTASHUI_DB_HOST: postgres.example.svc.cluster.local + LOGSTASHUI_DB_PORT: "5432" + LOGSTASHUI_DB_NAME: logstashui + LOGSTASHUI_DB_USER: logstashui + LOGSTASHUI_DB_SSLMODE: prefer + # PASSWORD is in secret.yaml. For CloudNativePG, prefer secretKeyRef from + # -app instead of these HOST/USER/NAME keys — see cnpg.yaml and + # ../../cnpg.md. + # + # NOT recommended. Automatic product TLS works out of the box. + # If you must run the UI and all agent connections over plain HTTP + # (no CA, no certs), uncomment. Do not enable without cause. + # LOGSTASHUI_INSECURE_HTTP: "true" + # + # OpenTelemetry (optional). The image already installs LogstashUI[otel]. + # Uncomment to export traces/metrics over OTLP/HTTP (not gRPC — gevent + # cannot patch grpcio). Point at a collector that accepts HTTP/protobuf + # on 4318. Leave commented to keep tracing off. + # LOGSTASHUI_OTEL: "true" + # OTEL_SERVICE_NAME: logstashui + # OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector.observability.svc:4318 diff --git a/docs/docs/logstashui/kubernetes/examples/postgresql/ingress.yaml b/docs/docs/logstashui/kubernetes/examples/postgresql/ingress.yaml new file mode 100644 index 00000000..a1bae431 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/postgresql/ingress.yaml @@ -0,0 +1,31 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Ingress-nginx: originate HTTPS to the pod and skip verify of the product leaf. +# Do not set LOGSTASHUI_TLS=false on the container. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: logstashui + namespace: logstashui + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + nginx.ingress.kubernetes.io/proxy-ssl-verify: "off" +spec: + ingressClassName: nginx + # tls: + # - hosts: + # - logstashui.example.com + # secretName: logstashui-tls + rules: + - host: logstashui.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: logstashui + port: + number: 8443 diff --git a/docs/docs/logstashui/kubernetes/examples/postgresql/namespace.yaml b/docs/docs/logstashui/kubernetes/examples/postgresql/namespace.yaml new file mode 100644 index 00000000..d2f1dc8a --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/postgresql/namespace.yaml @@ -0,0 +1,8 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: Namespace +metadata: + name: logstashui diff --git a/docs/docs/logstashui/kubernetes/examples/postgresql/secret.yaml b/docs/docs/logstashui/kubernetes/examples/postgresql/secret.yaml new file mode 100644 index 00000000..defc77c7 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/postgresql/secret.yaml @@ -0,0 +1,16 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: Secret +metadata: + name: logstashui + namespace: logstashui +type: Opaque +stringData: + SECRET_KEY: CHANGE-ME-generate-a-django-secret-key + LOGSTASHUI_DB_PASSWORD: CHANGE-ME + # Shared with Deployment/logstashagent so the embedded node can CSR without enroll. + # Must match what the agent reads. Do not use the default in a real cluster. + LOGSTASHUI_AGENT_CSR_SECRET: CHANGE-ME-shared-with-the-agent diff --git a/docs/docs/logstashui/kubernetes/examples/postgresql/statefulset.yaml b/docs/docs/logstashui/kubernetes/examples/postgresql/statefulset.yaml new file mode 100644 index 00000000..32bed1bc --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/postgresql/statefulset.yaml @@ -0,0 +1,106 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: Service +metadata: + name: logstashui + namespace: logstashui + labels: + app: logstashui +spec: + selector: + app: logstashui + ports: + - name: https + port: 8443 + targetPort: https +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: logstashui + namespace: logstashui +spec: + serviceName: logstashui + replicas: 1 + selector: + matchLabels: + app: logstashui + template: + metadata: + labels: + app: logstashui + spec: + securityContext: + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: logstashui + image: codyjackson032/logstashui:latest + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + envFrom: + - configMapRef: + name: logstashui + - secretRef: + name: logstashui + env: + - name: LOGSTASHUI_HOST_IPS + valueFrom: + fieldRef: + fieldPath: status.podIP + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /var/lib/logstashui + # Probes are HTTPS. If you uncomment LOGSTASHUI_INSECURE_HTTP in the + # ConfigMap, gunicorn speaks HTTP and these probes will fail until + # scheme is HTTP (and Ingress/HTTPRoute backend protocol too). + # Do not enable that flag without cause. Automatic TLS is supported. + startupProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 30 + failureThreshold: 6 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/docs/docs/logstashui/kubernetes/examples/sqlite/configmap.yaml b/docs/docs/logstashui/kubernetes/examples/sqlite/configmap.yaml new file mode 100644 index 00000000..736a9230 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/sqlite/configmap.yaml @@ -0,0 +1,35 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstashui + namespace: logstashui +data: + LOGSTASHUI_DATA_DIR: /var/lib/logstashui + LOGSTASHUI_BIND: 0.0.0.0:8443 + LOGSTASHUI_TLS: "true" + LOGSTASHUI_WORKERS: "2" + DEBUG: "false" + ALLOWED_HOSTS: logstashui.example.com,logstashui,logstashui.logstashui.svc,logstashui.logstashui.svc.cluster.local + CSRF_TRUSTED_ORIGINS: https://logstashui.example.com + LOGSTASHUI_TLS_SANS: logstashui.example.com,logstashui.logstashui.svc.cluster.local + LOGSTASHUI_HOST_HOSTNAME: logstashui.example.com + LOGSTASHUI_AGENT_UI_URL: https://logstashui.example.com + LOGSTASHUI_DB_ENGINE: sqlite + # db.sqlite3 is created at $LOGSTASHUI_DATA_DIR/db.sqlite3 on the PVC. + # + # NOT recommended. Automatic product TLS works out of the box. + # If you must run the UI and all agent connections over plain HTTP + # (no CA, no certs), uncomment. Do not enable without cause. + # LOGSTASHUI_INSECURE_HTTP: "true" + # + # OpenTelemetry (optional). The image already installs LogstashUI[otel]. + # Uncomment to export traces/metrics over OTLP/HTTP (not gRPC — gevent + # cannot patch grpcio). Point at a collector that accepts HTTP/protobuf + # on 4318. Leave commented to keep tracing off. + # LOGSTASHUI_OTEL: "true" + # OTEL_SERVICE_NAME: logstashui + # OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector.observability.svc:4318 diff --git a/docs/docs/logstashui/kubernetes/examples/sqlite/ingress.yaml b/docs/docs/logstashui/kubernetes/examples/sqlite/ingress.yaml new file mode 100644 index 00000000..a1bae431 --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/sqlite/ingress.yaml @@ -0,0 +1,31 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Ingress-nginx: originate HTTPS to the pod and skip verify of the product leaf. +# Do not set LOGSTASHUI_TLS=false on the container. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: logstashui + namespace: logstashui + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + nginx.ingress.kubernetes.io/proxy-ssl-verify: "off" +spec: + ingressClassName: nginx + # tls: + # - hosts: + # - logstashui.example.com + # secretName: logstashui-tls + rules: + - host: logstashui.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: logstashui + port: + number: 8443 diff --git a/docs/docs/logstashui/kubernetes/examples/sqlite/namespace.yaml b/docs/docs/logstashui/kubernetes/examples/sqlite/namespace.yaml new file mode 100644 index 00000000..d2f1dc8a --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/sqlite/namespace.yaml @@ -0,0 +1,8 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: Namespace +metadata: + name: logstashui diff --git a/docs/docs/logstashui/kubernetes/examples/sqlite/secret.yaml b/docs/docs/logstashui/kubernetes/examples/sqlite/secret.yaml new file mode 100644 index 00000000..ee80ea6b --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/sqlite/secret.yaml @@ -0,0 +1,15 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: Secret +metadata: + name: logstashui + namespace: logstashui +type: Opaque +stringData: + SECRET_KEY: CHANGE-ME-generate-a-django-secret-key + # Shared with Deployment/logstashagent so the embedded node can CSR without enroll. + # Must match what the agent reads. Do not use the default in a real cluster. + LOGSTASHUI_AGENT_CSR_SECRET: CHANGE-ME-shared-with-the-agent diff --git a/docs/docs/logstashui/kubernetes/examples/sqlite/statefulset.yaml b/docs/docs/logstashui/kubernetes/examples/sqlite/statefulset.yaml new file mode 100644 index 00000000..32bed1bc --- /dev/null +++ b/docs/docs/logstashui/kubernetes/examples/sqlite/statefulset.yaml @@ -0,0 +1,106 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +apiVersion: v1 +kind: Service +metadata: + name: logstashui + namespace: logstashui + labels: + app: logstashui +spec: + selector: + app: logstashui + ports: + - name: https + port: 8443 + targetPort: https +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: logstashui + namespace: logstashui +spec: + serviceName: logstashui + replicas: 1 + selector: + matchLabels: + app: logstashui + template: + metadata: + labels: + app: logstashui + spec: + securityContext: + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: logstashui + image: codyjackson032/logstashui:latest + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 8443 + envFrom: + - configMapRef: + name: logstashui + - secretRef: + name: logstashui + env: + - name: LOGSTASHUI_HOST_IPS + valueFrom: + fieldRef: + fieldPath: status.podIP + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /var/lib/logstashui + # Probes are HTTPS. If you uncomment LOGSTASHUI_INSECURE_HTTP in the + # ConfigMap, gunicorn speaks HTTP and these probes will fail until + # scheme is HTTP (and Ingress/HTTPRoute backend protocol too). + # Do not enable that flag without cause. Automatic TLS is supported. + startupProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui + periodSeconds: 30 + failureThreshold: 6 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/docs/docs/logstashui/kubernetes/index.md b/docs/docs/logstashui/kubernetes/index.md new file mode 100644 index 00000000..fc9b083d --- /dev/null +++ b/docs/docs/logstashui/kubernetes/index.md @@ -0,0 +1,121 @@ +# Kubernetes + +Run LogstashUI as a one-replica StatefulSet. The container image is `codyjackson032/logstashui:latest` (or a tag you built). Configuration is environment variables only — ConfigMap for non-secrets, Secret for `SECRET_KEY` and `LOGSTASHUI_DB_PASSWORD`. + +Copy-paste manifests: + +- [SQLite](examples/sqlite/) — database file on the data PVC (fine for a single replica; not for concurrent agents) +- [PostgreSQL](examples/postgresql/) — external Postgres 14+, optional [CloudNativePG](cnpg.md) +- [MySQL / MariaDB](examples/mysql/) — one engine (`mysql`); create the database as `utf8mb4` / `utf8mb4_bin` + +Envoy Gateway users: [skip backend TLS verify](envoy-gateway.md) (`Backend` CR). Ingress-nginx users: see the `ingress.yaml` in each examples folder. + +--- + +## Image and command + +The image already installs `LogstashUI[databases]` and `LogstashUI[otel]`. Tracing stays off until `LOGSTASHUI_OTEL=true`. `CMD` is `logstashui serve`. Do not override the command unless you are debugging. + +```bash +docker build -f docker/Dockerfile -t logstashui:0.5.2-dev . +# Apple Silicon → amd64 cluster: +docker build --platform linux/amd64 -f docker/Dockerfile -t logstashui:0.5.2-dev . +``` + +Build context is the **repository root**. + +--- + +## Data directory (required) + +Set `LOGSTASHUI_DATA_DIR=/var/lib/logstashui` and mount a PVC there. The PVC holds TLS (`tls/`), the Django secret file, logs, `staticfiles/`, and (SQLite only) `db.sqlite3`. Keep the PVC when the database is Postgres or MySQL — it is not optional. + +Use **one** `volumeClaimTemplate` named `data`. Splitting logs onto a second claim is not useful: `LOGSTASHUI_LOGS_DIR` still lives under `DATA_DIR` unless you override it, and TLS/secrets cannot move. + +**Replicas: 1.** LogstashUI is not horizontally scalable (product CA, gunicorn pidfile, SQLite). + +--- + +## Security context + +Image user is **appuser (uid/gid 10001)**. The entrypoint chowns `DATA_DIR` only when it is root. Kubernetes should not run the container as root: + +```yaml +spec: + template: + spec: + securityContext: + fsGroup: 10001 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: logstashui + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +``` + +--- + +## TLS (keep it on) + +Leave `LOGSTASHUI_TLS` unset or `true`. Gunicorn serves HTTPS on **8443** with the product CA under `$LOGSTASHUI_DATA_DIR/tls/`. Do **not** set `LOGSTASHUI_TLS=false` for an ingress or HTTPRoute. + +The gateway or Ingress **originates HTTPS** to the pod and **skips backend certificate verification** because the product leaf is self-signed: + +- Ingress-nginx: `backend-protocol: HTTPS` and `proxy-ssl-verify: "off"` (see examples) +- Envoy Gateway: [`Backend` with `tls.insecureSkipVerify: true`](envoy-gateway.md) + +Set `CSRF_TRUSTED_ORIGINS` and `ALLOWED_HOSTS` to the public hostname (keep `logstashui` in `ALLOWED_HOSTS`). `LOGSTASHUI_TLS_SANS` / `LOGSTASHUI_HOST_HOSTNAME` add that name to the product leaf. `LOGSTASHUI_AGENT_UI_URL` is the URL agents should use. + +Inject the pod IP so the product leaf SAN matches Kubernetes (compose does this with `LOGSTASHUI_HOST_IPS` on the host; the UDP-to-8.8.8.8 trick often has no egress in a cluster): + +```yaml +env: + - name: LOGSTASHUI_HOST_IPS + valueFrom: + fieldRef: + fieldPath: status.podIP +``` + +Django also appends `LOGSTASHUI_HOST_IPS` / `POD_IP` to `ALLOWED_HOSTS` unless that list is `*`. + +Probes: `httpGet` `scheme: HTTPS` on port 8443. kubelet does not verify the pod certificate, but it **does** send `Host: :8443`. Set the probe Host to a name already in `ALLOWED_HOSTS` (examples use `logstashui`): + +```yaml +httpGet: + path: / + port: https + scheme: HTTPS + httpHeaders: + - name: Host + value: logstashui +``` + +--- + +## Database + +Default engine is SQLite on the PVC. For Postgres or MySQL/MariaDB, set the discrete `LOGSTASHUI_DB_*` keys — see [Database](/docs/docs/logstashui/database/index.md). Put the password in a Secret. CloudNativePG: [CNPG](cnpg.md). + +--- + +## Embedded agent (optional) + +This uses the embedded agent image [embedded-agent.yaml](examples/embedded-agent.yaml). Apply a DB tree, set `LOGSTASHUI_AGENT_CSR_SECRET`, then apply the overlay. ClusterIP only (`9500` / `9560` / `9449`). Details: [examples README](examples/README.md#embedded-agent-optional). + +--- + +## Apply (generic) + +```bash +kubectl apply -f docs/docs/logstashui/kubernetes/examples/sqlite/ +# or postgresql/ or mysql/ +# optional embedded sim node: +# kubectl apply -f docs/docs/logstashui/kubernetes/examples/embedded-agent.yaml +``` + +Replace `logstashui.example.com` in ConfigMap, Secret, and Ingress. Replace `SECRET_KEY`. For Postgres/MySQL, create the empty database first ([SQL examples](/docs/docs/logstashui/database/examples/)), then apply the StatefulSet. First start runs `migrate`. diff --git a/packaging/offline/README.md b/packaging/offline/README.md new file mode 100644 index 00000000..82c439e5 --- /dev/null +++ b/packaging/offline/README.md @@ -0,0 +1,5 @@ +# Offline freeze templates + +Used by `bin/freeze_logstashui.sh`. Placeholders `__VERSION__`, `__GIT_SHA__`, `__IMAGE_NAME__` are substituted at freeze time. + +These files are **not** shipped in the default hatchling wheel (systemd templates stay in `src/logstashui/LogstashUI/packaging/`). diff --git a/packaging/offline/compose.offline.yml b/packaging/offline/compose.offline.yml new file mode 100644 index 00000000..87e42bcd --- /dev/null +++ b/packaging/offline/compose.offline.yml @@ -0,0 +1,38 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Air-gapped UI-only compose. No Agent, no embedded profile, no registry pull. +# Load the image first: ./load.sh +# Image: __IMAGE_NAME__ + +name: logstashui-offline + +services: + logstashui: + image: __IMAGE_NAME__ + ports: + - "8443:8443" + volumes: + - logstashui_data:/var/lib/logstashui + environment: + DEBUG: "False" + LOGSTASHUI_DATA_DIR: /var/lib/logstashui + LOGSTASHUI_TLS: "true" + ALLOWED_HOSTS: ${ALLOWED_HOSTS:-*} + CSRF_TRUSTED_ORIGINS: ${CSRF_TRUSTED_ORIGINS:-https://localhost:8443,https://127.0.0.1:8443} + LOGSTASHUI_HOST_HOSTNAME: ${LOGSTASHUI_HOST_HOSTNAME:-} + LOGSTASHUI_HOST_IPS: ${LOGSTASHUI_HOST_IPS:-} + LOGSTASHUI_TLS_SANS: ${LOGSTASHUI_TLS_SANS:-} + LOGSTASHUI_DB_ENGINE: ${LOGSTASHUI_DB_ENGINE:-sqlite} + LOGSTASHUI_DB_HOST: ${LOGSTASHUI_DB_HOST:-} + LOGSTASHUI_DB_PORT: ${LOGSTASHUI_DB_PORT:-} + LOGSTASHUI_DB_NAME: ${LOGSTASHUI_DB_NAME:-} + LOGSTASHUI_DB_USER: ${LOGSTASHUI_DB_USER:-} + LOGSTASHUI_DB_PASSWORD: ${LOGSTASHUI_DB_PASSWORD:-} + PUID: ${PUID:-} + PGID: ${PGID:-} + restart: unless-stopped + +volumes: + logstashui_data: diff --git a/packaging/offline/docker-README.md b/packaging/offline/docker-README.md new file mode 100644 index 00000000..c4c4ebbf --- /dev/null +++ b/packaging/offline/docker-README.md @@ -0,0 +1,16 @@ +# LogstashUI air-gapped Docker image (__VERSION__) + +Linux **x86_64** image tarball. No registry. The image already includes `LogstashUI[databases]` and `LogstashUI[otel]`. Tracing stays off until `LOGSTASHUI_OTEL=true`. There is **no** LogstashAgent in this zip. + +## Load and run + +```sh +./load.sh +docker compose -f compose.offline.yml up -d +``` + +`load.sh` never pulls. HTTPS UI is **:8443**. Named volume `logstashui_data` is `LOGSTASHUI_DATA_DIR=/var/lib/logstashui`. Keep `LOGSTASHUI_TLS` on. + +Set `ALLOWED_HOSTS`, `LOGSTASHUI_HOST_*`, and `LOGSTASHUI_DB_*` in the environment or a `.env` next to `compose.offline.yml` as needed. + +Git: `__GIT_SHA__` diff --git a/packaging/offline/docker-load.sh b/packaging/offline/docker-load.sh new file mode 100755 index 00000000..f9dc0a43 --- /dev/null +++ b/packaging/offline/docker-load.sh @@ -0,0 +1,27 @@ +#!/bin/sh +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +set -eu + +HERE=$(CDPATH= cd -- "$(dirname "$0")" && pwd) +TAR="$HERE/image.tar.gz" +IMAGE="__IMAGE_NAME__" + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required to load this image" >&2 + exit 1 +fi + +[ -f "$TAR" ] || { + echo "missing $TAR" >&2 + exit 1 +} + +echo "Loading $TAR ..." +docker load -i "$TAR" +echo "Loaded image: $IMAGE" +echo "Start (UI only, no Agent):" +echo " docker compose -f \"$HERE/compose.offline.yml\" up -d" +echo "Browse https://:8443 — data volume logstashui_data" diff --git a/packaging/offline/download_wheels.py b/packaging/offline/download_wheels.py new file mode 100644 index 00000000..48cea129 --- /dev/null +++ b/packaging/offline/download_wheels.py @@ -0,0 +1,176 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +"""Download cp312 manylinux wheels. Pure-Python sdists are wheeled on the builder.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +REQ_LINE = re.compile( + r"^(?P[A-Za-z0-9_.-]+)==(?P[^;\\\s]+)(?:\s*;\s*(?P.+))?$" +) + + +def parse_requirements(path: Path) -> list[tuple[str, str, str | None]]: + pkgs: list[tuple[str, str, str | None]] = [] + pending = "" + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or line.startswith("--"): + continue + if line.endswith("\\"): + pending += line[:-1].strip() + " " + continue + line = (pending + line).strip() + pending = "" + # Drop hash fragments pip export may still leave on the same line. + line = re.sub(r"\s+--hash=\S+", "", line).strip() + m = REQ_LINE.match(line) + if not m: + continue + marker = (m.group("marker") or "").strip() or None + pkgs.append((m.group("name"), m.group("ver"), marker)) + return pkgs + + +def pip(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "pip", *args], + text=True, + capture_output=True, + ) + + +def is_linux_x86_wheel(name: str) -> bool: + n = name.lower() + if n.endswith(".tar.gz") or n.endswith(".zip"): + return False + if "macosx" in n or "win_amd64" in n or "win32" in n or "aarch64" in n: + return False + return n.endswith(".whl") and ( + "manylinux" in n or "linux_x86_64" in n or "none-any" in n + ) + + +PLATFORMS = ( + "manylinux2014_x86_64", + "manylinux_2_28_x86_64", +) + + +def download_binary(name: str, ver: str, dest: Path) -> bool: + spec = f"{name}=={ver}" + common = [ + "download", + spec, + "-d", + str(dest), + "--no-deps", + "--python-version", + "3.12", + "--only-binary", + ":all:", + "--disable-pip-version-check", + ] + for platform in PLATFORMS: + proc = pip(*common, "--platform", platform) + if proc.returncode == 0: + return True + # py3-none-any (no --platform): login-required-middleware 0.8 has a wheel; 0.9 does not. + proc = pip(*common) + return proc.returncode == 0 + + +def wheel_sdist(name: str, ver: str, dest: Path) -> None: + spec = f"{name}=={ver}" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + proc = pip( + "download", + spec, + "-d", + str(tmp_path), + "--no-deps", + "--no-binary", + ":all:", + "--disable-pip-version-check", + ) + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + sys.stderr.write(proc.stderr) + raise SystemExit( + f"no manylinux cp312 wheel and sdist download failed for {spec}" + ) + sdists = list(tmp_path.glob("*.tar.gz")) + list(tmp_path.glob("*.zip")) + if not sdists: + raise SystemExit(f"sdist missing after download for {spec}") + proc = pip( + "wheel", + "--no-deps", + "--no-cache-dir", + "-w", + str(tmp_path), + str(sdists[0]), + "--disable-pip-version-check", + ) + if proc.returncode != 0: + sys.stderr.write(proc.stdout) + sys.stderr.write(proc.stderr) + raise SystemExit(f"could not build a wheel from sdist for {spec}") + wheels = [p for p in tmp_path.glob("*.whl") if is_linux_x86_wheel(p.name)] + if not wheels: + built = list(tmp_path.glob("*.whl")) + raise SystemExit( + f"{spec} is sdist-only and built {built or 'no wheel'} " + f"(need py3-none-any or manylinux x86_64). Freeze aborted." + ) + target = dest / wheels[0].name + target.write_bytes(wheels[0].read_bytes()) + print(f"built wheel from sdist: {wheels[0].name}", flush=True) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("requirements") + parser.add_argument("dest") + args = parser.parse_args() + dest = Path(args.dest) + dest.mkdir(parents=True, exist_ok=True) + pkgs = parse_requirements(Path(args.requirements)) + if not pkgs: + raise SystemExit(f"no packages parsed from {args.requirements}") + for name, ver, marker in pkgs: + marker_norm = (marker or "").replace('"', "'") + if "sys_platform == 'win32'" in marker_norm or "sys_platform == 'darwin'" in marker_norm: + print(f"skip {name}=={ver} (marker {marker})", flush=True) + continue + spec = f"{name}=={ver}" + print(f"download {spec}", flush=True) + if download_binary(name, ver, dest): + continue + print(f"binary miss {spec}; trying sdist→wheel", flush=True) + wheel_sdist(name, ver, dest) + bad = [ + p.name + for p in dest.iterdir() + if p.is_file() and not is_linux_x86_wheel(p.name) and p.suffix != ".txt" + ] + # Allow the LogstashUI wheel already copied in (py3-none-any). + leftover_sdists = list(dest.glob("*.tar.gz")) + list(dest.glob("*.zip")) + if leftover_sdists: + names = ", ".join(p.name for p in leftover_sdists) + raise SystemExit(f"sdist left in wheelhouse: {names}") + if bad: + raise SystemExit(f"non-linux-x86_64 artifacts: {', '.join(bad)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/offline/entry.py b/packaging/offline/entry.py new file mode 100644 index 00000000..c75fff80 --- /dev/null +++ b/packaging/offline/entry.py @@ -0,0 +1,10 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +"""PyInstaller entry for the experimental standalone freeze.""" + +from LogstashUI.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/offline/logstashui.spec b/packaging/offline/logstashui.spec new file mode 100644 index 00000000..e432a1aa --- /dev/null +++ b/packaging/offline/logstashui.spec @@ -0,0 +1,99 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# PyInstaller onedir spec for LogstashUI. Experimental: gunicorn+gevent+Django +# hidden imports. Do not change product --worker-class gevent from here. +# -*- mode: python ; coding: utf-8 -*- + +from __future__ import annotations + +import os + +from PyInstaller.building.api import COLLECT, EXE, PYZ +from PyInstaller.building.build_main import Analysis +from PyInstaller.utils.hooks import collect_all + +SPECDIR = os.path.dirname(os.path.abspath(SPEC)) +ROOT = os.path.abspath(os.path.join(SPECDIR, "..", "..")) +SRC = os.path.join(ROOT, "src", "logstashui") +ENTRY = os.path.join(SPECDIR, "entry.py") + +PACKAGES = [ + "LogstashUI", + "PipelineManager", + "Management", + "Utilities", + "SNMP", + "Monitoring", + "Site", + "Documentation", + "AI", + "theme", + "Common", + "django", + "gunicorn", + "gevent", + "greenlet", + "cryptography", + "pysnmp", + "lark", + "pygrok", + "whitenoise", + "psycopg", + "pymysql", + "yaml", + "django_htmx", + "tailwind", + "login_required", + "elasticsearch", + "requests", + "markdown", + "packaging", + "django_browser_reload", +] + +datas: list = [] +binaries: list = [] +hiddenimports: list = [] +for pkg in PACKAGES: + try: + d, b, h = collect_all(pkg) + except Exception: + continue + datas += d + binaries += b + hiddenimports += h + +a = Analysis( + [ENTRY], + pathex=[SRC], + binaries=binaries, + datas=datas, + hiddenimports=hiddenimports, + hookspath=[], + runtime_hooks=[], + excludes=[], + noarchive=False, +) +pyz = PYZ(a.pure) +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name="logstashui", + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=False, + console=True, +) +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=False, + name="logstashui", +) diff --git a/packaging/offline/standalone-README.md b/packaging/offline/standalone-README.md new file mode 100644 index 00000000..2c21d2f4 --- /dev/null +++ b/packaging/offline/standalone-README.md @@ -0,0 +1,19 @@ +# LogstashUI experimental standalone (__VERSION__) + +**Experimental.** PyInstaller onedir for Linux **x86_64**. No Python on the host. Not production-supported until `./logstashui serve` completes migrate, SNMP sync, collectstatic, and HTTPS :8443 without network. + +Gunicorn stays `--worker-class gevent`. If this bundle cannot serve, that is a freeze bug — do not switch workers as a workaround without amending the design spec. + +## Run + +```sh +./run.sh +# or: +./logstashui/logstashui serve +``` + +Data dir default: `$(pwd)/logstashui_data`. Same `LOGSTASHUI_*` env as a wheel/Docker install. + +systemd: set `ExecStart=` to the unpacked `logstashui/logstashui serve`. This zip does not write units. + +Git: `__GIT_SHA__` diff --git a/packaging/offline/standalone-run.sh b/packaging/offline/standalone-run.sh new file mode 100755 index 00000000..955b8410 --- /dev/null +++ b/packaging/offline/standalone-run.sh @@ -0,0 +1,16 @@ +#!/bin/sh +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +set -eu + +HERE=$(CDPATH= cd -- "$(dirname "$0")" && pwd) +BIN="$HERE/logstashui/logstashui" + +if [ ! -x "$BIN" ]; then + echo "missing $BIN" >&2 + exit 1 +fi + +exec "$BIN" serve "$@" diff --git a/packaging/offline/wheels-README.md b/packaging/offline/wheels-README.md new file mode 100644 index 00000000..b9955f37 --- /dev/null +++ b/packaging/offline/wheels-README.md @@ -0,0 +1,35 @@ +# LogstashUI air-gapped wheelhouse (__VERSION__) + +Linux **x86_64**, CPython **3.12** only. No PyPI. `[databases]` (psycopg + PyMySQL) and `[otel]` extras are in `wheels/`. SQLite is still the runtime default. Tracing stays off until `LOGSTASHUI_OTEL=true`. + +This zip is **not** the recommended install when the host can reach GitHub or a container registry. Prefer Docker Compose (connected) or `pip install` of the normal wheel. + +## Host packages + +- CPython 3.12 (64-bit x86_64) +- Distro venv module (Debian/Ubuntu: `python3.12-venv`) + +uv is **not** required. + +## Install and run + +```sh +./install.sh +.venv/bin/logstashui serve +``` + +Override the interpreter with `PYTHON=/path/to/python3.12 ./install.sh`. + +HTTPS UI is **:8443**. Data dir default is `$(pwd)/logstashui_data` (`LOGSTASHUI_DATA_DIR`). Same env vars as a normal install (`LOGSTASHUI_*`, `LOGSTASHUI_DB_*`). + +systemd (does **not** enable the unit): + +```sh +sudo .venv/bin/logstashui systemd +``` + +## Not included + +LogstashAgent. Enroll agents separately. arm64 / Windows freezes are not this zip. + +Git: `__GIT_SHA__` diff --git a/packaging/offline/wheels-install.sh b/packaging/offline/wheels-install.sh new file mode 100755 index 00000000..a2f3f347 --- /dev/null +++ b/packaging/offline/wheels-install.sh @@ -0,0 +1,56 @@ +#!/bin/sh +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +# Air-gapped install: CPython 3.12 x86_64 venv + pip --no-index. +# Do not upgrade pip (that hits PyPI). + +set -eu + +HERE=$(CDPATH= cd -- "$(dirname "$0")" && pwd) +WHEELS="$HERE/wheels" +PYTHON="${PYTHON:-python3.12}" + +die() { + printf '%s\n' "$*" >&2 + exit 1 +} + +if ! command -v "$PYTHON" >/dev/null 2>&1; then + if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 + else + die "need CPython 3.12 x86_64 (python3.12 not found). On Debian/Ubuntu: apt install python3.12 python3.12-venv" + fi +fi + +"$PYTHON" - <<'PY' || die "need CPython 3.12 x86_64" +import platform +import sys + +if sys.version_info[:2] != (3, 12): + sys.exit(1) +if sys.maxsize <= 2**32: + sys.exit(1) +mach = platform.machine().lower() +if mach not in ("x86_64", "amd64"): + sys.exit(1) +PY + +[ -d "$WHEELS" ] || die "missing $WHEELS" + +VENV="$HERE/.venv" +"$PYTHON" -m venv "$VENV" +"$VENV/bin/python" -m pip install \ + --disable-pip-version-check \ + --no-index \ + --no-cache-dir \ + --find-links "$WHEELS" \ + 'LogstashUI[databases,otel]' + +printf '\nInstalled into %s\n' "$VENV" +printf 'Start:\n' +printf ' %s/bin/logstashui serve\n' "$VENV" +printf 'Data directory default: $(pwd)/logstashui_data (override LOGSTASHUI_DATA_DIR)\n' +printf 'Configuration is environment variables only. See README.md.\n' diff --git a/pyproject.toml b/pyproject.toml index 10d90da0..7f76d8d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "LogstashUI" -version = "0.5.1" +version = "0.5.2" description = "A control plane and UI for Logstash!" readme = "README.md" license = { file = "LICENSE.txt" } @@ -35,6 +35,26 @@ logstashui = "LogstashUI.cli:main" [project.urls] Homepage = "https://github.com/elastic/LogstashUI" +[project.optional-dependencies] +postgres = [ + "psycopg[binary]>=3.2.0", +] +mysql = [ + "PyMySQL>=1.1.1", +] +databases = [ + "psycopg[binary]>=3.2.0", + "PyMySQL>=1.1.1", +] +# Deliberately HTTP/protobuf, never gRPC: grpcio's native C-core threads cannot +# be monkey-patched by gevent and deadlock under the gunicorn worker class. +otel = [ + "opentelemetry-sdk>=1.30.0", + "opentelemetry-exporter-otlp-proto-http>=1.30.0", + "opentelemetry-instrumentation-django>=0.51b0", + "opentelemetry-instrumentation-requests>=0.51b0", +] + [tool.uv] package = true @@ -87,13 +107,18 @@ dev = [ "pytest>=9.0.2", "pytest-cov>=7.1.0", "pytest-django>=4.10.0", + "testcontainers[postgres]>=4.15.0", + "testcontainers[mysql]>=4.15.0", ] [tool.pytest.ini_options] DJANGO_SETTINGS_MODULE = "LogstashUI.settings" python_files = ["tests.py", "test_*.py", "*_tests.py"] pythonpath = ["src/logstashui"] -testpaths = ["src/logstashui"] +testpaths = ["tests"] +markers = [ + "integration: marks tests that require running Docker containers", +] addopts = [ "--import-mode=importlib", "--cov", diff --git a/scripts/add_license_headers.py b/scripts/add_license_headers.py index 384bc7b7..aa299e75 100644 --- a/scripts/add_license_headers.py +++ b/scripts/add_license_headers.py @@ -172,7 +172,7 @@ def process_file(file_path, dry_run=False, verbose=False): return True else: try: - with open(file_path, 'w', encoding='utf-8') as f: + with open(file_path, 'w', encoding='utf-8', newline='\n') as f: f.write(new_content) if verbose: print(f"[MODIFIED] {file_path}") diff --git a/scripts/generate_notice.py b/scripts/generate_notice.py index fae77d96..a3069a5c 100644 --- a/scripts/generate_notice.py +++ b/scripts/generate_notice.py @@ -34,7 +34,9 @@ "d3": "https://github.com/d3/d3/blob/main/LICENSE", "codemirror": "https://github.com/codemirror/dev/blob/main/LICENSE", "js-yaml": "https://github.com/nodeca/js-yaml/blob/master/LICENSE", - "markedjs": "https://github.com/markedjs/marked/blob/master/LICENSE" + "markedjs": "https://github.com/markedjs/marked/blob/master/LICENSE", + "psycopg": "https://github.com/psycopg/psycopg/blob/master/LICENSE.txt", + "PyMySQL": "https://github.com/PyMySQL/PyMySQL/blob/main/LICENSE", } # Repository mappings for dependencies (fallback when automatic lookup fails) @@ -81,6 +83,9 @@ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.""", + "psycopg": "https://github.com/psycopg/psycopg/blob/master/LICENSE.txt", + "psycopg-binary": "https://github.com/psycopg/psycopg/blob/master/LICENSE.txt", + "PyMySQL": "https://github.com/PyMySQL/PyMySQL/blob/main/LICENSE", "greenlet": "https://github.com/python-greenlet/greenlet/blob/master/LICENSE", "portalocker": "https://github.com/wolph/portalocker/blob/develop/LICENSE", "zope-event": "https://github.com/zopefoundation/zope.event/blob/master/LICENSE.txt", @@ -463,7 +468,7 @@ def ensure_notice_header(): # Check if header is already present if not content.startswith("LogstashUI\nCopyright 2025-"): # Header missing or outdated, prepend it - with open(notice_path, "w", encoding="utf-8") as f: + with open(notice_path, "w", encoding="utf-8", newline='\n') as f: f.write(header) if content and not content.startswith("\n"): f.write("\n") @@ -481,12 +486,12 @@ def ensure_notice_header(): content, count=1 ) - with open(notice_path, "w", encoding="utf-8") as f: + with open(notice_path, "w", encoding="utf-8", newline='\n') as f: f.write(updated_content) print(f"Updated copyright year to {current_year}") else: # Create new file with header - with open(notice_path, "w", encoding="utf-8") as f: + with open(notice_path, "w", encoding="utf-8", newline='\n') as f: f.write(header) print("Created NOTICE.txt with header") @@ -526,7 +531,7 @@ def append_to_notice(package_name, license_text, license_name=None): metadata[package_name] = license_name or 'UNKNOWN' - with open(metadata_path, 'w', encoding='utf-8') as f: + with open(metadata_path, 'w', encoding='utf-8', newline='\n') as f: json.dump(metadata, f, indent=2, sort_keys=True) print(f"Added {package_name} to NOTICE.txt") @@ -734,7 +739,7 @@ def generate_dependency_tracking(all_deps, license_cache): class_width = max(max_class_len, len('Classification')) # Generate table - with open(tracking_path, 'w', encoding='utf-8') as f: + with open(tracking_path, 'w', encoding='utf-8', newline='\n') as f: # Header f.write(f"{'Dependency':<{name_width}} | {'Type':<{type_width}} | {'License':<{license_width}} | {'Classification':<{class_width}}\n") f.write(f"{'-' * name_width}-+-{'-' * type_width}-+-{'-' * license_width}-+-{'-' * class_width}\n") diff --git a/src/logstashui/Common/decorators.py b/src/logstashui/Common/decorators.py index 243e7cc0..dc705cc4 100644 --- a/src/logstashui/Common/decorators.py +++ b/src/logstashui/Common/decorators.py @@ -3,11 +3,26 @@ #you may not use this file except in compliance with the Elastic License. from functools import wraps -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse import logging logger = logging.getLogger(__name__) + +def _denied(request, message): + """Build a denial a browser *or* a script can read. + + The HX-Trigger toast is meaningless to curl, so API-token callers get JSON. + """ + if getattr(request, '_api_token', None) is not None: + return JsonResponse({'success': False, 'error': message}, status=403) + response = HttpResponse(message, status=403) + response['HX-Trigger'] = ( + '{"showToastEvent": {"message": "%s", "type": "error"}}' % message + ) + return response + + def require_admin_role(view_func): """ Decorator to check if user has admin role before allowing access to view. @@ -17,17 +32,13 @@ def require_admin_role(view_func): def wrapper(request, *args, **kwargs): # Check if user is authenticated if not request.user.is_authenticated: - response = HttpResponse('You must be logged in to perform this action', status=403) - response['HX-Trigger'] = '{"showToastEvent": {"message": "You must be logged in to perform this action", "type": "error"}}' - return response + return _denied(request, 'You must be logged in to perform this action') # Check if user has admin role if not hasattr(request.user, 'profile') or request.user.profile.role != 'admin': role_info = f"'{request.user.profile.role}'" if hasattr(request.user, 'profile') else 'no profile' logger.warning(f"User '{request.user.username}' with {role_info} attempted to access admin-only function: {view_func.__name__}") - response = HttpResponse('Access denied: Admin role required', status=403) - response['HX-Trigger'] = '{"showToastEvent": {"message": "Access denied: Admin role required", "type": "error"}}' - return response + return _denied(request, 'Access denied: Admin role required') # User is admin, proceed with the view return view_func(request, *args, **kwargs) diff --git a/src/logstashui/Common/middleware.py b/src/logstashui/Common/middleware.py index df1d7c9f..9b65a4e9 100644 --- a/src/logstashui/Common/middleware.py +++ b/src/logstashui/Common/middleware.py @@ -3,6 +3,99 @@ #you may not use this file except in compliance with the Elastic License. +import logging + +from django.http import JsonResponse +from django.utils import timezone + +logger = logging.getLogger(__name__) + +_AUTH_SCHEME = 'ApiKey ' + + +def _resolve_api_token(request): + """Resolve an admin API token from the Authorization header. + + Returns the ``ApiKey`` row on success, ``None`` when the request carries no + admin token at all (browser traffic, or an agent key — those have no + ``lsui_`` marker and are authenticated by the agent views themselves), or + the string ``'invalid'`` when a token was offered but is not usable. + """ + header = request.headers.get('Authorization', '') + if not header.startswith(_AUTH_SCHEME): + return None + + from PipelineManager.models import ApiKey + + prefix, secret = ApiKey.parse_token(header[len(_AUTH_SCHEME):].strip()) + if prefix is None: + # Not an admin token. Leave agent keys entirely alone. + return None + + token = ApiKey.objects.filter(prefix=prefix).select_related('user').first() + if token is None or token.user is None: + return 'invalid' + if not token.is_active or not token.user.is_active: + return 'invalid' + if not token.verify_api_key(secret): + return 'invalid' + return token + + +class ApiTokenCsrfMiddleware: + """Authenticate admin API tokens and exempt only those requests from CSRF. + + Must run immediately *before* ``CsrfViewMiddleware``. The companion + ``ApiTokenUserMiddleware`` assigns ``request.user`` afterwards, because + ``AuthenticationMiddleware`` runs after CSRF and would overwrite anything + set here. + + ``_dont_enforce_csrf_checks`` is set only once the token has verified, so a + forged or absent token cannot be used to switch CSRF off. + """ + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + try: + token = _resolve_api_token(request) + except Exception: + # DB not ready (pre-migration) — behave as if no token was offered. + logger.exception("API token resolution failed") + token = None + + if token == 'invalid': + return JsonResponse( + {'success': False, 'error': 'Invalid or expired API token'}, + status=401, + ) + + if token is not None: + request._api_token = token + request._dont_enforce_csrf_checks = True + + return self.get_response(request) + + +class ApiTokenUserMiddleware: + """Act as the token's owner. Runs just after ``AuthenticationMiddleware``.""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + token = getattr(request, '_api_token', None) + if token is not None: + request.user = token.user + # .update() rather than .save() so the write never round-trips + # through the hashing path in ApiKey.save(). + type(token).objects.filter(pk=token.pk).update( + last_used_at=timezone.now() + ) + return self.get_response(request) + + class NoAuthMiddleware: """ Sandbox middleware: auto-authenticates every request as the first active user. diff --git a/src/logstashui/Common/product_ca.py b/src/logstashui/Common/product_ca.py index 64ee9bba..5a192bdc 100644 --- a/src/logstashui/Common/product_ca.py +++ b/src/logstashui/Common/product_ca.py @@ -31,6 +31,19 @@ logger = logging.getLogger(__name__) + +class ProductCADisabled(RuntimeError): + """ensure_* called while LOGSTASHUI_INSECURE_HTTP=true.""" + + +def _raise_if_insecure_http() -> None: + from LogstashUI.insecure_http import insecure_http + + if insecure_http(): + raise ProductCADisabled( + "LOGSTASHUI_INSECURE_HTTP=true: product CA is not generated" + ) + _lock = threading.Lock() _cached_cert_pem: Optional[bytes] = None _cached_fingerprint: Optional[str] = None @@ -108,6 +121,7 @@ def ensure_product_ca() -> Tuple[bytes, str]: """ Ensure product CA exists on disk; return (cert_pem_bytes, fingerprint_hex). """ + _raise_if_insecure_http() global _cached_cert_pem, _cached_fingerprint with _lock: if _cached_cert_pem and _cached_fingerprint: @@ -158,10 +172,14 @@ def build_enrollment_token_payload(raw_token: str) -> dict: Includes fingerprint when agent.include_ca_fingerprint is true (default). Does not include ui_url (CLI --logstash-ui-url). """ + from LogstashUI.insecure_http import insecure_http + payload = { "enrollment_token": raw_token, "token_version": 2, } + if insecure_http(): + return payload cfg = getattr(settings, "LOGSTASHUI_CONFIG", {}) or {} agent_cfg = cfg.get("agent") or {} include_fp = agent_cfg.get("include_ca_fingerprint", True) @@ -186,13 +204,17 @@ def get_agent_ui_url_default() -> str: db_url = (AppSettings.get_settings().agent_ui_url or "").strip() if db_url: - return db_url.rstrip("/") + from LogstashUI.insecure_http import force_http_url + + return force_http_url(db_url.rstrip("/")) or "" except Exception: pass cfg = getattr(settings, "LOGSTASHUI_CONFIG", {}) or {} agent_cfg = cfg.get("agent") or {} url = (agent_cfg.get("ui_url") or "").strip() - return url.rstrip("/") if url else "" + from LogstashUI.insecure_http import force_http_url + + return force_http_url(url.rstrip("/") if url else "") or "" # --------------------------------------------------------------------------- @@ -555,6 +577,7 @@ def ensure_default_ui_server_cert( Re-issues when force=True or when desired SANs (host hostname/IPs, callback URL, LOGSTASHUI_TLS_SANS, etc.) are not all present on the current leaf. """ + _raise_if_insecure_http() if get_ui_server_mode() == "custom" and ui_server_cert_path().is_file(): return ui_server_cert_path(), ui_server_key_path() @@ -649,6 +672,7 @@ def save_custom_ui_certificate( Does not replace the product CA. Validates that key matches leaf cert. """ + _raise_if_insecure_http() if not cert_pem or not key_pem: raise ValueError("Certificate and private key are required") @@ -727,6 +751,7 @@ def save_custom_ui_certificate( def revert_ui_certificate_to_product_default() -> dict: """Remove custom cert and regenerate product-CA-signed leaf.""" + _raise_if_insecure_http() for p in (ui_server_cert_path(), ui_server_key_path(), ui_server_chain_path(), ui_server_mode_path()): try: if p.is_file(): @@ -749,6 +774,19 @@ def ui_tls_paths_for_display() -> dict: def get_ui_tls_status() -> dict: """Status blob for Management → Settings.""" + from LogstashUI.insecure_http import INSECURE_HTTP_WARNING, insecure_http + + if insecure_http(): + return { + "mode": "disabled", + "insecure_http": True, + "paths": {}, + "product_ca_fingerprint": None, + "certificate": None, + "has_custom": False, + "tls_hint": INSECURE_HTTP_WARNING, + "nginx_hint": INSECURE_HTTP_WARNING, + } ensure_product_ca() mode = get_ui_server_mode() # Ensure product leaf exists when in product mode @@ -760,6 +798,7 @@ def get_ui_tls_status() -> dict: status = { "mode": mode, + "insecure_http": False, "paths": ui_tls_paths_for_display(), "product_ca_fingerprint": get_ca_fingerprint(), "certificate": None, @@ -931,6 +970,10 @@ def agent_requests_verify() -> Union[bool, str]: truncated PEMs and intermittent ``[X509] PEM lib`` SSL failures mid-sim. """ global _agent_verify_bundle_path, _agent_verify_bundle_mtime + from LogstashUI.insecure_http import insecure_http + + if insecure_http(): + return False try: ensure_product_ca() product_path = ca_cert_path() diff --git a/src/logstashui/Documentation/views.py b/src/logstashui/Documentation/views.py index f8f4d99b..54c5d5df 100644 --- a/src/logstashui/Documentation/views.py +++ b/src/logstashui/Documentation/views.py @@ -24,6 +24,8 @@ 'logstashagent.yml': 'logstashagent.yml', 'logstashui.yml': 'logstashui.yml', 'SNMP': 'SNMP', + 'api_access': 'API Access', + 'logstash_proxy': 'Logstash Tarball Proxy', 'tsds_implementation': 'TSDS Implementation', 'data_overview': 'Data Overview', 'pipeline_generation': 'Pipeline Generation', diff --git a/src/logstashui/LogstashUI/cli.py b/src/logstashui/LogstashUI/cli.py index 53bcb121..dd689605 100644 --- a/src/logstashui/LogstashUI/cli.py +++ b/src/logstashui/LogstashUI/cli.py @@ -7,12 +7,15 @@ from __future__ import annotations import argparse +import logging import os import shutil import sys from importlib.resources import files from pathlib import Path +logger = logging.getLogger(__name__) + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( @@ -47,6 +50,30 @@ def build_parser() -> argparse.ArgumentParser: manage = sub.add_parser("manage", help="Django management command passthrough") manage.add_argument("manage_args", nargs=argparse.REMAINDER) + migrate = sub.add_parser( + "migrate-engine", + help="BETA: copy SQLite data to PostgreSQL or MySQL (stops gunicorn)", + ) + migrate.add_argument( + "--to", + required=True, + choices=("postgresql", "mysql", "mariadb"), + help="Target engine (mariadb is an alias of mysql)", + ) + migrate.add_argument( + "--i-have-a-backup", + dest="i_have_a_backup", + action="store_true", + help="Required. Confirms db.sqlite3 was copied aside.", + ) + migrate.add_argument("--pid", type=Path, default=None, help="gunicorn pidfile to signal") + migrate.add_argument( + "--write-env", + type=Path, + default=None, + help="Append LOGSTASHUI_DB_* to this EnvironmentFile", + ) + systemd = sub.add_parser( "systemd", help="Generate /etc/default/logstashui and the systemd unit (manual)", @@ -89,6 +116,11 @@ def build_parser() -> argparse.ArgumentParser: default="false", choices=("true", "false"), ) + systemd.add_argument("--db-engine", default="") + systemd.add_argument("--db-host", default="") + systemd.add_argument("--db-name", default="") + systemd.add_argument("--db-user", default="") + systemd.add_argument("--db-port", default="") parser.set_defaults( command="serve", bind=os.environ.get("LOGSTASHUI_BIND", "0.0.0.0:8443"), @@ -126,6 +158,11 @@ def render_default_env( tls_sans: str, agent_ui_url: str, no_auth: str, + db_engine: str = "", + db_host: str = "", + db_name: str = "", + db_user: str = "", + db_port: str = "", ) -> str: sample = _packaging_file("logstashui.default") replacements = { @@ -151,6 +188,18 @@ def render_default_env( extras.append(f"LOGSTASHUI_TLS_SANS={tls_sans}") if agent_ui_url: extras.append(f"LOGSTASHUI_AGENT_UI_URL={agent_ui_url}") + from LogstashUI.database import canonical_engine + + if db_engine and canonical_engine(db_engine) != "sqlite": + extras.append(f"LOGSTASHUI_DB_ENGINE={canonical_engine(db_engine)}") + if db_host: + extras.append(f"LOGSTASHUI_DB_HOST={db_host}") + if db_port: + extras.append(f"LOGSTASHUI_DB_PORT={db_port}") + if db_name: + extras.append(f"LOGSTASHUI_DB_NAME={db_name}") + if db_user: + extras.append(f"LOGSTASHUI_DB_USER={db_user}") if extras: text = text.rstrip() + "\n\n# Values from logstashui systemd\n" + "\n".join(extras) + "\n" return text @@ -182,11 +231,18 @@ def install_systemd( tls_sans: str = "", agent_ui_url: str = "", no_auth: str = "false", + db_engine: str = "", + db_host: str = "", + db_name: str = "", + db_user: str = "", + db_port: str = "", dry_run: bool = False, print_only: bool = False, interactive: bool = False, ) -> dict: if interactive and not dry_run and output_dir is None: + from LogstashUI.database import canonical_engine + exec_start = _prompt( "Path to logstashui executable", exec_start or _default_exec_start(), @@ -204,6 +260,15 @@ def install_systemd( tls_sans = _prompt("LOGSTASHUI_TLS_SANS", tls_sans) agent_ui_url = _prompt("LOGSTASHUI_AGENT_UI_URL", agent_ui_url) no_auth = _prompt("LOGSTASHUI_NO_AUTH (true/false)", no_auth) + db_engine = _prompt( + "LOGSTASHUI_DB_ENGINE (sqlite/postgresql/mysql)", + db_engine or "sqlite", + ) + if canonical_engine(db_engine) != "sqlite": + db_host = _prompt("LOGSTASHUI_DB_HOST", db_host) + db_port = _prompt("LOGSTASHUI_DB_PORT", db_port) + db_name = _prompt("LOGSTASHUI_DB_NAME", db_name or "logstashui") + db_user = _prompt("LOGSTASHUI_DB_USER", db_user) if not exec_start: exec_start = _default_exec_start() @@ -226,6 +291,11 @@ def install_systemd( tls_sans=tls_sans, agent_ui_url=agent_ui_url, no_auth=no_auth, + db_engine=db_engine, + db_host=db_host, + db_name=db_name, + db_user=db_user, + db_port=db_port, ) if print_only: @@ -299,16 +369,66 @@ def _best_effort_call(name: str, **kwargs) -> None: print(f"Warning: {name} failed: {exc}", file=sys.stderr) +def _check_db_floor() -> None: + """Connect and enforce engine version floors before migrate or gunicorn bind.""" + _django_setup() + from django.db import connection + + from LogstashUI.database import check_server_version + + connection.ensure_connection() + check_server_version(connection) + + +def _exec_gunicorn(gunicorn_cmd: list[str]) -> int: + """Replace this process with gunicorn, or run it in-process when frozen. + + PyInstaller onedir has no ``gunicorn`` console script on PATH. Calling + gunicorn's WSGI app in-process keeps ``--worker-class gevent``. + """ + if getattr(sys, "frozen", False): + from gunicorn.app.wsgiapp import run as gunicorn_run + + sys.argv = list(gunicorn_cmd) + result = gunicorn_run() + return int(result or 0) + os.execvp("gunicorn", gunicorn_cmd) + return 1 + + def cmd_serve(args: argparse.Namespace) -> int: + from LogstashUI.database import canonical_engine + from LogstashUI.insecure_http import insecure_http, warn_if_enabled + from LogstashUI.paths import resolve_data_dir + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "LogstashUI.settings") tls_env = os.environ.get("LOGSTASHUI_TLS", "true") - tls_on = not args.no_tls and tls_env.strip().lower() not in ("0", "false", "no", "off") + tls_on = ( + not args.no_tls + and not insecure_http() + and tls_env.strip().lower() not in ("0", "false", "no", "off") + ) + if insecure_http(): + warn_if_enabled() + + engine = canonical_engine(os.environ.get("LOGSTASHUI_DB_ENGINE")) + if engine == "sqlite" and args.workers > 1: + msg = ( + "SQLite is the small-install default; use PostgreSQL or MySQL/MariaDB " + "for concurrent agents (LOGSTASHUI_WORKERS>1)." + ) + logger.warning(msg) + print(msg, file=sys.stderr) + _check_db_floor() if not args.skip_migrate: _manage(["migrate", "--noinput"]) _best_effort_call("sync_snmp_official_data", cleanup=True) _best_effort_call("collectstatic", interactive=False) + data_dir = resolve_data_dir() + data_dir.mkdir(parents=True, exist_ok=True) + gunicorn_cmd = [ "gunicorn", "LogstashUI.wsgi:application", @@ -326,6 +446,8 @@ def cmd_serve(args: argparse.Namespace) -> int: "-", "--error-logfile", "-", + "--pid", + str(data_dir / "gunicorn.pid"), ] if tls_on: _django_setup() @@ -348,8 +470,7 @@ def cmd_serve(args: argparse.Namespace) -> int: fullchain.write_bytes(cert.read_bytes()) gunicorn_cmd += ["--certfile", str(fullchain), "--keyfile", str(key)] - os.execvp("gunicorn", gunicorn_cmd) - return 1 + return _exec_gunicorn(gunicorn_cmd) def cmd_systemd(args: argparse.Namespace) -> int: @@ -371,6 +492,11 @@ def cmd_systemd(args: argparse.Namespace) -> int: tls_sans=args.tls_sans, agent_ui_url=args.agent_ui_url, no_auth=args.no_auth, + db_engine=args.db_engine, + db_host=args.db_host, + db_name=args.db_name, + db_user=args.db_user, + db_port=args.db_port, dry_run=dry_run, print_only=args.print_only, interactive=interactive, @@ -387,6 +513,9 @@ def main(argv: list[str] | None = None) -> int: return 0 if command == "systemd": return cmd_systemd(args) + if command == "migrate-engine": + from LogstashUI.migrate_engine import cmd_migrate_engine + return cmd_migrate_engine(args) if command == "serve": if not hasattr(args, "bind"): args = parser.parse_args(["serve"]) diff --git a/src/logstashui/LogstashUI/config.py b/src/logstashui/LogstashUI/config.py index 60b19560..ce999f12 100644 --- a/src/logstashui/LogstashUI/config.py +++ b/src/logstashui/LogstashUI/config.py @@ -11,6 +11,38 @@ _FALSE = ("0", "false", "no", "off") +def _split_csv_hosts(raw: str) -> list[str]: + return [part.strip() for part in (raw or "").split(",") if part.strip()] + + +def merge_allowed_hosts( + allowed: str | None = None, + host_ips: str | None = None, + pod_ip: str | None = None, +) -> list[str]: + """Django ALLOWED_HOSTS, plus pod/host IPs used as kube-probe Host headers. + + ``ALLOWED_HOSTS=*`` stays a single wildcard. Otherwise ``LOGSTASHUI_HOST_IPS`` + and ``POD_IP`` are appended (Kubernetes Downward API / compose host IPs). + """ + if allowed is None: + allowed = os.environ.get("ALLOWED_HOSTS", "*") + hosts = _split_csv_hosts(allowed) + if not hosts: + hosts = ["*"] + if hosts == ["*"]: + return hosts + extras = _split_csv_hosts( + host_ips if host_ips is not None else os.environ.get("LOGSTASHUI_HOST_IPS", "") + ) + extra_pod = pod_ip if pod_ip is not None else os.environ.get("POD_IP", "") + extras.extend(_split_csv_hosts(extra_pod)) + for host in extras: + if host not in hosts: + hosts.append(host) + return hosts + + def env_bool(name: str, default: bool = False) -> bool: raw = os.environ.get(name) if raw is None: diff --git a/src/logstashui/LogstashUI/database.py b/src/logstashui/LogstashUI/database.py index 43cfeee9..a5201ca1 100644 --- a/src/logstashui/LogstashUI/database.py +++ b/src/logstashui/LogstashUI/database.py @@ -2,28 +2,89 @@ #or more contributor license agreements. Licensed under the Elastic License; #you may not use this file except in compliance with the Elastic License. -"""Database settings helper. - -SQLite is the only implemented engine. PostgreSQL and MySQL are anticipated -via ``LOGSTASHUI_DB_ENGINE`` but not wired in this release. -""" +"""Build Django DATABASES from discrete LOGSTASHUI_DB_* environment variables.""" from __future__ import annotations import os from pathlib import Path +from .config import env_bool + +_ENGINE_ALIASES = { + "": "sqlite", + "sqlite": "sqlite", + "sqlite3": "sqlite", + "postgres": "postgresql", + "postgresql": "postgresql", + "mysql": "mysql", + "mariadb": "mysql", + "my": "mysql", +} + +_CANONICAL = ("sqlite", "postgresql", "mysql") + + +def canonical_engine(raw: str | None) -> str: + key = (raw or "").strip().lower() + if key not in _ENGINE_ALIASES: + supported = "sqlite, postgresql, mysql (aliases: sqlite3, postgres, mariadb, my)" + raise RuntimeError( + f"Unknown LOGSTASHUI_DB_ENGINE={raw!r}. Supported: {supported}." + ) + return _ENGINE_ALIASES[key] + + +def _import_or_raise(module: str, extra: str): + try: + return __import__(module) + except ImportError as exc: + raise RuntimeError( + f"{module} is not installed. Install with: uv pip install 'LogstashUI[{extra}]' " + f"(Docker/K8s image already includes LogstashUI[databases])." + ) from exc + + +def _env(name: str, default: str = "") -> str: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip() + + +def _env_int(name: str, default: int) -> int: + raw = _env(name, "") + if raw == "": + return default + try: + return int(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer, not {raw!r}.") from exc + + +def _require(names: list[str]) -> None: + missing = [n for n in names if not _env(n)] + if missing: + raise RuntimeError( + "Missing required database settings: " + ", ".join(missing) + ) + def build_databases(data_dir: Path) -> dict: - engine = (os.environ.get("LOGSTASHUI_DB_ENGINE") or "sqlite").strip().lower() - if engine in ("", "sqlite", "sqlite3"): + engine = canonical_engine(os.environ.get("LOGSTASHUI_DB_ENGINE")) + conn_max_age = _env_int("LOGSTASHUI_DB_CONN_MAX_AGE", 60) + health = env_bool("LOGSTASHUI_DB_CONN_HEALTH_CHECKS", True) + + if engine == "sqlite": + name = _env("LOGSTASHUI_DB_NAME") + db_name = Path(name) if name else Path(data_dir) / "db.sqlite3" return { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": Path(data_dir) / "db.sqlite3", + "NAME": db_name, + "CONN_MAX_AGE": conn_max_age, + "CONN_HEALTH_CHECKS": health, "OPTIONS": { - # busy_timeout before journal_mode so the busy handler is - # armed before WAL mode is asserted on the connection. "init_command": ( "PRAGMA busy_timeout=20000;" "PRAGMA journal_mode=WAL;" @@ -32,8 +93,109 @@ def build_databases(data_dir: Path) -> dict: }, } } - raise RuntimeError( - f"LOGSTASHUI_DB_ENGINE={engine!r} is not implemented. " - "Supported: sqlite. PostgreSQL and MySQL are planned; keep a PVC on " - "LOGSTASHUI_DATA_DIR even after those engines land (TLS and secrets)." - ) + + if engine == "postgresql": + _import_or_raise("psycopg", "postgres") + _require(["LOGSTASHUI_DB_HOST", "LOGSTASHUI_DB_USER"]) + sslmode = _env("LOGSTASHUI_DB_SSLMODE", "prefer") or "prefer" + options: dict = {"sslmode": sslmode} + ca = _env("LOGSTASHUI_DB_SSL_CA") + if ca: + options["sslrootcert"] = ca + return { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": _env("LOGSTASHUI_DB_NAME", "logstashui") or "logstashui", + "USER": _env("LOGSTASHUI_DB_USER"), + "PASSWORD": _env("LOGSTASHUI_DB_PASSWORD"), + "HOST": _env("LOGSTASHUI_DB_HOST"), + "PORT": _env("LOGSTASHUI_DB_PORT", "5432") or "5432", + "CONN_MAX_AGE": conn_max_age, + "CONN_HEALTH_CHECKS": health, + "OPTIONS": options, + } + } + + pymysql = _import_or_raise("pymysql", "mysql") + if pymysql is not None: + # Django 6 MySQL backend requires Database.version_info >= (2, 2, 1) + # (mysqlclient). PyMySQL reports ~1.1.1, so spoof before the shim. + pymysql.version_info = (2, 2, 1, "final", 0) + pymysql.install_as_MySQLdb() + _require(["LOGSTASHUI_DB_HOST", "LOGSTASHUI_DB_USER"]) + options = { + "charset": "utf8mb4", + "init_command": ( + "SET sql_mode='STRICT_TRANS_TABLES', " + "NAMES utf8mb4 COLLATE utf8mb4_bin" + ), + } + ca = _env("LOGSTASHUI_DB_SSL_CA") + if ca: + options["ssl"] = {"ca": ca} + return { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": _env("LOGSTASHUI_DB_NAME", "logstashui") or "logstashui", + "USER": _env("LOGSTASHUI_DB_USER"), + "PASSWORD": _env("LOGSTASHUI_DB_PASSWORD"), + "HOST": _env("LOGSTASHUI_DB_HOST"), + "PORT": _env("LOGSTASHUI_DB_PORT", "3306") or "3306", + "CONN_MAX_AGE": conn_max_age, + "CONN_HEALTH_CHECKS": health, + "OPTIONS": options, + "TEST": { + "CHARSET": "utf8mb4", + "COLLATION": "utf8mb4_bin", + }, + } + } + + +def check_server_version(connection) -> None: + """Fail-fast if the server is below documented floors. No-op for SQLite.""" + vendor = getattr(connection, "vendor", "") + if vendor == "postgresql": + pg_version = int(getattr(connection, "pg_version", 0) or 0) + if pg_version < 140000: + raise RuntimeError( + f"PostgreSQL 14+ is required (server_version_num={pg_version})." + ) + return + if vendor != "mysql": + return + is_mariadb = bool(getattr(connection, "mysql_is_mariadb", False)) + info = (getattr(connection, "mysql_server_info", "") or "").lower() + if "mariadb" in info: + is_mariadb = True + if hasattr(connection, "get_database_version"): + tup = connection.get_database_version() + else: + tup = (0, 0, 0) + major_minor = (int(tup[0]), int(tup[1])) + if is_mariadb and major_minor < (10, 6): + raise RuntimeError( + f"MariaDB 10.6+ is required (server={getattr(connection, 'mysql_server_info', tup)})." + ) + if not is_mariadb and major_minor < (8, 0): + raise RuntimeError( + f"MySQL 8.0+ is required (server={getattr(connection, 'mysql_server_info', tup)})." + ) + + +def ensure_psycopg_gevent(waiting_module=None): + """Use psycopg wait_select so gunicorn gevent's patched select is cooperative. + + gunicorn --worker-class gevent monkey-patches select before loading WSGI. + psycopg 3.1.14+ also auto-detects that; assigning wait_select makes it explicit. + """ + waiting = waiting_module + if waiting is None: + try: + from psycopg import waiting as waiting + except ImportError: + return None + wait_select = getattr(waiting, "wait_select", None) + if wait_select is not None: + waiting.wait = wait_select + return waiting diff --git a/src/logstashui/LogstashUI/insecure_http.py b/src/logstashui/LogstashUI/insecure_http.py new file mode 100644 index 00000000..bbad2da3 --- /dev/null +++ b/src/logstashui/LogstashUI/insecure_http.py @@ -0,0 +1,61 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +"""LOGSTASHUI_INSECURE_HTTP helpers. Env-only — safe before django.setup().""" + +from __future__ import annotations + +import logging +import os + +from LogstashUI.config import env_bool + +logger = logging.getLogger(__name__) + +INSECURE_HTTP_WARNING = ( + "LOGSTASHUI_INSECURE_HTTP=true: UI and agent connections are plain HTTP. " + "Product CA and UI certificates will not be generated. " + "Automatic TLS is the supported path. LOGSTASHUI_TLS is overridden." +) + +_FALSE = ("0", "false", "no", "off") + + +def insecure_http() -> bool: + return env_bool("LOGSTASHUI_INSECURE_HTTP", False) + + +def force_http_url(url: str | None, enabled: bool | None = None) -> str | None: + if enabled is None: + enabled = insecure_http() + if not enabled or not url: + return url + if url[:8].lower() == "https://": + return "http://" + url[8:] + return url + + +def tls_enabled(tls_env: str | None = None, insecure: bool | None = None) -> bool: + if insecure is None: + insecure = insecure_http() + if insecure: + return False + if tls_env is None: + tls_env = os.environ.get("LOGSTASHUI_TLS", "true") + return (tls_env or "true").strip().lower() not in _FALSE + + +def force_http_origins( + origins: list[str], enabled: bool | None = None +) -> list[str]: + return [force_http_url(o, enabled=enabled) or o for o in origins] + + +def secure_cookies(*, debug: bool, insecure: bool) -> bool: + return (not debug) and (not insecure) + + +def warn_if_enabled(log: logging.Logger | None = None) -> None: + if insecure_http(): + (log or logger).warning(INSECURE_HTTP_WARNING) diff --git a/src/logstashui/LogstashUI/logging_config.py b/src/logstashui/LogstashUI/logging_config.py index 073688b1..1adf9127 100644 --- a/src/logstashui/LogstashUI/logging_config.py +++ b/src/logstashui/LogstashUI/logging_config.py @@ -6,11 +6,46 @@ from __future__ import annotations +import logging.handlers import os +import sys _ALLOWED = ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") +class WindowsSafeRotatingFileHandler(logging.handlers.RotatingFileHandler): + """RotatingFileHandler that handles Windows file-locking during rotation gracefully. + + On Windows, os.rename() raises PermissionError (WinError 32) when another + thread still holds the log file open at the moment rotation is triggered. + Python's logging machinery re-raises this from emit(), producing a flood of + '--- Logging error ---' lines that can crash a running instance. + + This subclass overrides rotate() to catch PermissionError and skip the + current rotation cycle instead of propagating the error. The file continues + to be written to, and the next emit() that triggers shouldRollover() will + attempt rotation again. + """ + + def rotate(self, source: str, dest: str) -> None: + try: + super().rotate(source, dest) + except PermissionError: + # Another thread holds the file open; skip this rotation cycle. + # The next shouldRollover check will retry. + pass + + +# Resolved at import time so settings.py can reference it as a dotted class +# path string. On Windows we use the safe subclass above; on POSIX platforms +# os.rename() is atomic on open files so the stock handler is fine. +ROTATING_FILE_HANDLER_CLASS = ( + "LogstashUI.logging_config.WindowsSafeRotatingFileHandler" + if sys.platform == "win32" + else "logging.handlers.RotatingFileHandler" +) + + def resolve_log_level(name: str, *, default: str = "INFO") -> str: raw = (os.environ.get(name) or "").strip().upper() if not raw: diff --git a/src/logstashui/LogstashUI/migrate_engine.py b/src/logstashui/LogstashUI/migrate_engine.py new file mode 100644 index 00000000..1da46aa3 --- /dev/null +++ b/src/logstashui/LogstashUI/migrate_engine.py @@ -0,0 +1,215 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +"""BETA: copy SQLite data to PostgreSQL or MySQL. Stops gunicorn; does not restart serve.""" + +from __future__ import annotations + +import os +import signal +import sqlite3 +import subprocess +import sys +import tempfile +import time +from argparse import Namespace +from pathlib import Path + +from LogstashUI.database import canonical_engine +from LogstashUI.paths import resolve_data_dir + +_TARGET_ENGINES = frozenset({"postgresql", "mysql"}) +_DUMPDATA_EXCLUDES = ("contenttypes", "auth.permission", "sessions") + + +def _with_package_pythonpath(env: dict[str, str]) -> dict[str, str]: + """Subprocesses must import this tree, not a stale site-packages copy.""" + pkg_root = str(Path(__file__).resolve().parent.parent) + existing = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = pkg_root if not existing else os.pathsep.join([pkg_root, existing]) + return env + + +def run_manage(argv: list[str], extra_env: dict[str, str]) -> None: + env = os.environ.copy() + env.update(extra_env) + env.setdefault("DJANGO_SETTINGS_MODULE", "LogstashUI.settings") + _with_package_pythonpath(env) + code = ( + "import sys; from django.core.management import execute_from_command_line; " + "execute_from_command_line(['logstashui'] + sys.argv[1:])" + ) + cmd = [sys.executable, "-c", code, *argv] + proc = subprocess.run(cmd, env=env, check=False) + if proc.returncode != 0: + raise SystemExit(proc.returncode) + + +def wal_checkpoint(db_path: Path) -> None: + conn = sqlite3.connect(str(db_path)) + try: + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + finally: + conn.close() + + +def stop_gunicorn(pidfile: Path) -> None: + raw = pidfile.read_text(encoding="utf-8").strip() + try: + pid = int(raw) + except ValueError: + pid = 0 + if pid <= 0: + print(f"invalid gunicorn pid {raw!r} in {pidfile}", file=sys.stderr) + raise SystemExit(1) + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pidfile.unlink(missing_ok=True) + return + deadline = time.monotonic() + 30 + while True: + time.sleep(0.2) + try: + os.kill(pid, 0) + except ProcessLookupError: + pidfile.unlink(missing_ok=True) + return + if time.monotonic() >= deadline: + print( + f"gunicorn pid {pid} did not exit within 30s after SIGTERM", + file=sys.stderr, + ) + raise SystemExit(1) + + +def write_env_file(path: Path, engine: str) -> None: + """Upsert LOGSTASHUI_DB_* keys (never password). Second run does not duplicate.""" + assignments = { + "LOGSTASHUI_DB_ENGINE": engine, + "LOGSTASHUI_DB_NAME": os.environ.get("LOGSTASHUI_DB_NAME") or "", + "LOGSTASHUI_DB_HOST": os.environ.get("LOGSTASHUI_DB_HOST") or "", + "LOGSTASHUI_DB_PORT": os.environ.get("LOGSTASHUI_DB_PORT") or "", + "LOGSTASHUI_DB_USER": os.environ.get("LOGSTASHUI_DB_USER") or "", + } + assignments = {key: value for key, value in assignments.items() if value} + existing = path.read_text(encoding="utf-8") if path.is_file() else "" + kept = [] + for line in existing.splitlines(): + key = line.split("=", 1)[0] if "=" in line else "" + if key in assignments: + continue + kept.append(line) + while kept and kept[-1] == "": + kept.pop() + kept.extend(f"{key}={value}" for key, value in assignments.items()) + path.write_text("\n".join(kept) + "\n", encoding="utf-8") + + +def _reset_postgres_sequences(extra_env: dict[str, str]) -> None: + """Apply sqlsequencereset via the Django connection (no psql CLI).""" + env = os.environ.copy() + env.update(extra_env) + env.setdefault("DJANGO_SETTINGS_MODULE", "LogstashUI.settings") + _with_package_pythonpath(env) + reset_code = ( + "import os, django\n" + "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'LogstashUI.settings')\n" + "django.setup()\n" + "from django.apps import apps\n" + "from django.core.management.color import no_style\n" + "from django.db import connection\n" + "models = [\n" + " m for c in apps.get_app_configs() if c.models_module\n" + " for m in c.get_models(include_auto_created=True)\n" + "]\n" + "sql_list = connection.ops.sequence_reset_sql(no_style(), models)\n" + "with connection.cursor() as cursor:\n" + " for sql in sql_list:\n" + " cursor.execute(sql)\n" + ) + proc = subprocess.run( + [sys.executable, "-c", reset_code], + env=env, + check=False, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + sys.stderr.write(proc.stderr or "") + raise SystemExit(proc.returncode) + + +def cmd_migrate_engine(args: Namespace) -> int: + if not args.i_have_a_backup: + print( + "Refusing to run migrate-engine without --i-have-a-backup. " + "Back up db.sqlite3 (and the WAL) before copying data to another engine.", + file=sys.stderr, + ) + raise SystemExit(2) + + try: + engine = canonical_engine(getattr(args, "to", None)) + except RuntimeError as exc: + print(str(exc), file=sys.stderr) + raise SystemExit(2) from exc + if engine not in _TARGET_ENGINES: + print( + f"migrate-engine --to must be postgresql or mysql, not {engine}.", + file=sys.stderr, + ) + raise SystemExit(2) + + data_dir = resolve_data_dir(migrate_legacy=False) + sqlite_path = data_dir / "db.sqlite3" + if not sqlite_path.is_file(): + print( + f"SQLite database not found: {sqlite_path} (expected db.sqlite3).", + file=sys.stderr, + ) + raise SystemExit(1) + + print( + "BETA: migrate-engine stops gunicorn and does not restart serve. " + "Start `logstashui serve` yourself after verifying the target database.", + file=sys.stderr, + ) + + pidfile = Path(args.pid) if args.pid is not None else data_dir / "gunicorn.pid" + if pidfile.is_file(): + stop_gunicorn(pidfile) + + wal_checkpoint(sqlite_path) + + dump_fd, dump_name = tempfile.mkstemp(prefix="logstashui-migrate-", suffix=".json") + os.close(dump_fd) + dump_path = Path(dump_name) + sqlite_env = { + "LOGSTASHUI_DB_ENGINE": "sqlite", + "LOGSTASHUI_DB_NAME": str(sqlite_path), + } + target_env = {"LOGSTASHUI_DB_ENGINE": engine} + dump_argv = [ + "dumpdata", + "--natural-foreign", + "--natural-primary", + "--output", + str(dump_path), + ] + for label in _DUMPDATA_EXCLUDES: + dump_argv.extend(["--exclude", label]) + try: + run_manage(dump_argv, sqlite_env) + run_manage(["migrate", "--noinput"], target_env) + run_manage(["loaddata", str(dump_path)], target_env) + if engine == "postgresql": + _reset_postgres_sequences(target_env) + finally: + dump_path.unlink(missing_ok=True) + + if args.write_env is not None: + write_env_file(Path(args.write_env), engine) + + return 0 diff --git a/src/logstashui/LogstashUI/packaging/logstashui.default b/src/logstashui/LogstashUI/packaging/logstashui.default index fc10229d..1a2db93d 100644 --- a/src/logstashui/LogstashUI/packaging/logstashui.default +++ b/src/logstashui/LogstashUI/packaging/logstashui.default @@ -16,6 +16,11 @@ DEBUG=false LOGSTASHUI_BIND=0.0.0.0:8443 LOGSTASHUI_WORKERS=2 LOGSTASHUI_TLS=true +# NOT recommended. Automatic product TLS works out of the box. +# If you must run the UI and all agent connections over plain HTTP +# (no CA, no certs), set true. Overrides LOGSTASHUI_TLS. Do not enable +# without cause. Agents need their own TLS-off flag. +# LOGSTASHUI_INSECURE_HTTP=false ALLOWED_HOSTS=* # CSRF_TRUSTED_ORIGINS=https://logstashui.example.com:8443 @@ -43,10 +48,21 @@ LOGSTASHUI_INCLUDE_CA_FINGERPRINT=true # In-app markdown docs (wheel ships a copy; checkout uses the git tree) # LOGSTASHUI_DOCS_DIR= -# Future database backends (not implemented — sqlite only for now) +# Database. Unset engine = sqlite at $LOGSTASHUI_DATA_DIR/db.sqlite3. +# DATA_DIR is still required for TLS, secrets, logs, staticfiles. # LOGSTASHUI_DB_ENGINE=sqlite # LOGSTASHUI_DB_NAME=logstashui # LOGSTASHUI_DB_HOST= # LOGSTASHUI_DB_PORT= # LOGSTASHUI_DB_USER= # LOGSTASHUI_DB_PASSWORD= +# LOGSTASHUI_DB_SSLMODE=prefer +# LOGSTASHUI_DB_SSL_CA= +# LOGSTASHUI_DB_CONN_MAX_AGE=60 +# LOGSTASHUI_DB_CONN_HEALTH_CHECKS=true +# +# Native extras: uv pip install 'LogstashUI[postgres]' or 'LogstashUI[mysql]' +# or 'LogstashUI[databases]'. The container image already includes both drivers. +# +# Create MySQL/MariaDB with utf8mb4_bin: +# CREATE DATABASE logstashui CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; diff --git a/src/logstashui/LogstashUI/paths.py b/src/logstashui/LogstashUI/paths.py index 437e274f..62a4ec4d 100644 --- a/src/logstashui/LogstashUI/paths.py +++ b/src/logstashui/LogstashUI/paths.py @@ -4,7 +4,8 @@ """Resolve runtime data / logs directories (no Django import). -Precedence: LOGSTASHUI_DATA_DIR / LOGSTASHUI_LOGS_DIR → default. +Precedence: LOGSTASHUI_DATA_DIR / LOGSTASHUI_LOGS_DIR / LOGSTASHUI_LOGSTASH_DIR +→ default. Default data root is ``$(pwd)/logstashui_data``. Pytest keeps using ``/data`` so test runs do not touch a checkout bind-mount. @@ -78,6 +79,22 @@ def resolve_logs_dir(data_dir: Optional[Path] = None) -> Path: return chosen +def resolve_logstash_dir(data_dir: Optional[Path] = None) -> Path: + """Cache root for proxied Logstash release tarballs. + + Deliberately a sibling of ``staticfiles``, never a child: STATIC_ROOT is + served by WhiteNoise at ``/static/``, which is in LOGIN_REQUIRED_IGNORE_PATHS, + so anything under it is an unauthenticated public download. ``collectstatic`` + also runs on every ``serve`` and would churn over half-gigabyte files. + """ + env = os.environ.get("LOGSTASHUI_LOGSTASH_DIR") + chosen = _coerce_path(env, relative_to=Path.cwd()) + if chosen is None: + root = data_dir if data_dir is not None else resolve_data_dir() + chosen = root / "logstashes" + return chosen + + def maybe_migrate_legacy_data(dest: Path) -> None: """Copy src/logstashui/data → dest when dest has no sqlite and legacy does.""" try: diff --git a/src/logstashui/LogstashUI/settings.py b/src/logstashui/LogstashUI/settings.py index bed0bb3b..69547e49 100644 --- a/src/logstashui/LogstashUI/settings.py +++ b/src/logstashui/LogstashUI/settings.py @@ -1,6 +1,6 @@ -#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -#or more contributor license agreements. Licensed under the Elastic License; -#you may not use this file except in compliance with the Elastic License. +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. """ Django settings for logstashui project. @@ -18,10 +18,23 @@ import os, platform from importlib.metadata import version, PackageNotFoundError from Common.encryption import get_django_secret_key -from .config import CONFIG +from .config import CONFIG, merge_allowed_hosts +from .insecure_http import ( + force_http_origins, + force_http_url, + insecure_http, + secure_cookies, + tls_enabled, + warn_if_enabled, +) from .database import build_databases -from .logging_config import resolve_django_log_levels, resolve_log_level -from .paths import resolve_data_dir, resolve_docs_dir, resolve_logs_dir +from .logging_config import resolve_django_log_levels, resolve_log_level, ROTATING_FILE_HANDLER_CLASS +from .paths import ( + resolve_data_dir, + resolve_docs_dir, + resolve_logs_dir, + resolve_logstash_dir, +) # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -48,11 +61,12 @@ # SECURITY WARNING: don't run with debug turned on in production! # Set DEBUG=False in production via environment variable -DEBUG = os.environ.get('DEBUG', 'True').lower() in ('true', '1', 'yes') +DEBUG = os.environ.get("DEBUG", "True").lower() in ("true", "1", "yes") # SECURITY WARNING: Set ALLOWED_HOSTS to your domain(s) in production # Example: ALLOWED_HOSTS=yourdomain.com,www.yourdomain.com -ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '*').split(',') +ALLOWED_HOSTS = merge_allowed_hosts() + def _get_version(): """Get version from installed package metadata or pyproject.toml""" @@ -61,87 +75,91 @@ def _get_version(): except PackageNotFoundError: try: import tomllib + pyproject_path = PROJECT_ROOT / "pyproject.toml" if pyproject_path.exists(): with open(pyproject_path, "rb") as f: pyproject_data = tomllib.load(f) - return pyproject_data.get("project", {}).get("version", "0.0.0+unknown") + return pyproject_data.get("project", {}).get( + "version", "0.0.0+unknown" + ) except Exception: pass return "0.0.0+unknown" + __VERSION__ = _get_version() -__PREFERRED_LS_AGENT_VERSION__ = "0.5.1" +__PREFERRED_LS_AGENT_VERSION__ = "0.5.2" # Application definition INSTALLED_APPS = [ # Shipped with Django - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", # Apps of logstashui - 'PipelineManager', - 'Management', - 'Utilities', - 'SNMP', - 'Monitoring', - 'Site', - 'Documentation', - 'AI', - + "PipelineManager", + "Management", + "Utilities", + "SNMP", + "Monitoring", + "Site", + "Documentation", + "AI", # Frameworks - 'django_htmx', - 'tailwind', - 'theme' # Belongs to tailwind + "django_htmx", + "tailwind", + "theme", # Belongs to tailwind ] TAILWIND_APP_NAME = "theme" - - MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'whitenoise.middleware.WhiteNoiseMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django_htmx.middleware.HtmxMiddleware', - 'login_required.middleware.LoginRequiredMiddleware', - 'Common.middleware.SecurityHeadersMiddleware', - + "django.middleware.security.SecurityMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + # Must precede CsrfViewMiddleware: it exempts verified API-token requests + # from CSRF. Browser and agent traffic pass through untouched. + "Common.middleware.ApiTokenCsrfMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + # Must follow AuthenticationMiddleware, which would otherwise overwrite + # request.user with the lazy session user. + "Common.middleware.ApiTokenUserMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django_htmx.middleware.HtmxMiddleware", + "login_required.middleware.LoginRequiredMiddleware", + "Common.middleware.SecurityHeadersMiddleware", ] -ROOT_URLCONF = 'LogstashUI.urls' +ROOT_URLCONF = "LogstashUI.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - ], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'Common.context_processors.version_update_info', - 'Common.context_processors.navigation_highlight', - 'Common.context_processors.experimental_mode', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + "Common.context_processors.version_update_info", + "Common.context_processors.navigation_highlight", + "Common.context_processors.experimental_mode", ], }, }, ] -WSGI_APPLICATION = 'LogstashUI.wsgi.application' +WSGI_APPLICATION = "LogstashUI.wsgi.application" # Database @@ -155,16 +173,16 @@ def _get_version(): AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] @@ -172,9 +190,9 @@ def _get_version(): # Internationalization # https://docs.djangoproject.com/en/5.2/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -184,12 +202,12 @@ def _get_version(): # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.2/howto/static-files/ -STATIC_URL = '/static/' +STATIC_URL = "/static/" # Default primary key field type # https://docs.djangoproject.com/en/5.2/ref/settings/#default-auto-field -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" STATIC_ROOT = DATA_DIR / "staticfiles" @@ -209,22 +227,25 @@ def _get_version(): if DEBUG: # Add django_browser_reload only in DEBUG mode - INSTALLED_APPS += ['django_browser_reload'] + INSTALLED_APPS += ["django_browser_reload"] MIDDLEWARE += [ "django_browser_reload.middleware.BrowserReloadMiddleware", ] -NO_AUTH_MODE = LOGSTASHUI_CONFIG.get('no_auth', {}).get('enabled', False) +NO_AUTH_MODE = LOGSTASHUI_CONFIG.get("no_auth", {}).get("enabled", False) if NO_AUTH_MODE: import logging + logging.getLogger(__name__).warning( "*** NO_AUTH MODE IS ENABLED — All authentication is bypassed. " "Do not use in production! ***" ) - _auth_idx = MIDDLEWARE.index('django.contrib.auth.middleware.AuthenticationMiddleware') - MIDDLEWARE.insert(_auth_idx + 1, 'Common.middleware.NoAuthMiddleware') + _auth_idx = MIDDLEWARE.index( + "django.contrib.auth.middleware.AuthenticationMiddleware" + ) + MIDDLEWARE.insert(_auth_idx + 1, "Common.middleware.NoAuthMiddleware") LOGIN_REDIRECT_URL = "/" LOGOUT_REDIRECT_URL = "/Management/Login/" @@ -246,6 +267,9 @@ def _get_version(): "/ConnectionManager/GetConfigChanges", "/ConnectionManager/IssueServerCert/", "/ConnectionManager/IssueServerCert", + # Prefix match: covers /LogstashArtifact//. + # The view authenticates the agent key itself. + "/ConnectionManager/LogstashArtifact/", ] # Session Configuration @@ -255,67 +279,90 @@ def _get_version(): SESSION_EXPIRE_AT_BROWSER_CLOSE = False # Allow persistent sessions # Proxy/HTTPS settings for nginx reverse proxy -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") USE_X_FORWARDED_HOST = True USE_X_FORWARDED_PORT = True # CSRF Trusted Origins - configurable via environment variable # For self-hosted deployments, users should set CSRF_TRUSTED_ORIGINS env var # Example: CSRF_TRUSTED_ORIGINS=https://myserver.com,https://192.168.1.100 -csrf_origins_env = os.environ.get('CSRF_TRUSTED_ORIGINS', '') +csrf_origins_env = os.environ.get("CSRF_TRUSTED_ORIGINS", "") if csrf_origins_env: - CSRF_TRUSTED_ORIGINS = [origin.strip() for origin in csrf_origins_env.split(',')] + CSRF_TRUSTED_ORIGINS = [origin.strip() for origin in csrf_origins_env.split(",") if origin.strip()] else: # Default for local development CSRF_TRUSTED_ORIGINS = [ - 'http://localhost:8080', - 'http://127.0.0.1:8080', - 'https://localhost', - 'https://127.0.0.1', + "http://localhost:8080", + "http://127.0.0.1:8080", + "https://localhost", + "https://127.0.0.1", ] +INSECURE_HTTP = insecure_http() +if INSECURE_HTTP: + CSRF_TRUSTED_ORIGINS = force_http_origins(CSRF_TRUSTED_ORIGINS, enabled=True) + # Security Headers # These settings protect against common web vulnerabilities # Only enforce in production (when DEBUG=False) +_tls_env = os.environ.get("LOGSTASHUI_TLS", "true") +TLS_ENABLED = tls_enabled(tls_env=_tls_env, insecure=INSECURE_HTTP) +if INSECURE_HTTP: + warn_if_enabled() + +_secure_cookies = secure_cookies(debug=DEBUG, insecure=INSECURE_HTTP) + if not DEBUG: - # Ensure cookies are only sent over HTTPS - SESSION_COOKIE_SECURE = True - CSRF_COOKIE_SECURE = True - - # HTTP Strict Transport Security (HSTS) - # Tells browsers to only access the site via HTTPS for the next year - SECURE_HSTS_SECONDS = 31536000 # 1 year - SECURE_HSTS_INCLUDE_SUBDOMAINS = True - SECURE_HSTS_PRELOAD = True - - # Redirect all HTTP requests to HTTPS at Django level - # Note: nginx already does this, but this adds defense in depth - SECURE_SSL_REDIRECT = True - - # Prevent the site from being embedded in iframes (clickjacking protection) - X_FRAME_OPTIONS = 'DENY' - - # Prevent browsers from guessing content types + SESSION_COOKIE_SECURE = _secure_cookies + CSRF_COOKIE_SECURE = _secure_cookies + if _secure_cookies: + SECURE_HSTS_SECONDS = 31536000 # 1 year + SECURE_HSTS_INCLUDE_SUBDOMAINS = True + SECURE_HSTS_PRELOAD = True + else: + SECURE_HSTS_SECONDS = 0 + SECURE_HSTS_INCLUDE_SUBDOMAINS = False + SECURE_HSTS_PRELOAD = False + SECURE_SSL_REDIRECT = TLS_ENABLED + X_FRAME_OPTIONS = "DENY" SECURE_CONTENT_TYPE_NOSNIFF = True - - # Enable browser's XSS filtering SECURE_BROWSER_XSS_FILTER = True else: # Development mode - allow HTTP for local testing SESSION_COOKIE_SECURE = False CSRF_COOKIE_SECURE = False SECURE_SSL_REDIRECT = False - X_FRAME_OPTIONS = 'SAMEORIGIN' + X_FRAME_OPTIONS = "SAMEORIGIN" # logstashagent Configuration # URL for the logstashagent API (HTTPS direct; no nginx). # Override with LOGSTASH_AGENT_URL. Compose sets https://logstashagent:9500. if DEBUG: - LOGSTASH_AGENT_URL = os.environ.get('LOGSTASH_AGENT_URL', 'http://127.0.0.1:9500') + LOGSTASH_AGENT_URL = os.environ.get("LOGSTASH_AGENT_URL", "http://127.0.0.1:9500") else: LOGSTASH_AGENT_URL = os.environ.get( - 'LOGSTASH_AGENT_URL', 'https://logstashagent:9500' + "LOGSTASH_AGENT_URL", "https://logstashagent:9500" ) +LOGSTASH_AGENT_URL = force_http_url(LOGSTASH_AGENT_URL, enabled=INSECURE_HTTP) + +# Logstash tarball proxy +# Cache of Logstash release tarballs served to agents whose policy sets +# logstash_via_ui. See PipelineManager/artifacts.py. +LOGSTASH_DIR = resolve_logstash_dir(DATA_DIR) +LOGSTASH_DIR.mkdir(parents=True, exist_ok=True) + +# Upstream fetches in flight cluster-wide, counted in the DB (the only state +# shared between gunicorn workers -- there is no CACHES backend configured). +LOGSTASH_ARTIFACT_MAX_UPSTREAM = int( + os.environ.get("LOGSTASHUI_ARTIFACT_MAX_UPSTREAM", "2") +) +# Concurrent agent downloads *per worker*. Effective total is this value times +# LOGSTASHUI_WORKERS. Deliberately not a global divided by worker count: that +# truncates to zero and makes the knob lie. +LOGSTASH_ARTIFACT_MAX_SERVE_PER_WORKER = int( + os.environ.get("LOGSTASHUI_ARTIFACT_MAX_SERVE_PER_WORKER", "4") +) +LOGSTASH_ARTIFACT_DEFAULT_BASE_URL = "https://artifacts.elastic.co/downloads/logstash" # Logging Configuration # https://docs.djangoproject.com/en/5.2/topics/logging/ @@ -331,44 +378,46 @@ def _get_version(): DJANGO_LOGGER_LEVEL, DJANGO_REQUEST_LOG_LEVEL = resolve_django_log_levels() LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'verbose': { - 'format': '[{levelname}] {asctime} {name} {module}.{funcName}: {message}', - 'style': '{', - 'datefmt': '%Y-%m-%d %H:%M:%S', + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "[{levelname}] {asctime} {name} {module}.{funcName}: {message}", + "style": "{", + "datefmt": "%Y-%m-%d %H:%M:%S", }, }, - 'handlers': { - 'console': { - 'level': LOGSTASHUI_LOG_LEVEL, - 'class': 'logging.StreamHandler', - 'formatter': 'verbose', + "handlers": { + "console": { + "level": LOGSTASHUI_LOG_LEVEL, + "class": "logging.StreamHandler", + "formatter": "verbose", + }, + "file": { + "level": LOGSTASHUI_LOG_LEVEL, + "class": ROTATING_FILE_HANDLER_CLASS, + "filename": LOGS_DIR / "logstashui.log", + "maxBytes": 1024 * 1024 * 10, # 10 MB + "backupCount": 5, + "formatter": "verbose", + "delay": True, + "encoding": "utf-8", }, - 'file': { - 'level': LOGSTASHUI_LOG_LEVEL, - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': LOGS_DIR / 'logstashui.log', - 'maxBytes': 1024 * 1024 * 10, # 10 MB - 'backupCount': 5, - 'formatter': 'verbose', - } }, - 'loggers': { - 'django': { - 'handlers': ['console', 'file'], - 'level': DJANGO_LOGGER_LEVEL, - 'propagate': False, + "loggers": { + "django": { + "handlers": ["console", "file"], + "level": DJANGO_LOGGER_LEVEL, + "propagate": False, }, - 'django.request': { - 'handlers': ['console', 'file'], - 'level': DJANGO_REQUEST_LOG_LEVEL, - 'propagate': False, + "django.request": { + "handlers": ["console", "file"], + "level": DJANGO_REQUEST_LOG_LEVEL, + "propagate": False, }, }, - 'root': { - 'handlers': ['console', 'file'], - 'level': LOGSTASHUI_LOG_LEVEL, + "root": { + "handlers": ["console", "file"], + "level": LOGSTASHUI_LOG_LEVEL, }, -} \ No newline at end of file +} diff --git a/src/logstashui/LogstashUI/telemetry.py b/src/logstashui/LogstashUI/telemetry.py new file mode 100644 index 00000000..2f0c1a09 --- /dev/null +++ b/src/logstashui/LogstashUI/telemetry.py @@ -0,0 +1,148 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +"""Optional OpenTelemetry bootstrap. + +Enabled by ``LOGSTASHUI_OTEL=true`` and the ``LogstashUI[otel]`` extra. Absent +either, every entry point here is a no-op — LogstashUI ships as an air-gapped +wheel bundle and an offline Docker image, so tracing cannot be a hard dependency. + +Two constraints come from running under gunicorn's gevent worker, and both are +easy to get wrong: + +* **The gRPC exporter is unusable.** ``grpcio`` runs its own C-core native + threads, which gevent cannot monkey-patch; it deadlocks or silently drops + spans. This module only ever configures the HTTP/protobuf exporter, which goes + through ``requests`` and is fully cooperative. +* **``BatchSpanProcessor`` is required, not optional.** Its worker is a + ``threading.Thread``, which under gevent is a greenlet, and it blocks on a + patched ``Condition.wait`` — so it is safe. ``SimpleSpanProcessor`` would put a + synchronous OTLP round-trip inside every request, including agent check-ins. + +Initialization belongs in ``wsgi.build_application()``: once per worker, after +the fork and after monkey-patching, and **before** ``get_wsgi_application()`` +so ``DjangoInstrumentor`` can mutate ``MIDDLEWARE`` before the handler +snapshots it. ``settings.py`` is imported by ``manage.py``, every migration, +and every ``cli.py`` management command, so initializing there would spin up a +tracer provider for ``collectstatic``. +""" + +import logging +import os +import threading +import time + +logger = logging.getLogger(__name__) + +_initialized = False + +#: How often the hub-lag probe samples. Cheap: one greenlet, one sleep. +_LAG_INTERVAL = 1.0 + + +def _enabled(): + return os.environ.get('LOGSTASHUI_OTEL', 'false').lower() in ('true', '1', 'yes') + + +def init_telemetry(): + """Set up tracing and metrics. Returns True when instrumentation is live.""" + global _initialized + if _initialized or not _enabled(): + return False + + try: + from opentelemetry import metrics, trace # type: ignore[import-not-found] + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( # type: ignore[import-not-found] + OTLPMetricExporter, + ) + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( # type: ignore[import-not-found] + OTLPSpanExporter, + ) + from opentelemetry.instrumentation.django import DjangoInstrumentor # type: ignore[import-not-found] + from opentelemetry.instrumentation.requests import RequestsInstrumentor # type: ignore[import-not-found] + from opentelemetry.sdk.metrics import MeterProvider # type: ignore[import-not-found] + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader # type: ignore[import-not-found] + from opentelemetry.sdk.resources import Resource # type: ignore[import-not-found] + from opentelemetry.sdk.trace import TracerProvider # type: ignore[import-not-found] + from opentelemetry.sdk.trace.export import BatchSpanProcessor # type: ignore[import-not-found] + except ImportError: + logger.error( + "LOGSTASHUI_OTEL is set but OpenTelemetry is not installed; " + "tracing disabled. Install the 'otel' extra to enable it." + ) + return False + + try: + import django + + if not django.apps.apps.ready: + django.setup() + + resource = Resource.create({ + 'service.name': os.environ.get('OTEL_SERVICE_NAME', 'logstashui'), + }) + + tracer_provider = TracerProvider(resource=resource) + # Batch, never Simple -- see the module docstring. + tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + trace.set_tracer_provider(tracer_provider) + + meter_provider = MeterProvider( + resource=resource, + metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())], + ) + metrics.set_meter_provider(meter_provider) + + # Mutates settings.MIDDLEWARE, so it has to run before the handler is built. + DjangoInstrumentor().instrument() + RequestsInstrumentor().instrument() + + from PipelineManager import artifact_metrics + + artifact_metrics.init(meter_provider) + _start_hub_lag_probe(meter_provider) + + _initialized = True + logger.info("OpenTelemetry instrumentation enabled (OTLP over HTTP)") + return True + except Exception: + # Telemetry must never be the reason a worker fails to boot. + logger.exception("OpenTelemetry initialization failed; continuing without it") + return False + + +def _start_hub_lag_probe(meter_provider): + """Measure how late a greenlet wakes up from a 1-second sleep. + + This is the single most useful signal for capacity decisions and the only one + that sees the problems a span cannot. A greenlet that asks to sleep 1.0 s and + wakes at 1.4 s spent 0.4 s waiting behind something that would not yield -- + a TLS write, a SQLite writer blocked on ``busy_timeout``, a large disk read. + + Reading it: lag stays flat as load rises => the NIC is the ceiling and adding + workers will not help. Lag climbs => greenlets are starving, so more workers + or cores will. + """ + meter = meter_provider.get_meter('logstashui.runtime') + state = {'lag': 0.0} + + def _observe(_options): + from opentelemetry.metrics import Observation # type: ignore[import-not-found] + + return [Observation(state['lag'])] + + meter.create_observable_gauge( + 'logstashui.gevent.hub.lag', + callbacks=[_observe], + unit='s', + description='How far past its deadline a sleeping greenlet actually woke', + ) + + def _probe(): + while True: + start = time.monotonic() + time.sleep(_LAG_INTERVAL) + state['lag'] = max(0.0, time.monotonic() - start - _LAG_INTERVAL) + + threading.Thread(target=_probe, daemon=True).start() diff --git a/src/logstashui/LogstashUI/tests/test_cli.py b/src/logstashui/LogstashUI/tests/test_cli.py deleted file mode 100644 index 2685c113..00000000 --- a/src/logstashui/LogstashUI/tests/test_cli.py +++ /dev/null @@ -1,87 +0,0 @@ -#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -#or more contributor license agreements. Licensed under the Elastic License; -#you may not use this file except in compliance with the Elastic License. - -from argparse import Namespace - -from django.core.management.base import CommandError - -from LogstashUI.cli import build_parser, cmd_serve, install_systemd - - -def test_parser_defaults_to_serve(): - parser = build_parser() - ns = parser.parse_args([]) - assert ns.command == "serve" - - -def test_parser_manage_passthrough(): - parser = build_parser() - ns = parser.parse_args(["manage", "migrate", "--noinput"]) - assert ns.command == "manage" - assert ns.manage_args == ["migrate", "--noinput"] - - -def test_systemd_dry_run_writes_unit_and_default(tmp_path): - result = install_systemd( - output_dir=tmp_path, - exec_start="/usr/bin/logstashui serve", - user="logstashui", - group="logstashui", - data_dir="/var/lib/logstashui", - bind="0.0.0.0:8443", - workers=2, - allowed_hosts="ui.example", - csrf_trusted_origins="https://ui.example:8443", - tls="true", - host_hostname="ui.example", - host_ips="10.0.0.5", - tls_sans="ui.example,10.0.0.5", - agent_ui_url="https://ui.example:8443", - no_auth="false", - dry_run=True, - ) - unit = tmp_path / "logstashui.service" - envf = tmp_path / "logstashui.default" - assert unit.is_file() - assert envf.is_file() - unit_text = unit.read_text() - env_text = envf.read_text() - assert "EnvironmentFile=-/etc/default/logstashui" in unit_text - assert "ExecStart=/usr/bin/logstashui serve" in unit_text - assert "User=logstashui" in unit_text - assert "LOGSTASHUI_DATA_DIR=/var/lib/logstashui" in env_text - assert "LOGSTASHUI_BIND=0.0.0.0:8443" in env_text - assert "LOGSTASHUI_NO_AUTH=false" in env_text - assert result["unit"] == unit - assert result["default"] == envf - - -def test_serve_snmp_commanderror_does_not_abort(monkeypatch): - """execute_from_command_line sys.exits on CommandError; call_command must not.""" - from LogstashUI import cli - - monkeypatch.setattr(cli, "_manage", lambda argv: None) - monkeypatch.setenv("LOGSTASHUI_TLS", "false") - - def fake_call(name, *args, **kwargs): - if name == "sync_snmp_official_data": - raise CommandError("sync failed") - return None - - monkeypatch.setattr("django.core.management.call_command", fake_call) - - exec_called = {} - - def fake_execvp(file, args): - exec_called["file"] = file - raise SystemExit(0) - - monkeypatch.setattr(cli.os, "execvp", fake_execvp) - - ns = Namespace(skip_migrate=False, no_tls=True, bind="127.0.0.1:8443", workers=1) - try: - cmd_serve(ns) - except SystemExit as exc: - assert exc.code == 0 - assert exec_called.get("file") == "gunicorn" diff --git a/src/logstashui/LogstashUI/tests/test_database.py b/src/logstashui/LogstashUI/tests/test_database.py deleted file mode 100644 index 867f1dd6..00000000 --- a/src/logstashui/LogstashUI/tests/test_database.py +++ /dev/null @@ -1,26 +0,0 @@ -#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -#or more contributor license agreements. Licensed under the Elastic License; -#you may not use this file except in compliance with the Elastic License. - -from pathlib import Path - -import pytest - -from LogstashUI.database import build_databases - - -def test_build_databases_sqlite_default(tmp_path, monkeypatch): - monkeypatch.delenv("LOGSTASHUI_DB_ENGINE", raising=False) - db = build_databases(tmp_path) - assert db["default"]["ENGINE"] == "django.db.backends.sqlite3" - assert db["default"]["NAME"] == tmp_path / "db.sqlite3" - assert db["default"]["OPTIONS"]["timeout"] == 20 - - -def test_build_databases_rejects_unimplemented_engine(tmp_path, monkeypatch): - monkeypatch.setenv("LOGSTASHUI_DB_ENGINE", "postgresql") - with pytest.raises(RuntimeError, match="not implemented"): - build_databases(tmp_path) - monkeypatch.setenv("LOGSTASHUI_DB_ENGINE", "mysql") - with pytest.raises(RuntimeError, match="not implemented"): - build_databases(Path(tmp_path)) diff --git a/src/logstashui/LogstashUI/wsgi.py b/src/logstashui/LogstashUI/wsgi.py index 0eeb86e5..461feff2 100644 --- a/src/logstashui/LogstashUI/wsgi.py +++ b/src/logstashui/LogstashUI/wsgi.py @@ -15,11 +15,40 @@ import ssl import sys -from django.core.wsgi import get_wsgi_application - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'LogstashUI.settings') -application = get_wsgi_application() +from LogstashUI.database import ensure_psycopg_gevent + + +def build_application(): + """Init OTEL (mutates MIDDLEWARE) then build the Django WSGI handler. + + gunicorn --worker-class gevent monkey-patches before this module loads. + """ + try: + from LogstashUI.telemetry import init_telemetry + + init_telemetry() + except Exception: + pass + from django.core.wsgi import get_wsgi_application + + return get_wsgi_application() + + +application = build_application() +ensure_psycopg_gevent() + +# A tarball download dies with its worker, so a restart can leave a .part behind. +# Clear them here rather than at import of settings, which also runs for every +# management command. +try: + from PipelineManager.artifacts import sweep_partials + + sweep_partials() +except Exception: + pass + # Quiet gevent/gunicorn spam: clients that reject the product CA (browser # probes, scanners, default-trust Python) abort the handshake with diff --git a/src/logstashui/Management/migrations/0004_settings_logstash_artifact_base_url.py b/src/logstashui/Management/migrations/0004_settings_logstash_artifact_base_url.py new file mode 100644 index 00000000..c3aacc7c --- /dev/null +++ b/src/logstashui/Management/migrations/0004_settings_logstash_artifact_base_url.py @@ -0,0 +1,29 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('Management', '0003_settings_agent_ui_url'), + ] + + operations = [ + migrations.AddField( + model_name='settings', + name='logstash_artifact_base_url', + field=models.CharField( + blank=True, + default='', + help_text=( + 'Upstream source for Logstash release tarballs. Blank uses ' + 'https://artifacts.elastic.co/downloads/logstash. Point this at an ' + 'internal mirror to keep tarball fetches inside your network.' + ), + max_length=512, + ), + ), + ] diff --git a/src/logstashui/Management/models.py b/src/logstashui/Management/models.py index 1fdfa7d6..32639f92 100644 --- a/src/logstashui/Management/models.py +++ b/src/logstashui/Management/models.py @@ -41,6 +41,16 @@ class Settings(models.Model): "May differ from the browser reverse-proxy URL." ), ) + logstash_artifact_base_url = models.CharField( + max_length=512, + blank=True, + default="", + help_text=( + "Upstream source for Logstash release tarballs. Blank uses " + "https://artifacts.elastic.co/downloads/logstash. Point this at an " + "internal mirror to keep tarball fetches inside your network." + ), + ) class Meta: db_table = 'settings' diff --git a/src/logstashui/Management/templates/api_tokens.html b/src/logstashui/Management/templates/api_tokens.html new file mode 100644 index 00000000..831ecc61 --- /dev/null +++ b/src/logstashui/Management/templates/api_tokens.html @@ -0,0 +1,153 @@ + + +{% extends "base.html" %} + +{% block content %} + +
+
+

API Tokens

+ +
+

+ Authenticate scripted requests with Authorization: ApiKey <token>. + A token acts as the user who created it and inherits that user's role. +

+ +
+ + + + + + + + + + + + + + {% for token in tokens %} + {% include 'components/api_token_row.html' %} + {% empty %} + + + + {% endfor %} + +
NameTokenOwnerStatusLast UsedExpiresActions
+ No API tokens yet. +
+
+
+ + + + + +{% endblock %} diff --git a/src/logstashui/Management/templates/components/api_token_created.html b/src/logstashui/Management/templates/components/api_token_created.html new file mode 100644 index 00000000..0cc3da1e --- /dev/null +++ b/src/logstashui/Management/templates/components/api_token_created.html @@ -0,0 +1,51 @@ + + +
+
+ Token {{ token.name }} created. Copy it now — it cannot be shown again. +
+ +
+
+ Token + +
+ {{ raw_token }} +
+ +
+ Example + 
curl -X POST \
+  -H "Authorization: ApiKey {{ raw_token }}" \
+  -d "connection_type=CENTRALIZED&name=my-cluster&host=https://es.example.com&port=443&api_key=<es-api-key>" \
+  https://localhost:8443/ConnectionManager/AddConnection
+
+ +
+ +
+
+ + diff --git a/src/logstashui/Management/templates/components/api_token_row.html b/src/logstashui/Management/templates/components/api_token_row.html new file mode 100644 index 00000000..c2b8854e --- /dev/null +++ b/src/logstashui/Management/templates/components/api_token_row.html @@ -0,0 +1,84 @@ + + + + + {{ token.name|default:"—" }} + + + {{ token.masked }} + + + {{ token.user.username }} + {% if token.user.profile.role != 'admin' %} + + readonly + + {% endif %} + + + {% if token.revoked_at %} + + Revoked + + {% elif token.is_expired %} + + Expired + + {% else %} + + Active + + {% endif %} + + + {{ token.last_used_at|date:"Y-m-d H:i"|default:"never" }} + + + {{ token.expires_at|date:"Y-m-d"|default:"never" }} + + +
+ + +
+ + diff --git a/src/logstashui/Management/templates/components/logstash_artifact_row.html b/src/logstashui/Management/templates/components/logstash_artifact_row.html new file mode 100644 index 00000000..58046ca6 --- /dev/null +++ b/src/logstashui/Management/templates/components/logstash_artifact_row.html @@ -0,0 +1,100 @@ + + + + + {{ artifact.filename }} + + + {{ artifact.version }} + {% if artifact.in_use %} + + in use + + {% endif %} + + + {% if artifact.status == 'READY' %} + + Ready + + {% elif artifact.status == 'FAILED' %} + + Failed + + {% if artifact.error %} +

{{ artifact.error|truncatechars:160 }}

+ {% endif %} + {% elif artifact.status == 'PENDING' %} + + Queued + + {% else %} + + {% if artifact.status == 'IMPORTING' %}Verifying{% else %}Downloading{% endif %} + +
+
+
+

+ {% if artifact.percent is not None %}{{ artifact.percent }}%{% else %}starting…{% endif %} +

+ {% endif %} + + + {% if artifact.size_bytes %}{{ artifact.size_bytes|filesizeformat }}{% else %}—{% endif %} + + + {{ artifact.serve_count }}× + {% if artifact.last_served_at %} + {{ artifact.last_served_at|date:"Y-m-d H:i" }} + {% endif %} + + +
+ + +
+ + diff --git a/src/logstashui/Management/templates/components/logstash_artifact_tbody.html b/src/logstashui/Management/templates/components/logstash_artifact_tbody.html new file mode 100644 index 00000000..2f06d17f --- /dev/null +++ b/src/logstashui/Management/templates/components/logstash_artifact_tbody.html @@ -0,0 +1,26 @@ + + + {% for artifact in artifacts %} + {% include 'components/logstash_artifact_row.html' %} + {% empty %} + + + No tarballs cached yet. Use Download Tarball, + or copy one into the cache directory and click + Import from disk. + + + {% endfor %} + diff --git a/src/logstashui/Management/templates/logstash_artifacts.html b/src/logstashui/Management/templates/logstash_artifacts.html new file mode 100644 index 00000000..90bbc7b0 --- /dev/null +++ b/src/logstashui/Management/templates/logstash_artifacts.html @@ -0,0 +1,172 @@ + + +{% extends "base.html" %} + +{% block content %} + +
+
+

Logstash Tarballs

+
+ + + +
+
+

+ LogstashUI fetches each release once and serves it to every agent whose policy has + Download the tarball from LogstashUI enabled, + instead of each agent pulling ~450 MB from Elastic. +

+

+ Cache: {{ artifact_dir }} + · Source: {{ base_url }} + (change under Settings) +

+ +
+ +
+ + + + + + + + + + + + {% include 'components/logstash_artifact_tbody.html' %} +
TarballVersionStatusSizeServedActions
+
+
+ + + + + +{% endblock %} diff --git a/src/logstashui/Management/templates/management.html b/src/logstashui/Management/templates/management.html index f120024e..e3d5c7a0 100644 --- a/src/logstashui/Management/templates/management.html +++ b/src/logstashui/Management/templates/management.html @@ -36,6 +36,32 @@

Logs

+ + +
+
+ + + +
+

API Tokens

+

Issue tokens for scripted access to LogstashUI

+
+ + + + +
+
+ + + +
+

Logstash Tarballs

+

Cache Logstash releases once and serve them to agents

+
+ +
diff --git a/src/logstashui/Management/templates/management_settings.html b/src/logstashui/Management/templates/management_settings.html index 2bc72dee..8edb8e49 100644 --- a/src/logstashui/Management/templates/management_settings.html +++ b/src/logstashui/Management/templates/management_settings.html @@ -70,6 +70,26 @@

Agent callback URL

{% if app_settings.agent_ui_url %}{{ app_settings.agent_ui_url }}{% else %}<ui_url>{% endif %}/.well-known/logstashui/ca.crt

+ + +
+

Logstash tarball source

+

+ Where LogstashUI fetches Logstash release tarballs for the + tarball proxy. + Leave blank to use Elastic's artifact host. Point it at an internal mirror to + keep fetches inside your network. +

+ +

+ Tarballs are read as <base>/logstash-<version>-linux-x86_64.tar.gz, + with the matching .sha512 alongside. +

+
@@ -94,9 +114,16 @@

HTTPS certificate

+ {% if tls_status.insecure_http %} +
+ LOGSTASHUI_INSECURE_HTTP is on. The UI is serving HTTP and will not generate a product CA or UI certificate. Automatic TLS is the supported path. +
+ {% endif %}
Current mode: - {% if tls_status.mode == 'custom' %} + {% if tls_status.insecure_http or tls_status.mode == 'disabled' %} + HTTP (insecure) + {% elif tls_status.mode == 'custom' %} Custom certificate {% else %} Product CA (default) @@ -129,6 +156,7 @@

HTTPS certificate

{{ tls_status.product_ca_fingerprint }}

+ {% if not tls_status.insecure_http %}
{% csrf_token %}

Use custom certificate

@@ -167,6 +195,7 @@

Use custom certificate

(use bin/start_logstashui.sh so host TLS SANs stay on the leaf).

+ {% endif %}
@@ -174,13 +203,16 @@

Use custom certificate

{% endblock %} diff --git a/src/logstashui/Management/urls.py b/src/logstashui/Management/urls.py index a147fbbe..84f54929 100644 --- a/src/logstashui/Management/urls.py +++ b/src/logstashui/Management/urls.py @@ -15,6 +15,8 @@ path("Logs/", views.Logs, name="Logs"), path("Logs/filter", views.LogsFilter, name="LogsFilter"), path("Logs/download", views.LogsDownload, name="LogsDownload"), + path("ApiTokens/", views.ApiTokens, name="ApiTokens"), + path("LogstashArtifacts/", views.LogstashArtifacts, name="LogstashArtifacts"), path("Settings/", views.SettingsView, name="Settings"), path("Settings/Tls/", views.SettingsTlsUpload, name="SettingsTlsUpload"), path("Settings/Tls/Revert/", views.SettingsTlsRevert, name="SettingsTlsRevert"), diff --git a/src/logstashui/Management/views.py b/src/logstashui/Management/views.py index 40506d0a..abf6d477 100644 --- a/src/logstashui/Management/views.py +++ b/src/logstashui/Management/views.py @@ -13,8 +13,10 @@ from django.template.loader import render_to_string from django.conf import settings from django.db import transaction +from django.utils import timezone from .models import UserProfile, Settings from django.http import JsonResponse +from datetime import timedelta import logging import json import os @@ -345,24 +347,40 @@ def SettingsView(request): try: experimental_mode = request.POST.get('experimental_mode') == 'on' agent_ui_url = (request.POST.get('agent_ui_url') or '').strip() + artifact_base_url = ( + request.POST.get('logstash_artifact_base_url') or '' + ).strip() + + if artifact_base_url and not artifact_base_url.startswith( + ('http://', 'https://') + ): + return JsonResponse({ + 'success': False, + 'message': 'Tarball source must start with http:// or https://', + }) app_settings = Settings.get_settings() previous_url = (app_settings.agent_ui_url or "").strip() app_settings.experimental_mode = experimental_mode app_settings.agent_ui_url = agent_ui_url + app_settings.logstash_artifact_base_url = artifact_base_url app_settings.save() logger.info( f"User '{request.user.username}' updated settings " - f"(experimental_mode={experimental_mode}, agent_ui_url={agent_ui_url!r})" + f"(experimental_mode={experimental_mode}, agent_ui_url={agent_ui_url!r}, " + f"logstash_artifact_base_url={artifact_base_url!r})" ) cert_note = "" if agent_ui_url != previous_url: try: + from LogstashUI.insecure_http import insecure_http from Common.product_ca import ensure_default_ui_server_cert, get_ui_server_mode - if get_ui_server_mode() == "product": + if insecure_http(): + cert_note = "" + elif get_ui_server_mode() == "product": ensure_default_ui_server_cert() # re-issues when SANs change cert_note = ( " Product UI certificate was re-checked for new callback URL SANs; " @@ -407,6 +425,13 @@ def SettingsTlsUpload(request): """Upload a custom UI server certificate (replaces product default leaf only).""" if request.method != 'POST': return JsonResponse({'success': False, 'message': 'Method not allowed'}, status=405) + from LogstashUI.insecure_http import INSECURE_HTTP_WARNING, insecure_http + + if insecure_http(): + return JsonResponse( + {"success": False, "message": INSECURE_HTTP_WARNING}, + status=409, + ) try: from Common.product_ca import save_custom_ui_certificate, get_ui_tls_status @@ -445,6 +470,13 @@ def SettingsTlsRevert(request): """Revert UI server cert to product-CA-signed default.""" if request.method != 'POST': return JsonResponse({'success': False, 'message': 'Method not allowed'}, status=405) + from LogstashUI.insecure_http import INSECURE_HTTP_WARNING, insecure_http + + if insecure_http(): + return JsonResponse( + {"success": False, "message": INSECURE_HTTP_WARNING}, + status=409, + ) try: from Common.product_ca import revert_ui_certificate_to_product_default @@ -457,4 +489,254 @@ def SettingsTlsRevert(request): }) except Exception as e: logger.error(f"Error reverting TLS certificate: {e}", exc_info=True) - return JsonResponse({'success': False, 'message': f'Error: {e}'}, status=500) \ No newline at end of file + return JsonResponse({'success': False, 'message': f'Error: {e}'}, status=500) + + +# --------------------------------------------------------------------------- +# API tokens +# --------------------------------------------------------------------------- + +def _token_error(message): + return HttpResponse( + '
{escape(message)}
' + ) + + +def _generate_token_table_rows(tokens, request): + """Render the token table body, reused for htmx swaps after revoke/delete.""" + rows_html = '' + for token in tokens: + rows_html += render_to_string('components/api_token_row.html', { + 'token': token, + 'csrf_token': request.META.get('CSRF_COOKIE', ''), + }, request=request) + return rows_html + + +@require_admin_role +def ApiTokens(request): + """Mint, list, revoke and delete admin API tokens. + + A token acts as its owning user, so the caller's own account is the owner — + that keeps audit lines like "User 'x' added connection" meaningful, and + means a readonly user's token is readonly. + """ + from PipelineManager.models import ApiKey + + def _all_tokens(): + return ( + ApiKey.objects.filter(user__isnull=False) + .select_related('user') + .order_by('-created_at', '-id') + ) + + if request.method == 'POST': + action = request.POST.get('action') + + if action == 'create': + name = (request.POST.get('name') or '').strip() + if not name: + return _token_error('A token name is required.') + if len(name) > 100: + return _token_error('Token name must be 100 characters or fewer.') + + expires_at = None + raw_days = (request.POST.get('expires_days') or '').strip() + if raw_days: + try: + days = int(raw_days) + except ValueError: + return _token_error('Expiry must be a whole number of days.') + if days < 1: + return _token_error('Expiry must be at least 1 day.') + expires_at = timezone.now() + timedelta(days=days) + + token, raw = ApiKey.issue_for_user( + request.user, name=name, expires_at=expires_at + ) + # Deliberately not logged — this is the only time the secret exists. + logger.info( + f"User '{request.user.username}' created API token '{name}' " + f"(prefix {token.prefix})" + ) + return render(request, 'components/api_token_created.html', { + 'token': token, + 'raw_token': raw, + }) + + if action in ('revoke', 'delete'): + token_id = request.POST.get('token_id') + token = ApiKey.objects.filter( + id=token_id, user__isnull=False + ).first() + if token is None: + return _token_error('Token not found.') + + if action == 'revoke': + if token.revoked_at is None: + token.revoked_at = timezone.now() + token.save() + logger.warning( + f"User '{request.user.username}' revoked API token " + f"'{token.name}' (prefix {token.prefix})" + ) + else: + logger.warning( + f"User '{request.user.username}' deleted API token " + f"'{token.name}' (prefix {token.prefix})" + ) + token.delete() + + return HttpResponse(_generate_token_table_rows(_all_tokens(), request)) + + return _token_error('Unknown action.') + + return render(request, 'api_tokens.html', {'tokens': _all_tokens()}) + + +# --------------------------------------------------------------------------- +# Logstash tarball cache +# --------------------------------------------------------------------------- + +def _artifact_error(message): + return HttpResponse( + '
{escape(message)}
' + ) + + +def _artifact_in_flight(artifacts): + from PipelineManager.models import LogstashArtifact + + return any( + a.status in ( + LogstashArtifact.Status.FETCHING, + LogstashArtifact.Status.IMPORTING, + ) + for a in artifacts + ) + + +def _render_artifact_tbody(artifacts, request): + """Render the whole tbody, not just rows. + + The polling attributes live on the tbody, so it has to be swapped as a unit + (outerHTML) for polling to be able to stop. + """ + return render_to_string('components/logstash_artifact_tbody.html', { + 'artifacts': artifacts, + 'in_flight': _artifact_in_flight(artifacts), + 'csrf_token': request.META.get('CSRF_COOKIE', ''), + }, request=request) + + +@require_admin_role +def LogstashArtifacts(request): + """Manage the cache of Logstash tarballs served to agents. + + Downloads happen in a background greenlet, so both the download and import + actions return immediately and the table polls itself while anything is in + flight. There is no upload action: 450 MB through a browser is not viable. + """ + from PipelineManager import artifacts as artifact_lib + from PipelineManager.models import LogstashArtifact, parse_artifact_filename + + def _all_artifacts(): + """Artifacts, each tagged with whether a policy still pins its version. + + Deleting an in-use tarball is allowed — an operator may be reclaiming + disk deliberately — but it silently sends every agent on that policy + into a 503 retry loop, so the row says so before they click. + """ + from PipelineManager.models import Policy + + in_use = set( + Policy.objects.filter( + logstash_via_ui=True, + logstash_source=Policy.LogstashSource.VERSION, + ).values_list('logstash_version', flat=True) + ) + rows = list(LogstashArtifact.objects.all()) + for row in rows: + row.in_use = row.version in in_use + return rows + + def _table(): + return HttpResponse(_render_artifact_tbody(_all_artifacts(), request)) + + if request.method == 'POST': + action = request.POST.get('action') + + if action == 'download': + url = (request.POST.get('source_url') or '').strip() + if url: + if not url.startswith(('http://', 'https://')): + return _artifact_error('A full URL must start with http:// or https://.') + filename = url.rsplit('/', 1)[-1] + else: + version = (request.POST.get('version') or '').strip() + arch = (request.POST.get('arch') or 'linux-x86_64').strip() + if not version: + return _artifact_error('A Logstash version is required.') + filename = f'logstash-{version}-{arch}.tar.gz' + + if parse_artifact_filename(filename) is None: + return _artifact_error( + f'"{filename}" is not a recognized Logstash tarball name. ' + 'Expected logstash--linux-x86_64.tar.gz or similar.' + ) + + artifact = artifact_lib.get_or_create_artifact(filename, source_url=url) + if artifact.status == LogstashArtifact.Status.READY: + return _artifact_error(f'{filename} is already cached.') + + artifact_lib.start_fetch(artifact) + logger.info( + f"User '{request.user.username}' requested Logstash tarball {filename}" + ) + # 204 is the "work started" signal the page reloads on. An error + # comes back as 200 with an HTML fragment, so the two never collide. + return HttpResponse(status=204) + + if action == 'import': + imported = artifact_lib.scan_for_imports() + if not imported: + return _artifact_error( + 'No new tarballs found. Copy them into ' + f'{artifact_lib.artifact_dir()} and try again.' + ) + logger.info( + f"User '{request.user.username}' imported {len(imported)} " + f"Logstash tarball(s): {', '.join(imported)}" + ) + return HttpResponse(status=204) + + if action == 'delete': + artifact = LogstashArtifact.objects.filter( + id=request.POST.get('artifact_id') + ).first() + if artifact is None: + return _artifact_error('Tarball not found.') + logger.warning( + f"User '{request.user.username}' deleted Logstash tarball " + f"{artifact.filename}" + ) + artifact_lib.delete_artifact(artifact) + return _table() + + if action == 'rows': + return _table() + + return _artifact_error('Unknown action.') + + if request.GET.get('rows'): + return _table() + + artifacts = _all_artifacts() + return render(request, 'logstash_artifacts.html', { + 'artifacts': artifacts, + 'artifact_dir': artifact_lib.artifact_dir(), + 'base_url': artifact_lib.upstream_base_url(), + 'in_flight': _artifact_in_flight(artifacts), + }) \ No newline at end of file diff --git a/src/logstashui/PipelineManager/agent_api.py b/src/logstashui/PipelineManager/agent_api.py index 60172be2..90a0b447 100644 --- a/src/logstashui/PipelineManager/agent_api.py +++ b/src/logstashui/PipelineManager/agent_api.py @@ -24,6 +24,7 @@ materialize_simulate_logstash_yml, next_managed_instance_id, next_simulate_instance_id, + logstash_via_ui, normalize_policy_type, simulate_ports, ) @@ -124,6 +125,10 @@ def expand_instance_path(path: str | None, instance_id) -> str | None: def _sign_csr_if_present(data: dict) -> dict | None: """If request includes csr_pem, sign with product CA and return payload fragment.""" + from LogstashUI.insecure_http import insecure_http + + if insecure_http(): + return None csr_pem = data.get("csr_pem") or data.get("certificate_signing_request") if not csr_pem: return None @@ -502,6 +507,11 @@ def issue_server_cert(request): if not authorized: return JsonResponse({"success": False, "error": "Unauthorized"}, status=401) + from LogstashUI.insecure_http import insecure_http + + if insecure_http(): + return JsonResponse({"success": True}) + from Common.product_ca import sign_agent_csr signed = sign_agent_csr(csr_pem if isinstance(csr_pem, bytes) else csr_pem.encode("utf-8")) @@ -580,10 +590,22 @@ def check_in(request): if status_blob: connection.status_blob = status_blob logger.debug(f"Updated status_blob: {status_blob}") - # Surface resolved Logstash version for sim target dropdown - resolved = status_blob.get("logstash_version_resolved") or status_blob.get( - "logstash_version" + # Surface resolved Logstash version for the LS pill and the sim + # target dropdown. logstash_api.version comes from the running + # instance's own API and is the only key that tracks a version + # change on the host, so it leads. The bare logstash_version key is + # the policy-*desired* version everywhere else (see + # build_policy_config and the check-in response below), so it is the + # weakest signal and must stay last. + api_blob = status_blob.get("logstash_api") + resolved = ( + (api_blob.get("version") if isinstance(api_blob, dict) else None) + or status_blob.get("logstash_version_resolved") + or status_blob.get("logstash_version") ) + # Only overwrite on a truthy value: a check-in sent while Logstash + # is stopped reports nothing, and blanking the column would drop the + # last known version out of the UI. if resolved: connection.logstash_version_resolved = str(resolved)[:64] if status_blob.get("agent_api_port") is not None: @@ -637,6 +659,8 @@ def check_in(request): or "/opt/logstash-agent/logstash-versions", iid, ), + # True => fetch the tarball from LogstashUI's proxy, not artifacts.elastic.co + "logstash_via_ui": logstash_via_ui(policy), "restart": should_restart, "desired_agent_version": connection.desired_agent_version, "managed_changes_available": managed_changes_available, @@ -709,6 +733,7 @@ def get_config_changes(request): agent_logstash_source = (data.get("logstash_source") or "SYSTEM").upper() agent_logstash_version = data.get("logstash_version") or "" agent_logstash_download_dir = data.get("logstash_download_dir") or "" + agent_logstash_via_ui = bool(data.get("logstash_via_ui", False)) if not connection.policy: return JsonResponse({"success": False, "error": "No policy assigned to this connection"}, status=400) @@ -765,6 +790,7 @@ def get_config_changes(request): policy_download_dir = "/opt/logstash-agent" + policy_download_dir[ len("/opt/LogstashAgent") : ] + policy_via_ui = logstash_via_ui(policy) runtime_changed = ( agent_logstash_source != policy_source or (policy_source == "VERSION" and agent_logstash_version != policy_version) @@ -773,6 +799,9 @@ def get_config_changes(request): and (agent_logstash_download_dir or policy_download_dir) and agent_logstash_download_dir != policy_download_dir ) + # Toggling the checkbox alone must re-materialize, or the agent keeps + # pulling from whichever source it used last. + or (policy_source == "VERSION" and agent_logstash_via_ui != policy_via_ui) or (policy_source == "SYSTEM" and agent_binary_path != policy.binary_path) ) if runtime_changed: @@ -781,6 +810,7 @@ def get_config_changes(request): "version": policy_version, "download_dir": policy_download_dir, "binary_path": policy.binary_path, + "via_ui": policy_via_ui, } else: changes["logstash_runtime"] = False diff --git a/src/logstashui/PipelineManager/agent_modes.py b/src/logstashui/PipelineManager/agent_modes.py index 88513102..3f1d3a4a 100644 --- a/src/logstashui/PipelineManager/agent_modes.py +++ b/src/logstashui/PipelineManager/agent_modes.py @@ -283,6 +283,25 @@ def materialize_simulate_logstash_yml( return "\n".join(lines) + ("\n" if lines else "") +def logstash_via_ui(policy: Policy) -> bool: + """Effective value of the "download Logstash from LogstashUI" flag. + + The stored field is only meaningful for a MANAGED or SIMULATE policy that + pins a version; PACKAGED uses the OS package and EMBEDDED runs in-process, + so neither ever downloads a tarball. Normalizing here rather than at each + call site means a stale True left behind by a policy-type change cannot leak + out to an agent. + """ + if not getattr(policy, "logstash_via_ui", False): + return False + if policy.logstash_source != Policy.LogstashSource.VERSION: + return False + return normalize_policy_type(policy.policy_type) in ( + Policy.PolicyType.MANAGED, + Policy.PolicyType.SIMULATE, + ) + + def build_policy_config(policy: Policy, *, instance_id: int | None = None) -> dict: """ Build enrollment / apply policy_config payload. @@ -307,6 +326,7 @@ def build_policy_config(policy: Policy, *, instance_id: int | None = None) -> di "logstash_download_dir": normalize_agent_opt_path( policy.logstash_download_dir or f"{AGENT_OPT_ROOT}/logstash-versions" ), + "logstash_via_ui": False, "logstash_yml": materialize_simulate_logstash_yml( policy.logstash_yml, EMBEDDED_LOGSTASH_API_PORT, instance_id=None ), @@ -342,6 +362,7 @@ def build_policy_config(policy: Policy, *, instance_id: int | None = None) -> di "logstash_source": policy.logstash_source, "logstash_version": policy.logstash_version or "", "logstash_download_dir": download_dir, + "logstash_via_ui": logstash_via_ui(policy), "logstash_unit": f"ls-simulate@{instance_id}", "agent_unit": f"lsagent-simulate@{instance_id}", "logstash_yml": yml, @@ -375,6 +396,7 @@ def build_policy_config(policy: Policy, *, instance_id: int | None = None) -> di "logstash_source": policy.logstash_source, "logstash_version": policy.logstash_version or "", "logstash_download_dir": download_dir or f"{AGENT_OPT_ROOT}/logstash-versions", + "logstash_via_ui": logstash_via_ui(policy), "logstash_unit": f"logstash-managed@{instance_id}", "agent_unit": f"logstash-agent@{instance_id}", "path_root": paths["path_root"], @@ -397,6 +419,7 @@ def build_policy_config(policy: Policy, *, instance_id: int | None = None) -> di "logstash_source": policy.logstash_source or "SYSTEM", "logstash_version": policy.logstash_version or "", "logstash_download_dir": normalize_agent_opt_path(policy.logstash_download_dir or ""), + "logstash_via_ui": False, "logstash_unit": "logstash", "agent_unit": "logstash-agent", "logstash_yml": policy.logstash_yml, @@ -407,14 +430,15 @@ def build_policy_config(policy: Policy, *, instance_id: int | None = None) -> di def embedded_agent_base_url() -> str: """URL the UI uses to reach the docker/local embedded agent FastAPI.""" + from LogstashUI.insecure_http import force_http_url + try: from django.conf import settings - return (getattr(settings, "LOGSTASH_AGENT_URL", None) or "https://127.0.0.1:9500").rstrip( - "/" - ) + url = getattr(settings, "LOGSTASH_AGENT_URL", None) or "https://127.0.0.1:9500" except Exception: - return "https://127.0.0.1:9500" + url = "https://127.0.0.1:9500" + return force_http_url(url).rstrip("/") def probe_embedded_agent_online(timeout: float = 2.0) -> bool: @@ -528,7 +552,8 @@ def ensure_embedded_connection(*, probe: bool = True) -> Connection | None: agent_api_port=port, logstash_api_port=EMBEDDED_LOGSTASH_API_PORT, last_check_in=now if online else None, - status_blob=update_fields.get("status_blob") or {"embedded": True, "online": online}, + # probe=False leaves online *unknown* rather than asserting offline + status_blob=update_fields.get("status_blob") or {"embedded": True}, ) conn.save() return conn @@ -574,6 +599,21 @@ def is_embedded_discovered(conn) -> bool: return (now - ts).total_seconds() < 600 +def embedded_probe_failed(conn) -> bool: + """True only when a probe explicitly reported the embedded agent offline. + + A row that has never been probed carries no ``online`` key — that is + unknown, not failed, and the sticky picker row stays visible. + """ + if conn is None: + return False + if isinstance(conn, dict): + blob = conn.get("status_blob") or {} + else: + blob = getattr(conn, "status_blob", None) or {} + return blob.get("online") is False + + def is_embedded_connection(conn) -> bool: """True for the docker/local pseudo agent (dict or model).""" if conn is None: @@ -595,6 +635,8 @@ def list_simulation_targets(active_only: bool = True, *, ensure_embedded: bool = """ Return list of dicts describing simulate-capable connections for the editor. """ + from LogstashUI.insecure_http import force_http_url + if ensure_embedded: ensure_embedded_connection(probe=False) @@ -620,9 +662,12 @@ def list_simulation_targets(active_only: bool = True, *, ensure_embedded: bool = or ("system" if policy.policy_type == Policy.PolicyType.SIMULATE else "") ) if policy.policy_type == Policy.PolicyType.EMBEDDED: - # Picker only — and only after a successful live probe - if not is_embedded_discovered(conn): + # Picker only. This path never probes (see + # refresh_embedded_connection_async), so keep the sticky row until + # a probe has actually failed — unprobed is unknown, not offline. + if embedded_probe_failed(conn): continue + discovered = is_embedded_discovered(conn) # Closed select: terse; detail on hover / open option list label = "embedded" ver_label = version or "docker" @@ -637,9 +682,11 @@ def list_simulation_targets(active_only: bool = True, *, ensure_embedded: bool = if not base_url: host = conn.host or "127.0.0.1" base_url = f"https://{host}:{agent_port}" + base_url = force_http_url(base_url) host = conn.host or "127.0.0.1" detail = f"embedded · {host} · Logstash {ver_label}" else: + discovered = True n = conn.instance_id or "?" ver_label = version or "system" host = conn.host or "127.0.0.1" @@ -648,7 +695,9 @@ def list_simulation_targets(active_only: bool = True, *, ensure_embedded: bool = agent_port = conn.agent_api_port if agent_port is None and conn.instance_id: agent_port = SIMULATE_AGENT_API_BASE + conn.instance_id - base_url = f"https://{host}:{agent_port}" if agent_port else None + base_url = ( + force_http_url(f"https://{host}:{agent_port}") if agent_port else None + ) row = { "connection_id": conn.id, @@ -664,6 +713,7 @@ def list_simulation_targets(active_only: bool = True, *, ensure_embedded: bool = "logstash_source": policy.logstash_source, "host": host, "base_url": base_url, + "discovered": discovered, "last_selected_at": conn.last_selected_at.isoformat() if conn.last_selected_at else None, diff --git a/src/logstashui/PipelineManager/agent_policies.py b/src/logstashui/PipelineManager/agent_policies.py index be97166b..d98e380b 100644 --- a/src/logstashui/PipelineManager/agent_policies.py +++ b/src/logstashui/PipelineManager/agent_policies.py @@ -5,6 +5,7 @@ from django.http import JsonResponse from PipelineManager.models import Revision, Policy, Connection as ConnectionTable, Keystore +from PipelineManager.agent_versions import resolve_running_logstash_version from Common.decorators import require_admin_role @@ -636,6 +637,10 @@ def get_policy_nodes(request): "status_class": status_class, "last_check_in": node.last_check_in.isoformat() if node.last_check_in else None, "agent_version": node.status_blob.get('agent_version') if node.status_blob else None, + "logstash_version": resolve_running_logstash_version( + logstash_version_resolved=node.logstash_version_resolved, + status_blob=node.status_blob if isinstance(node.status_blob, dict) else None, + ), "cpm_enabled": cpm_enabled, "snmp_networks": sorted(snmp_networks_by_conn.get(node.id, [])), }) diff --git a/src/logstashui/PipelineManager/agent_versions.py b/src/logstashui/PipelineManager/agent_versions.py new file mode 100644 index 00000000..f6ca0d11 --- /dev/null +++ b/src/logstashui/PipelineManager/agent_versions.py @@ -0,0 +1,104 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +"""Logstash / agent version display and VERSION binary-path helpers.""" + +from __future__ import annotations + +from packaging.version import InvalidVersion, Version + +SYSTEM_BINARY_PATH = "/usr/share/logstash/bin" +DEFAULT_DOWNLOAD_DIR = "/opt/logstash-agent/logstash-versions" + + +def derive_version_binary_path(download_dir: str | None, version: str | None) -> str | None: + ver = (version or "").strip() + if not ver: + return None + root = (download_dir or DEFAULT_DOWNLOAD_DIR).rstrip("/") or DEFAULT_DOWNLOAD_DIR + return f"{root}/logstash-{ver}/bin" + + +def is_derived_version_binary_path(path: str | None, download_dir: str | None) -> bool: + p = (path or "").rstrip("/") + if not p: + return False + root = (download_dir or DEFAULT_DOWNLOAD_DIR).rstrip("/") or DEFAULT_DOWNLOAD_DIR + prefix = f"{root}/logstash-" + suffix = "/bin" + if not (p.startswith(prefix) and p.endswith(suffix)): + return False + mid = p[len(prefix) : -len(suffix)] + return bool(mid) and "/" not in mid + + +def resolve_running_logstash_version( + *, + logstash_version_resolved: str | None = None, + status_blob: dict | None = None, +) -> str | None: + """Best available answer to "which Logstash is running on that host?". + + The status blob is the current check-in and the column is history, so the + blob leads. Preferring the column stranded the display: it is only ever + written on a truthy value and never cleared, so one recorded version + shadowed every later one and the pill froze for the life of the row. + + The column still backs the blob up, which is what keeps the last known + version on screen while Logstash is stopped or its API is unreachable. + ``logstash_version`` is last because elsewhere it means the policy-*desired* + version rather than the running one. + """ + blob = status_blob if isinstance(status_blob, dict) else {} + api = blob.get("logstash_api") + if isinstance(api, dict): + ver = str(api.get("version") or "").strip() + if ver: + return ver + ver = str(blob.get("logstash_version_resolved") or "").strip() + if ver: + return ver + resolved = (logstash_version_resolved or "").strip() + if resolved: + return resolved + ver = str(blob.get("logstash_version") or "").strip() + if ver: + return ver + return None + + +def agent_version_relation(current: str | None, preferred: str | None) -> str: + try: + cur = Version(str(current or "").strip()) + pref = Version(str(preferred or "").strip()) + except InvalidVersion: + return "unknown" + if cur < pref: + return "older" + if cur > pref: + return "newer" + return "equal" + + +def resolve_persisted_binary_path( + *, + source: str | None, + version: str | None, + download_dir: str | None, + binary_path: str | None, +) -> str: + current = (binary_path or "").strip() or SYSTEM_BINARY_PATH + src = (source or "SYSTEM").upper() + derived = derive_version_binary_path(download_dir, version) + if src == "VERSION": + if derived and ( + not (binary_path or "").strip() + or current.rstrip("/") == SYSTEM_BINARY_PATH.rstrip("/") + or is_derived_version_binary_path(current, download_dir) + ): + return derived + return current + if is_derived_version_binary_path(current, download_dir): + return SYSTEM_BINARY_PATH + return current diff --git a/src/logstashui/PipelineManager/artifact_metrics.py b/src/logstashui/PipelineManager/artifact_metrics.py new file mode 100644 index 00000000..01733fac --- /dev/null +++ b/src/logstashui/PipelineManager/artifact_metrics.py @@ -0,0 +1,134 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +"""OpenTelemetry instruments for the tarball proxy, and no-ops without it. + +OpenTelemetry is an optional extra (``LogstashUI[otel]``) because LogstashUI ships +as an air-gapped wheel bundle and an offline Docker image, where every added +dependency has a real cost. Every function here is safe to call unconditionally: +when the packages are absent, or ``LOGSTASHUI_OTEL`` is off, they do nothing. + +Django auto-instrumentation alone cannot answer the question these exist for — +"should I add workers, cores, or concurrency?". A span shows that a request was +slow, never that the gevent hub was blocked. These four do: + +``logstashui.gevent.hub.lag`` + The most useful of the set. Flat under load means the NIC is the ceiling and + more workers will not help; spiking means greenlet starvation, so more + workers or cores will. It is also the only thing that catches a SQLite + ``busy_timeout`` stall, which blocks every greenlet in the process at once. +``logstashui.artifact.downloads.active`` + If this never reaches the cap, the cap is not the constraint and tuning it + is wasted effort. +``logstashui.artifact.requests`` + The 429/503 rate is the direct "raise the cap or add capacity" signal. +``logstashui.artifact.serve.bytes_per_second`` + Cross-plotted against ``downloads.active``: per-stream throughput falling + while aggregate stays flat means you are at the NIC. +""" + +import logging + +logger = logging.getLogger(__name__) + +_enabled = False +_meter = None +_downloads_active = None +_requests_counter = None +_throughput = None +_otel_context = None + +_active_downloads = 0 + + +def _noop(*_args, **_kwargs): + return None + + +def init(meter_provider=None): + """Create the instruments. Called from the OTel bootstrap; safe to skip.""" + global _enabled, _meter, _downloads_active, _requests_counter, _throughput + global _otel_context + + try: + from opentelemetry import context as otel_context # type: ignore[import-not-found] + from opentelemetry import metrics # type: ignore[import-not-found] + except ImportError: + return False + + _otel_context = otel_context + _meter = (meter_provider or metrics).get_meter('logstashui.artifacts') + + _downloads_active = _meter.create_up_down_counter( + 'logstashui.artifact.downloads.active', + unit='{download}', + description='Agent tarball downloads currently streaming from this worker', + ) + _requests_counter = _meter.create_counter( + 'logstashui.artifact.requests', + unit='{request}', + description='Tarball requests by outcome', + ) + _throughput = _meter.create_histogram( + 'logstashui.artifact.serve.bytes_per_second', + unit='By/s', + description='Effective throughput of a completed tarball transfer', + ) + _enabled = True + return True + + +def downloads_active_add(delta): + global _active_downloads + _active_downloads += delta + if _enabled and _downloads_active is not None: + _downloads_active.add(delta) + + +def active_downloads(): + """In-flight streams on this worker. Useful without OTel too.""" + return _active_downloads + + +def record_request(result): + if _enabled and _requests_counter is not None: + _requests_counter.add(1, {'result': result}) + + +def record_throughput(bytes_per_second): + if _enabled and _throughput is not None: + _throughput.record(bytes_per_second) + + +def current_context(): + """Capture the calling greenlet's trace context, if any. + + A spawned fetch greenlet does not inherit the request's context, so without + this its spans orphan and the download cannot be tied to the request that + triggered it. + """ + if _otel_context is None: + return None + try: + return _otel_context.get_current() + except Exception: + return None + + +def attach_context(context): + if context is None or _otel_context is None: + return None + try: + return _otel_context.attach(context) + except Exception: + return None + + +def detach_context(token): + if token is None or _otel_context is None: + return + try: + _otel_context.detach(token) + except Exception: + pass diff --git a/src/logstashui/PipelineManager/artifacts.py b/src/logstashui/PipelineManager/artifacts.py new file mode 100644 index 00000000..c08100b0 --- /dev/null +++ b/src/logstashui/PipelineManager/artifacts.py @@ -0,0 +1,745 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +"""Logstash tarball proxy: fetch each release once, serve it to every agent. + +A ``MANAGED`` or ``SIMULATE`` policy pinned to ``logstash_source=VERSION`` makes +every agent pull its own ~450 MB tarball from artifacts.elastic.co. This module +caches each tarball under ``settings.LOGSTASH_DIR`` and streams it to agents, +which turns N downloads into one and makes air-gapped operation possible. + +Three properties are load-bearing, and each is here for a reason specific to how +LogstashUI is deployed: + +**Single-flight lives in the database.** There is no ``CACHES`` backend, so +Django falls back to per-process LocMemCache and gunicorn runs 2+ workers. The +``cache.add()`` lock used elsewhere in this codebase cannot coordinate them. A +conditional UPDATE on the artifact row can — see ``LogstashArtifact.claim_for_fetch``. + +**A fetch "thread" is a greenlet.** ``GeventWorker.patch()`` monkey-patches +threading before the app loads, so a download dies with its worker: on SIGTERM +gunicorn waits out ``graceful_timeout`` and then kills it. Every download writes +to ``.part`` and is only ``os.replace``d into place after its SHA-512 +verifies, so a ``.tar.gz`` in the cache is always complete. Stale claims are +reclaimed by heartbeat. + +**Serving is capped per worker.** A gevent worker is one OS thread; concurrent +streams serialize their TLS writes on one core. The cap is a fairness knob, not a +throughput knob: it decides whether 20 agents each crawl or 8 finish fast and 12 +retry. The latter is better for a rollout, and ``Retry-After`` makes it orderly. +""" + +import hashlib +import logging +import os +import re +import threading +import time + +import django.db +from django.conf import settings +from django.db import transaction +from django.db.models import F +from django.http import FileResponse, HttpResponse, JsonResponse +from django.utils import timezone +from django.views.decorators.csrf import csrf_exempt + +import requests + +from . import artifact_metrics +from .models import ( + SMALL_FILE_BYTES, + Connection as ConnectionTable, + LogstashArtifact, + parse_artifact_filename, +) + +logger = logging.getLogger(__name__) + +#: Read/write size for the upstream fetch. Disk writes are not gevent-cooperative, +#: but a 1 MiB write to page cache is microseconds. +FETCH_CHUNK = 1024 * 1024 + +#: Chunk size when streaming to an agent. Each chunk is one SSL_write, a C call +#: the greenlet cannot yield inside, so this is a latency knob rather than a +#: throughput one. Django's 4096 default would mean ~110k iterations per tarball; +#: 1 MiB would hold the hub for ~1 ms at a time. 256 KiB sits between them. +SERVE_BLOCK_SIZE = 256 * 1024 + +#: Progress and heartbeat are written on a strict time floor, never per chunk. +#: On SQLite a blocked writer sits in C for up to busy_timeout (20 s), which +#: stalls the entire gevent hub -- every greenlet, including agent check-ins. +HEARTBEAT_INTERVAL = 5.0 + +RETRY_AFTER_FETCHING = 30 +RETRY_AFTER_BUSY = 60 +RETRY_AFTER_FAILED = 300 + +_RANGE_RE = re.compile(r'^bytes=(\d*)-(\d*)$') + +#: Per-worker, deliberately. Cross-worker coordination would mean adding Redis; +#: the imperfection (429 while a sibling worker is idle) is harmless given +#: Retry-After. Effective total is this value times LOGSTASHUI_WORKERS. +_serve_semaphore = threading.BoundedSemaphore( + max(1, getattr(settings, 'LOGSTASH_ARTIFACT_MAX_SERVE_PER_WORKER', 4)) +) + + +# --- storage --------------------------------------------------------------- + + +def artifact_dir(): + return settings.LOGSTASH_DIR + + +def artifact_path(filename): + return os.path.join(artifact_dir(), filename) + + +def upstream_base_url(): + """Operator-configured mirror, falling back to Elastic's artifact host. + + Read defensively: this runs on the agent-facing hot path and must not break + on an un-migrated database. + """ + default = settings.LOGSTASH_ARTIFACT_DEFAULT_BASE_URL + try: + from Management.models import Settings as AppSettings + + configured = (AppSettings.get_settings().logstash_artifact_base_url or '').strip() + return configured or default + except Exception: + return default + + +def sweep_partials(): + """Remove ``.part`` files orphaned by a worker that died mid-download. + + Safe to call at any time: a live download holds its ``.part`` open, and on + POSIX unlinking an open file only removes the name, so the writer fails at + its final rename rather than corrupting anything. + """ + removed = 0 + try: + entries = os.listdir(artifact_dir()) + except OSError: + return 0 + for name in entries: + if not name.endswith('.part'): + continue + try: + os.unlink(os.path.join(artifact_dir(), name)) + removed += 1 + except OSError: + pass + if removed: + logger.info(f"Swept {removed} orphaned Logstash tarball download(s)") + return removed + + +# --- fetching -------------------------------------------------------------- + + +def _fetch(pk, otel_context=None): + """Download and verify one artifact. Runs in its own greenlet. + + Never raises into the caller; every exit path records terminal state on the + row so a watching UI and a polling agent both see the outcome. + """ + detach = artifact_metrics.attach_context(otel_context) + try: + _fetch_inner(pk) + except Exception as exc: + logger.error(f"Logstash tarball fetch failed for artifact {pk}: {exc}") + try: + django.db.close_old_connections() + LogstashArtifact.objects.filter(pk=pk).update( + status=LogstashArtifact.Status.FAILED, + error=str(exc)[:2000], + heartbeat_at=timezone.now(), + ) + except Exception: + logger.exception("Could not record tarball fetch failure") + finally: + artifact_metrics.detach_context(detach) + # Mandatory. This greenlet gets its own DB connection and never receives + # request_started/request_finished, so nothing else will ever close it. + django.db.connections.close_all() + + +def _fetch_inner(pk): + artifact = LogstashArtifact.objects.get(pk=pk) + base = upstream_base_url() + tarball_url = artifact.resolve_source_url(base) + checksum_url = f"{tarball_url}.sha512" + + dest = artifact_path(artifact.filename) + part = f"{dest}.part" + checksum_dest = artifact_path(artifact.checksum_filename) + + logger.info(f"Fetching Logstash tarball {artifact.filename} from {tarball_url}") + + expected = _fetch_expected_sha512(checksum_url, artifact.filename) + + digest = hashlib.sha512() + downloaded = 0 + last_beat = 0.0 + + with requests.get(tarball_url, stream=True, timeout=(10, 120)) as response: + response.raise_for_status() + total = response.headers.get('Content-Length') + LogstashArtifact.objects.filter(pk=pk).update( + size_bytes=int(total) if total and total.isdigit() else None, + ) + with open(part, 'wb') as handle: + for chunk in response.iter_content(chunk_size=FETCH_CHUNK): + if not chunk: + continue + handle.write(chunk) + digest.update(chunk) + downloaded += len(chunk) + now = time.monotonic() + if now - last_beat >= HEARTBEAT_INTERVAL: + last_beat = now + django.db.close_old_connections() + LogstashArtifact.objects.filter(pk=pk).update( + bytes_downloaded=downloaded, + heartbeat_at=timezone.now(), + ) + handle.flush() + os.fsync(handle.fileno()) + + actual = digest.hexdigest() + if expected and actual != expected: + os.unlink(part) + raise ValueError( + f"SHA-512 mismatch for {artifact.filename}: " + f"upstream published {expected[:16]}…, downloaded {actual[:16]}…" + ) + + # Atomic on POSIX, so a .tar.gz in the cache directory is never partial. + os.replace(part, dest) + if expected: + with open(checksum_dest, 'w', encoding='utf-8') as handle: + handle.write(f"{expected} {artifact.filename}\n") + + django.db.close_old_connections() + LogstashArtifact.objects.filter(pk=pk).update( + status=LogstashArtifact.Status.READY, + sha512=actual, + size_bytes=downloaded, + bytes_downloaded=downloaded, + error='', + heartbeat_at=timezone.now(), + ) + logger.info( + f"Logstash tarball {artifact.filename} ready ({downloaded} bytes, sha512 verified)" + ) + + +def _fetch_expected_sha512(url, filename): + """Pull the ``.sha512`` sidecar. Absent is tolerated; malformed is not. + + A mirror may not publish checksums, and refusing to cache in that case would + make internal mirrors unusable. A checksum that exists but does not parse is + a different matter and fails the fetch. + """ + try: + response = requests.get(url, timeout=(10, 30)) + if response.status_code == 404: + logger.warning(f"No upstream checksum for {filename}; skipping verification") + return None + response.raise_for_status() + except requests.RequestException as exc: + logger.warning(f"Could not fetch checksum for {filename}: {exc}") + return None + + token = response.text.strip().split()[0] if response.text.strip() else '' + if not re.fullmatch(r'[0-9a-fA-F]{128}', token): + raise ValueError(f"Malformed upstream SHA-512 for {filename}") + return token.lower() + + +def start_fetch(artifact): + """Try to become the one process downloading this artifact. + + Returns True when a download was started here, False when someone else owns + it or we are at the upstream cap. Either way the caller answers 503; the + distinction only matters for logging. + """ + if not LogstashArtifact.claim_for_fetch(artifact.pk): + return False + + # Claim first, then check the cap, then hand the claim back if we are over. + # Counting before claiming would be a TOCTOU window, and closing it properly + # needs select_for_update -- which SQLite ignores and which would take gap + # locks on MySQL in a hot path. Overshoot here is bounded by worker count. + limit = getattr(settings, 'LOGSTASH_ARTIFACT_MAX_UPSTREAM', 2) + if LogstashArtifact.active_fetch_count() > limit: + LogstashArtifact.release_claim(artifact.pk) + logger.info( + f"Deferring fetch of {artifact.filename}: at the upstream cap ({limit})" + ) + return False + + context = artifact_metrics.current_context() + # A no-op under autocommit (ATOMIC_REQUESTS is unset), but correct if this + # ever runs inside a transaction -- otherwise the greenlet could start before + # the claim is visible to anyone else. + transaction.on_commit( + lambda: threading.Thread( + target=_fetch, args=(artifact.pk,), kwargs={'otel_context': context}, + daemon=True, + ).start() + ) + return True + + +def get_or_create_artifact(filename, *, source_url=''): + """Find or register the row for a tarball. Returns None for a bad filename.""" + parsed = parse_artifact_filename(filename) + if parsed is None: + return None + tarball, version, arch, _is_checksum = parsed + artifact, _created = LogstashArtifact.objects.get_or_create( + filename=tarball, + defaults={'version': version, 'arch': arch, 'source_url': source_url}, + ) + if source_url and artifact.source_url != source_url: + LogstashArtifact.objects.filter(pk=artifact.pk).update(source_url=source_url) + artifact.source_url = source_url + return artifact + + +# --- importing from disk (air-gapped) -------------------------------------- + + +def scan_for_imports(): + """Register tarballs dropped into the cache directory by hand. + + The air-gapped path: an operator copies the tarball in over sneakernet and + clicks Import rather than uploading 450 MB through a browser. Hashing is + deferred to a greenlet because SHA-512 over half a gigabyte takes seconds. + + Returns the list of filenames newly registered. + """ + try: + entries = sorted(os.listdir(artifact_dir())) + except OSError: + return [] + + known = set(LogstashArtifact.objects.values_list('filename', flat=True)) + imported = [] + for name in entries: + # .part files belong to a download in flight, and .sha512 sidecars are + # covered by their tarball's row. + if name.endswith('.part') or name.endswith('.sha512'): + continue + if name in known: + continue + parsed = parse_artifact_filename(name) + if parsed is None: + continue + tarball, version, arch, _is_checksum = parsed + path = artifact_path(tarball) + artifact = LogstashArtifact.objects.create( + filename=tarball, + version=version, + arch=arch, + status=LogstashArtifact.Status.IMPORTING, + size_bytes=os.path.getsize(path), + heartbeat_at=timezone.now(), + ) + imported.append(tarball) + transaction.on_commit( + lambda pk=artifact.pk: threading.Thread( + target=_verify_import, args=(pk,), daemon=True + ).start() + ) + return imported + + +def _verify_import(pk): + """Hash an imported tarball and publish it, or fail the row.""" + try: + artifact = LogstashArtifact.objects.get(pk=pk) + path = artifact_path(artifact.filename) + digest = hashlib.sha512() + total = 0 + last_beat = 0.0 + with open(path, 'rb') as handle: + while True: + chunk = handle.read(FETCH_CHUNK) + if not chunk: + break + digest.update(chunk) + total += len(chunk) + now = time.monotonic() + if now - last_beat >= HEARTBEAT_INTERVAL: + last_beat = now + django.db.close_old_connections() + LogstashArtifact.objects.filter(pk=pk).update( + bytes_downloaded=total, heartbeat_at=timezone.now() + ) + + actual = digest.hexdigest() + expected = _read_local_checksum(artifact) + if expected and actual != expected: + raise ValueError( + f"SHA-512 mismatch for imported {artifact.filename}: the " + f"accompanying .sha512 does not match the file" + ) + if not expected: + # No sidecar supplied: record what we computed and write one, so the + # agent's own verification step still has something to check against. + with open(artifact_path(artifact.checksum_filename), 'w', encoding='utf-8') as fh: + fh.write(f"{actual} {artifact.filename}\n") + + django.db.close_old_connections() + LogstashArtifact.objects.filter(pk=pk).update( + status=LogstashArtifact.Status.READY, + sha512=actual, + size_bytes=total, + bytes_downloaded=total, + error='', + heartbeat_at=timezone.now(), + ) + logger.info(f"Imported Logstash tarball {artifact.filename} ({total} bytes)") + except Exception as exc: + logger.error(f"Import verification failed for artifact {pk}: {exc}") + try: + django.db.close_old_connections() + LogstashArtifact.objects.filter(pk=pk).update( + status=LogstashArtifact.Status.FAILED, + error=str(exc)[:2000], + heartbeat_at=timezone.now(), + ) + except Exception: + logger.exception("Could not record import failure") + finally: + django.db.connections.close_all() + + +def _read_local_checksum(artifact): + """Read a hand-supplied ``.sha512`` sidecar, if there is one.""" + path = artifact_path(artifact.checksum_filename) + if not os.path.exists(path): + return None + try: + with open(path, 'r', encoding='utf-8') as handle: + token = handle.read().strip().split()[0] + except (OSError, IndexError): + return None + if not re.fullmatch(r'[0-9a-fA-F]{128}', token): + return None + return token.lower() + + +def delete_artifact(artifact): + """Remove an artifact row along with its tarball and checksum on disk.""" + for name in (artifact.filename, artifact.checksum_filename, f"{artifact.filename}.part"): + try: + os.unlink(artifact_path(name)) + except OSError: + pass + artifact.delete() + + +# --- serving --------------------------------------------------------------- + + +class _SemaphoreGuardedFile: + """A file object that stops at ``limit`` bytes and frees a semaphore slot. + + Two jobs, both of which ``FileResponse`` cannot do itself. + + *Releasing the slot.* Django registers this object's ``close`` as a resource + closer and gunicorn calls ``respiter.close()`` in a ``finally``, so it fires + both on a completed transfer and on a client that hangs up mid-stream — + exactly the guarantee the cap depends on. Wrapping the file beats appending + to ``response._resource_closers``, which is private and ordering-sensitive. + + *Enforcing the range.* ``FileResponse`` reads to EOF regardless of the + ``Content-Length`` header, so a seek alone would send the whole remainder of + the file for a bounded range and the client would see more bytes than it was + promised. ``limit`` caps it. + """ + + def __init__(self, handle, semaphore, on_close=None, limit=None): + self._handle = handle + self._semaphore = semaphore + self._on_close = on_close + self._remaining = limit + self._released = False + + def read(self, size=-1): + if self._remaining is None: + return self._handle.read(size) + if self._remaining <= 0: + return b'' + if size is None or size < 0: + size = self._remaining + chunk = self._handle.read(min(size, self._remaining)) + self._remaining -= len(chunk) + return chunk + + def seek(self, *args, **kwargs): + return self._handle.seek(*args, **kwargs) + + def tell(self): + return self._handle.tell() + + def fileno(self): + return self._handle.fileno() + + def close(self): + try: + self._handle.close() + finally: + # close() can legitimately fire twice; a BoundedSemaphore raises on + # an over-release, which would surface as a 500 on an otherwise + # successful download. + if not self._released: + self._released = True + if self._semaphore is not None: + self._semaphore.release() + if self._on_close is not None: + self._on_close() + + @property + def closed(self): + return self._handle.closed + + +def _parse_range(header, size): + """Parse a single byte range. Returns (start, end) inclusive, or a sentinel. + + Returns ``None`` when there is no usable range (absent, malformed, or + multi-range — all of which are answered with a normal 200), and the string + ``'unsatisfiable'`` when the range is well-formed but outside the file. + """ + if not header: + return None + # Multi-range gains nothing here and would let a segmented downloader + # multiply itself against the semaphore. Answer it with the whole file. + if ',' in header: + return None + match = _RANGE_RE.match(header.strip()) + if match is None: + return None + start_raw, end_raw = match.group(1), match.group(2) + if not start_raw and not end_raw: + return None + if not start_raw: + # Suffix form: bytes=-500 means the last 500 bytes. + length = int(end_raw) + if length <= 0: + return 'unsatisfiable' + start = max(0, size - length) + end = size - 1 + else: + start = int(start_raw) + end = int(end_raw) if end_raw else size - 1 + end = min(end, size - 1) + if start >= size or start > end: + return 'unsatisfiable' + return start, end + + +def _retry(status, payload, retry_after): + response = JsonResponse(payload, status=status) + response['Retry-After'] = str(retry_after) + return response + + +def _authenticate_agent(request, connection_id): + """Resolve and verify the calling agent. Returns (connection, error_response). + + Same inline sequence as the other agent endpoints; the only difference is + that ``connection_id`` arrives in the URL, because a GET has no body to put + it in. It has to come from somewhere: an agent key is a bare PBKDF2 hash with + no lookup column, so the header alone cannot identify a row without hashing + against every key in the table. + """ + auth_header = request.headers.get('Authorization', '') + if not auth_header.startswith('ApiKey '): + return None, JsonResponse( + {'success': False, 'error': 'Invalid authorization header'}, status=401 + ) + raw_api_key = auth_header[len('ApiKey '):].strip() + + try: + connection = ConnectionTable.objects.get( + id=connection_id, connection_type='AGENT' + ) + except ConnectionTable.DoesNotExist: + # 401, not 404: an unauthenticated caller must not be able to probe which + # connection ids exist. + return None, JsonResponse( + {'success': False, 'error': 'Invalid API key'}, status=401 + ) + + try: + api_key_obj = connection.api_keys.first() + if not api_key_obj or not api_key_obj.verify_api_key(raw_api_key): + return None, JsonResponse( + {'success': False, 'error': 'Invalid API key'}, status=401 + ) + except Exception as exc: + logger.error(f"API key verification error: {exc}") + return None, JsonResponse( + {'success': False, 'error': 'Authentication failed'}, status=401 + ) + + return connection, None + + +@csrf_exempt +def serve_artifact(request, connection_id, filename): + """Serve a cached Logstash tarball to an enrolled agent. + + ``GET /ConnectionManager/LogstashArtifact//`` with + ``Authorization: ApiKey ``. + + Answers a cache miss with 503 + ``Retry-After`` and kicks off the download, + and a full serve queue with 429 + ``Retry-After``. Both are retryable; the + agent must not fall back to artifacts.elastic.co, which would defeat the + point in an air-gapped site. + """ + if request.method not in ('GET', 'HEAD'): + return JsonResponse({'success': False, 'error': 'Method not allowed'}, status=405) + + _connection, error = _authenticate_agent(request, connection_id) + if error is not None: + artifact_metrics.record_request('401') + return error + + parsed = parse_artifact_filename(filename) + if parsed is None: + logger.warning(f"Rejected Logstash tarball request for {filename!r}") + artifact_metrics.record_request('404') + return JsonResponse({'status': 'not_found'}, status=404) + tarball, version, arch, is_checksum = parsed + + artifact = LogstashArtifact.objects.filter(filename=tarball).first() + if artifact is None: + artifact = LogstashArtifact.objects.create( + filename=tarball, version=version, arch=arch + ) + + if artifact.status == LogstashArtifact.Status.FAILED: + # Retry the fetch, but tell this caller to come back later either way: + # a 502 body is more useful to an operator reading agent logs than a + # silent stall. + start_fetch(artifact) + artifact_metrics.record_request('502') + return _retry( + 502, + {'status': 'failed', 'error': artifact.error or 'Upstream fetch failed'}, + RETRY_AFTER_FAILED, + ) + + path = artifact_path(filename) + if artifact.status != LogstashArtifact.Status.READY or not os.path.exists(path): + started = start_fetch(artifact) + artifact_metrics.record_request('503') + return _retry( + 503, + { + 'status': 'fetching', + 'filename': filename, + 'percent': 0 if started else (artifact.percent or 0), + }, + RETRY_AFTER_FETCHING, + ) + + return _stream_file(request, artifact, path, is_checksum) + + +def _stream_file(request, artifact, path, is_checksum): + size = os.path.getsize(path) + + # A checksum sidecar is a few dozen bytes. Making it queue behind four + # 450 MB transfers would be absurd, and letting a burst of them occupy the + # slots would starve the transfers themselves. + needs_slot = size > SMALL_FILE_BYTES + if needs_slot and not _serve_semaphore.acquire(blocking=False): + artifact_metrics.record_request('429') + return _retry(429, {'status': 'busy'}, RETRY_AFTER_BUSY) + + semaphore = _serve_semaphore if needs_slot else None + started = time.monotonic() + + def _finished(): + if needs_slot: + artifact_metrics.downloads_active_add(-1) + elapsed = max(time.monotonic() - started, 1e-6) + artifact_metrics.record_throughput(size / elapsed) + + if needs_slot: + artifact_metrics.downloads_active_add(1) + + try: + handle = open(path, 'rb') + except OSError: + if semaphore is not None: + semaphore.release() + artifact_metrics.downloads_active_add(-1) + artifact_metrics.record_request('404') + return JsonResponse({'status': 'not_found'}, status=404) + + content_type = 'text/plain' if is_checksum else 'application/gzip' + byte_range = _parse_range(request.headers.get('Range'), size) + + if byte_range == 'unsatisfiable': + handle.close() + if semaphore is not None: + semaphore.release() + artifact_metrics.downloads_active_add(-1) + response = HttpResponse(status=416) + response['Content-Range'] = f'bytes */{size}' + artifact_metrics.record_request('416') + return response + + if byte_range is not None: + start, end = byte_range + handle.seek(start) + length = end - start + 1 + response = FileResponse( + _SemaphoreGuardedFile( + handle, semaphore, on_close=_finished, limit=length + ), + content_type=content_type, + status=206, + ) + response['Content-Range'] = f'bytes {start}-{end}/{size}' + response['Content-Length'] = str(length) + artifact_metrics.record_request('206') + else: + response = FileResponse( + _SemaphoreGuardedFile(handle, semaphore, on_close=_finished), + content_type=content_type, + ) + response['Content-Length'] = str(size) + artifact_metrics.record_request('served') + + response.block_size = SERVE_BLOCK_SIZE + response['Accept-Ranges'] = 'bytes' + response['Content-Disposition'] = f'attachment; filename="{os.path.basename(path)}"' + + # One serve == one fresh tarball download, which is not the same as one + # request. A row covers both the .tar.gz and its .sha512, so counting every + # request made a single agent download read as 2x; a HEAD is a probe with no + # body, and a resume continues a download already counted. + counts_as_serve = ( + not is_checksum + and request.method == 'GET' + and (byte_range is None or byte_range[0] == 0) + ) + if counts_as_serve: + LogstashArtifact.objects.filter(pk=artifact.pk).update( + serve_count=F('serve_count') + 1, + last_served_at=timezone.now(), + ) + return response diff --git a/src/logstashui/PipelineManager/forms.py b/src/logstashui/PipelineManager/forms.py index 9bff67df..74928ba0 100644 --- a/src/logstashui/PipelineManager/forms.py +++ b/src/logstashui/PipelineManager/forms.py @@ -136,16 +136,42 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super().clean() connection_type = cleaned_data.get('connection_type') - + if connection_type == Connection.ConnectionType.AGENT: # For Agent connections, require either host or cloud_url if not cleaned_data.get('host') and not cleaned_data.get('cloud_url'): raise forms.ValidationError("Either Host or Cloud URL is required for Agent connections.") else: # CENTRALIZED - # For Centralized connections, require either cloud_id or cloud_url - if not cleaned_data.get('cloud_id') and not cleaned_data.get('host'): - raise forms.ValidationError("Either Cloud ID or host is required for centralized connections.") - + # Read the radio-button selections from raw POST data so we can zero + # out whichever fields the user hid before submitting. Hidden inputs + # are still included in the POST payload, so without this a stale + # cloud_id (or stale auth credential) silently wins even after the + # user has switched modes. + connection_mode = self.data.get('connection_mode', 'cloud') + auth_type = self.data.get('auth_type', 'basic') + + # --- Connection mode --- + if connection_mode == 'cloud': + # URL-mode fields should be ignored + cleaned_data['host'] = '' + cleaned_data['port'] = None + if not cleaned_data.get('cloud_id'): + raise forms.ValidationError("Cloud ID is required when using Cloud ID mode.") + else: # 'url' + # Cloud ID field should be ignored + cleaned_data['cloud_id'] = '' + if not cleaned_data.get('host'): + raise forms.ValidationError("Host is required when using Elastic Node URL mode.") + + # --- Auth type --- + if auth_type == 'basic': + # API key field should be ignored + cleaned_data['api_key'] = '' + else: # 'apiKey' + # Username / password fields should be ignored + cleaned_data['username'] = '' + cleaned_data['password'] = '' + return cleaned_data def save(self, commit=True): diff --git a/src/logstashui/PipelineManager/manager_views.py b/src/logstashui/PipelineManager/manager_views.py index 89802cf0..4fc3a05b 100644 --- a/src/logstashui/PipelineManager/manager_views.py +++ b/src/logstashui/PipelineManager/manager_views.py @@ -86,6 +86,10 @@ def PipelineManager(request): pass from PipelineManager.agent_modes import is_embedded_connection + from PipelineManager.agent_versions import ( + agent_version_relation, + resolve_running_logstash_version, + ) connections = [ conn @@ -93,6 +97,7 @@ def PipelineManager(request): "connection_type", "name", "host", "cloud_id", "cloud_url", "pk", "policy__name", "policy_id", "policy__policy_type", "agent_id", "last_check_in", "status_blob", "desired_agent_version", + "logstash_version_resolved", ) if not is_embedded_connection(conn) ] @@ -109,6 +114,16 @@ def PipelineManager(request): # reports "unknown" (keeps the page-load fallback inspect card in sync # with the live AgentInspect endpoint). _normalize_status_blob_api_status(conn.get('status_blob')) + + blob = conn.get("status_blob") if isinstance(conn.get("status_blob"), dict) else None + agent_ver = blob.get("agent_version") if blob else None + conn["logstash_version"] = resolve_running_logstash_version( + logstash_version_resolved=conn.get("logstash_version_resolved"), + status_blob=blob, + ) + conn["agent_version_relation"] = agent_version_relation( + agent_ver, settings.__PREFERRED_LS_AGENT_VERSION__ + ) # Sort connections: centralized first, then by policy name # This groups agents with the same policy together @@ -359,8 +374,10 @@ def agent_status_stream(request): """ SSE endpoint — streams agent status for all agent connections every 5 seconds. - Each event is a JSON array of objects: {id, name, status} + Each event is a JSON array of objects: {id, name, status, logstash_version} where status is one of: 'restarting' | 'unhealthy' | 'healthy' | 'offline' + and logstash_version is the running Logstash version, or null if the agent + has never reported one. This mirrors the priority logic in the pipeline_manager.html template so the JS can update badges without a full page reload. @@ -409,12 +426,16 @@ def _event_stream(): now = datetime.now(timezone.utc) from PipelineManager.agent_modes import is_embedded_connection + from PipelineManager.agent_versions import ( + resolve_running_logstash_version, + ) connections = [ conn for conn in ConnectionTable.objects .filter(connection_type=ConnectionTable.ConnectionType.AGENT) - .values('pk', 'name', 'last_check_in', 'status_blob', 'agent_id', 'policy__policy_type') + .values('pk', 'name', 'last_check_in', 'status_blob', 'agent_id', + 'policy__policy_type', 'logstash_version_resolved') if not is_embedded_connection(conn) ] @@ -424,8 +445,20 @@ def _event_stream(): else: conn['is_online'] = False + # status_blob is already selected for _compute_status, so the + # version rides along for free and the LS pill stops needing a + # page reload to notice a Logstash upgrade. payload = json.dumps([ - {'id': conn['pk'], 'name': conn['name'], 'status': _compute_status(conn)} + { + 'id': conn['pk'], + 'name': conn['name'], + 'status': _compute_status(conn), + 'logstash_version': resolve_running_logstash_version( + logstash_version_resolved=conn.get('logstash_version_resolved'), + status_blob=conn.get('status_blob') + if isinstance(conn.get('status_blob'), dict) else None, + ), + } for conn in connections ]) yield f"data: {payload}\n\n" diff --git a/src/logstashui/PipelineManager/migrations/0028_apikey_admin_tokens.py b/src/logstashui/PipelineManager/migrations/0028_apikey_admin_tokens.py new file mode 100644 index 00000000..b3a8047d --- /dev/null +++ b/src/logstashui/PipelineManager/migrations/0028_apikey_admin_tokens.py @@ -0,0 +1,80 @@ +# Generated by Django 6.0.3 on 2026-09-03 21:00 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('PipelineManager', '0027_packaged_managed_default_ports'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='apikey', + name='created_at', + field=models.DateTimeField(auto_now_add=True, null=True), + ), + migrations.AddField( + model_name='apikey', + name='expires_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='apikey', + name='last_used_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='apikey', + name='name', + field=models.CharField(blank=True, default='', help_text='Human-readable label for this token', max_length=100), + ), + migrations.AddField( + model_name='apikey', + name='prefix', + field=models.CharField(blank=True, db_index=True, help_text='Unhashed lookup key for admin tokens; null for agent keys', max_length=12, null=True, unique=True), + ), + migrations.AddField( + model_name='apikey', + name='revoked_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='apikey', + name='user', + field=models.ForeignKey(blank=True, help_text='User this API token acts as (admin tokens only)', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='api_tokens', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='apikey', + name='connection', + field=models.ForeignKey(blank=True, help_text='Connection this API key belongs to (agent keys only)', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='api_keys', to='PipelineManager.connection'), + ), + # The four AlterFields below are unrelated pre-existing drift: help_text + # was edited in 638acfb without regenerating a migration. They emit no + # SQL, and absorbing them here stops the drift trailing into the next + # unrelated migration. + migrations.AlterField( + model_name='connection', + name='agent_api_port', + field=models.PositiveIntegerField(blank=True, help_text='Agent FastAPI port for this connection (packaged 9550; managed 9550+N; simulate 9500+N; embedded 9500)', null=True), + ), + migrations.AlterField( + model_name='connection', + name='logstash_api_port', + field=models.PositiveIntegerField(blank=True, help_text='Logstash HTTP API port for this connection (packaged 9600; managed 9700+N; simulate 9560+N; embedded 9560)', null=True), + ), + migrations.AlterField( + model_name='policy', + name='agent_api_port', + field=models.PositiveIntegerField(default=9500, help_text='Policy default agent FastAPI port (Packaged 9550 as-is; Managed 9550+N; Simulate/Embedded 9500+N or 9500)'), + ), + migrations.AlterField( + model_name='policy', + name='logstash_api_port', + field=models.PositiveIntegerField(default=9560, help_text='Policy default Logstash HTTP API port (Packaged 9600 as-is; Managed 9700+N; Simulate/Embedded 9560+N or 9560)'), + ), + ] diff --git a/src/logstashui/PipelineManager/migrations/0029_logstash_artifacts.py b/src/logstashui/PipelineManager/migrations/0029_logstash_artifacts.py new file mode 100644 index 00000000..8398526b --- /dev/null +++ b/src/logstashui/PipelineManager/migrations/0029_logstash_artifacts.py @@ -0,0 +1,45 @@ +#Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +#or more contributor license agreements. Licensed under the Elastic License; +#you may not use this file except in compliance with the Elastic License. + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('PipelineManager', '0028_apikey_admin_tokens'), + ] + + operations = [ + migrations.CreateModel( + name='LogstashArtifact', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('filename', models.CharField(help_text='Tarball filename, e.g. logstash-9.4.3-linux-x86_64.tar.gz', max_length=255, unique=True)), + ('version', models.CharField(db_index=True, help_text='Logstash version, e.g. 9.4.3', max_length=32)), + ('arch', models.CharField(help_text='Platform and architecture, e.g. linux-x86_64', max_length=32)), + ('source_url', models.CharField(blank=True, default='', help_text='Explicit upstream URL. Blank derives one from the base URL setting.', max_length=512)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('FETCHING', 'Downloading'), ('READY', 'Ready'), ('FAILED', 'Failed'), ('IMPORTING', 'Verifying import')], db_index=True, default='PENDING', max_length=16)), + ('size_bytes', models.BigIntegerField(blank=True, help_text='Total size, from the upstream Content-Length or the file on disk', null=True)), + ('bytes_downloaded', models.BigIntegerField(default=0, help_text='Progress counter, written on a time floor rather than per chunk')), + ('sha512', models.CharField(blank=True, default='', help_text='Verified SHA-512 of the published tarball', max_length=128)), + ('error', models.TextField(blank=True, default='', help_text='Why the last fetch failed')), + ('claimed_at', models.DateTimeField(blank=True, null=True)), + ('heartbeat_at', models.DateTimeField(blank=True, null=True)), + ('serve_count', models.PositiveIntegerField(default=0, help_text='Fresh tarball downloads by agents. Checksum fetches and resumed range requests hit the same row but are not counted')), + ('last_served_at', models.DateTimeField(blank=True, help_text='When the tarball was last downloaded in full', null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + options={ + 'db_table': 'logstash_artifact', + 'ordering': ['-created_at'], + }, + ), + migrations.AddField( + model_name='policy', + name='logstash_via_ui', + field=models.BooleanField(default=False, help_text='Fetch the Logstash tarball from LogstashUI instead of artifacts.elastic.co. Only meaningful when logstash_source=VERSION on a MANAGED or SIMULATE policy.'), + ), + ] diff --git a/src/logstashui/PipelineManager/models.py b/src/logstashui/PipelineManager/models.py index 90610ebf..159f5845 100644 --- a/src/logstashui/PipelineManager/models.py +++ b/src/logstashui/PipelineManager/models.py @@ -2,11 +2,16 @@ #or more contributor license agreements. Licensed under the Elastic License; #you may not use this file except in compliance with the Elastic License. +from django.conf import settings from django.db import models from Common.encryption import encrypt_credential, decrypt_credential from django.core.exceptions import ValidationError -from django.contrib.auth.hashers import make_password, check_password +from django.contrib.auth.hashers import make_password, check_password, identify_hasher +from django.utils import timezone +from datetime import timedelta import hashlib +import re +import secrets from Common import logstash_config_parse @@ -110,6 +115,14 @@ class LogstashSource(models.TextChoices): default="/opt/logstash-agent/logstash-versions", help_text="Directory for auto-downloaded Logstash versions" ) + logstash_via_ui = models.BooleanField( + default=False, + help_text=( + "Fetch the Logstash tarball from LogstashUI instead of " + "artifacts.elastic.co. Only meaningful when logstash_source=VERSION " + "on a MANAGED or SIMULATE policy." + ) + ) logstash_yml = models.TextField( help_text="Content of logstash.yml configuration file" ) @@ -701,35 +714,343 @@ def __str__(self): return f"{self.policy.name} - {self.name}" +#: Namespace marker on admin API tokens. Agent keys carry no prefix, so the +#: token middleware can tell the two apart from the header alone. +API_TOKEN_SCHEME = 'lsui' + + class ApiKey(models.Model): """ - Represents an API key used by an enrolled agent for authenticated polling/check-in. - Each API key belongs to a specific connection. + A hashed bearer credential. Two flavours share this table: + + * **Agent keys** — issued at enrollment, scoped to a ``connection``. The + agent sends ``connection_id`` in the request body, so the row is found + before the hash is ever checked and no lookup column is needed. These + rows have ``prefix=None``. + * **Admin API tokens** — issued from Management, scoped to a ``user``, and + presented as ``Authorization: ApiKey lsui__``. Here the + header is the *only* identifier, so ``prefix`` is stored unhashed and + indexed; without it, resolving a token would mean a PBKDF2 comparison + against every row in the table on every request. + + Exactly one of ``connection`` / ``user`` is set. """ connection = models.ForeignKey( Connection, on_delete=models.CASCADE, related_name='api_keys', - help_text="Connection this API key belongs to" + null=True, + blank=True, + help_text="Connection this API key belongs to (agent keys only)" + ) + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.CASCADE, + related_name='api_tokens', + null=True, + blank=True, + help_text="User this API token acts as (admin tokens only)" + ) + name = models.CharField( + max_length=100, + blank=True, + default='', + help_text="Human-readable label for this token" + ) + prefix = models.CharField( + max_length=12, + null=True, + blank=True, + unique=True, + db_index=True, + help_text="Unhashed lookup key for admin tokens; null for agent keys" ) api_key = models.CharField( max_length=512, help_text="Hashed API key for agent authentication" ) - + created_at = models.DateTimeField(null=True, blank=True, auto_now_add=True) + last_used_at = models.DateTimeField(null=True, blank=True) + revoked_at = models.DateTimeField(null=True, blank=True) + expires_at = models.DateTimeField(null=True, blank=True) + class Meta: verbose_name = 'API Key' verbose_name_plural = 'API Keys' - + def __str__(self): - return f"{self.connection.name} - API Key" - + if self.connection_id: + return f"{self.connection.name} - API Key" + return f"{self.name or 'unnamed'} - API Token" + + def clean(self): + # Not a DB CheckConstraint: constraint enforcement is uneven across the + # MySQL 8.0 floor, and this is only ever violated by our own code. + if bool(self.connection_id) == bool(self.user_id): + raise ValidationError( + "An ApiKey must belong to exactly one of connection or user." + ) + def save(self, *args, **kwargs): - # Always hash the API key before saving - if self.api_key: + # Hash on the way in, but only once. Renaming or revoking a token + # re-saves the row, and re-running make_password on a stored hash + # would silently invalidate the credential. + if self.api_key and not self._is_hashed(self.api_key): self.api_key = make_password(self.api_key) super().save(*args, **kwargs) - + + @staticmethod + def _is_hashed(value): + try: + identify_hasher(value) + except ValueError: + return False + return True + def verify_api_key(self, raw_api_key): """Verify a raw API key against the stored hash""" return check_password(raw_api_key, self.api_key) + + # -- admin API tokens --------------------------------------------------- + + @classmethod + def issue_for_user(cls, user, name='', expires_at=None): + """Mint an admin API token. Returns ``(instance, raw_token)``. + + The raw token is the only time the secret exists in plaintext — it is + not recoverable afterwards. + """ + # token_hex, not token_urlsafe: the prefix must contain no '_' so that + # split('_', 2) on the wire format is unambiguous. + prefix = secrets.token_hex(6) + secret = secrets.token_urlsafe(32) + token = cls( + user=user, + name=name, + prefix=prefix, + api_key=secret, + expires_at=expires_at, + ) + token.full_clean(exclude=['api_key'], validate_unique=False) + token.save() + return token, f"{API_TOKEN_SCHEME}_{prefix}_{secret}" + + @staticmethod + def parse_token(raw): + """Split a wire-format token into ``(prefix, secret)``. + + Returns ``(None, None)`` for anything that is not an admin token, + including agent keys, which carry no scheme marker. + """ + parts = (raw or '').split('_', 2) + if len(parts) != 3 or parts[0] != API_TOKEN_SCHEME: + return None, None + if not parts[1] or not parts[2]: + return None, None + return parts[1], parts[2] + + @property + def masked(self): + """Display form for the token list — prefix only, never the secret.""" + return f"{API_TOKEN_SCHEME}_{self.prefix}_…" if self.prefix else '' + + @property + def is_expired(self): + return self.expires_at is not None and self.expires_at <= timezone.now() + + @property + def is_active(self): + return self.revoked_at is None and not self.is_expired + + +#: Release tarballs LogstashUI is willing to cache and serve. Anchored, and with +#: no path separators in any branch, so a filename that matches can never escape +#: the cache directory. +ARTIFACT_FILENAME_RE = re.compile( + r'^logstash-(?P[0-9][0-9A-Za-z.+-]{0,31})' + r'-(?Plinux|darwin|windows)' + r'-(?Px86_64|aarch64)' + r'\.tar\.gz(?P\.sha512)?$' +) + +#: Checksum sidecars are a few dozen bytes. Serving one must not consume a slot +#: in the download semaphore, or a burst of them starves real transfers. +SMALL_FILE_BYTES = 1024 * 1024 + + +def parse_artifact_filename(filename): + """Validate a requested filename. + + Returns ``(tarball_name, version, arch, is_checksum)``, where ``tarball_name`` + is the ``.tar.gz`` even when the checksum sidecar was requested — both files + belong to one :class:`LogstashArtifact` row and one upstream fetch. + + Returns ``None`` for anything unrecognized, which callers answer with 404. + """ + match = ARTIFACT_FILENAME_RE.match(filename or '') + if match is None: + return None + is_checksum = bool(match.group('checksum')) + tarball = filename[:-len('.sha512')] if is_checksum else filename + arch = f"{match.group('platform')}-{match.group('arch')}" + return tarball, match.group('version'), arch, is_checksum + + +class LogstashArtifact(models.Model): + """A Logstash release tarball cached locally and served to agents. + + One row covers the ``.tar.gz`` and its ``.sha512`` sidecar; a request for + either resolves here and a single fetch pulls both. + + The status field doubles as the cross-process lock. LogstashUI has no shared + cache backend (no ``CACHES`` in settings, so Django falls back to per-process + LocMemCache), and gunicorn runs 2+ worker processes, so an in-memory lock + cannot prevent two workers starting the same 450 MB download. A conditional + UPDATE on this row can — see :meth:`claim_for_fetch`. + """ + + class Status(models.TextChoices): + PENDING = 'PENDING', 'Pending' + FETCHING = 'FETCHING', 'Downloading' + READY = 'READY', 'Ready' + FAILED = 'FAILED', 'Failed' + IMPORTING = 'IMPORTING', 'Verifying import' + + #: A claim whose heartbeat is older than this is assumed dead and may be + #: taken over. Fetch greenlets die with their gunicorn worker, so on any + #: restart mid-download this is the recovery path, not an edge case. + STALE_CLAIM_SECONDS = 120 + + filename = models.CharField( + max_length=255, + unique=True, + help_text="Tarball filename, e.g. logstash-9.4.3-linux-x86_64.tar.gz" + ) + version = models.CharField( + max_length=32, + db_index=True, + help_text="Logstash version, e.g. 9.4.3" + ) + arch = models.CharField( + max_length=32, + help_text="Platform and architecture, e.g. linux-x86_64" + ) + source_url = models.CharField( + max_length=512, + blank=True, + default="", + help_text="Explicit upstream URL. Blank derives one from the base URL setting." + ) + status = models.CharField( + max_length=16, + choices=Status.choices, + default=Status.PENDING, + db_index=True, + ) + size_bytes = models.BigIntegerField( + null=True, + blank=True, + help_text="Total size, from the upstream Content-Length or the file on disk" + ) + bytes_downloaded = models.BigIntegerField( + default=0, + help_text="Progress counter, written on a time floor rather than per chunk" + ) + sha512 = models.CharField( + max_length=128, + blank=True, + default="", + help_text="Verified SHA-512 of the published tarball" + ) + error = models.TextField( + blank=True, + default="", + help_text="Why the last fetch failed" + ) + claimed_at = models.DateTimeField(null=True, blank=True) + heartbeat_at = models.DateTimeField(null=True, blank=True) + serve_count = models.PositiveIntegerField( + default=0, + help_text=( + "Fresh tarball downloads by agents. Checksum fetches and resumed " + "range requests hit the same row but are not counted" + ) + ) + last_served_at = models.DateTimeField( + null=True, + blank=True, + help_text="When the tarball was last downloaded in full" + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + db_table = 'logstash_artifact' + ordering = ['-created_at'] + + def __str__(self): + return f"{self.filename} ({self.status})" + + @property + def checksum_filename(self): + return f"{self.filename}.sha512" + + @property + def percent(self): + """Whole-percent progress, or None when the total is not yet known.""" + if not self.size_bytes: + return None + return min(100, int(self.bytes_downloaded * 100 / self.size_bytes)) + + def resolve_source_url(self, base_url): + """Upstream URL for the tarball. An explicit source_url wins.""" + if self.source_url: + return self.source_url + return f"{base_url.rstrip('/')}/{self.filename}" + + @classmethod + def claim_for_fetch(cls, pk, *, now=None): + """Atomically take ownership of a download. Returns True if we won. + + A single conditional UPDATE is the whole mechanism. It is race-free on + every supported engine: PostgreSQL re-evaluates the WHERE clause after + taking the row lock, InnoDB does a current read so the loser sees the + winner's committed FETCHING, and SQLite has one global writer. Exactly + one caller comes back with a rowcount of 1. + + A FETCHING row whose heartbeat has gone stale is also claimable, which + is how a download orphaned by a worker restart gets picked back up. + """ + now = now or timezone.now() + stale_before = now - timedelta(seconds=cls.STALE_CLAIM_SECONDS) + updated = cls.objects.filter(pk=pk).filter( + models.Q(status__in=[cls.Status.PENDING, cls.Status.FAILED]) + | models.Q(status=cls.Status.FETCHING, heartbeat_at__lt=stale_before) + | models.Q(status=cls.Status.FETCHING, heartbeat_at__isnull=True) + ).update( + status=cls.Status.FETCHING, + claimed_at=now, + heartbeat_at=now, + bytes_downloaded=0, + error='', + ) + return updated == 1 + + @classmethod + def release_claim(cls, pk): + """Hand a claim back without failing it, for the over-capacity path.""" + return cls.objects.filter(pk=pk, status=cls.Status.FETCHING).update( + status=cls.Status.PENDING, + claimed_at=None, + heartbeat_at=None, + ) + + @classmethod + def active_fetch_count(cls, *, now=None): + """Fetches genuinely in flight, ignoring rows abandoned by dead workers.""" + now = now or timezone.now() + stale_before = now - timedelta(seconds=cls.STALE_CLAIM_SECONDS) + return cls.objects.filter( + status=cls.Status.FETCHING, + heartbeat_at__gte=stale_before, + ).count() diff --git a/src/logstashui/PipelineManager/policies_crud.py b/src/logstashui/PipelineManager/policies_crud.py index 65191e09..f4d455af 100644 --- a/src/logstashui/PipelineManager/policies_crud.py +++ b/src/logstashui/PipelineManager/policies_crud.py @@ -35,6 +35,7 @@ def get_policies(request): 'settings_path', 'logs_path', 'binary_path', 'data_path', 'agent_api_port', 'logstash_api_port', 'keystore_env_file', 'logstash_source', 'logstash_version', 'logstash_download_dir', + 'logstash_via_ui', 'logstash_yml', 'jvm_options', 'log4j2_properties', 'current_revision_number', 'last_deployed_at', 'connection_count', 'created_at', 'updated_at' @@ -146,6 +147,7 @@ def _optional_int(key, default): logstash_download_dir=normalize_agent_opt_path( data.get('logstash_download_dir') or '/opt/logstash-agent/logstash-versions' ) or '/opt/logstash-agent/logstash-versions', + logstash_via_ui=bool(data.get('logstash_via_ui', False)), agent_api_port=_optional_int('agent_api_port', agent_port_default), logstash_api_port=_optional_int('logstash_api_port', ls_port_default), logstash_yml=logstash_yml, @@ -160,6 +162,18 @@ def _optional_int(key, default): apply_simulate_path_bundle(policy) policy.save() + from PipelineManager.agent_versions import resolve_persisted_binary_path + + resolved_binary = resolve_persisted_binary_path( + source=policy.logstash_source, + version=policy.logstash_version, + download_dir=policy.logstash_download_dir, + binary_path=policy.binary_path, + ) + if resolved_binary != policy.binary_path: + policy.binary_path = resolved_binary + policy.save(update_fields=["binary_path"]) + # Generate enrollment token for the new policy enrollment_token = secrets.token_urlsafe(32) @@ -253,6 +267,8 @@ def update_policy(request): policy.logstash_download_dir = normalize_agent_opt_path( data['logstash_download_dir'] ) or data['logstash_download_dir'] + if 'logstash_via_ui' in data: + policy.logstash_via_ui = bool(data['logstash_via_ui']) if 'binary_path' in data: policy.binary_path = normalize_agent_opt_path(data['binary_path']) or data[ 'binary_path' @@ -290,6 +306,8 @@ def update_policy(request): policy.logstash_download_dir = normalize_agent_opt_path( data['logstash_download_dir'] ) or data['logstash_download_dir'] + if 'logstash_via_ui' in data: + policy.logstash_via_ui = bool(data['logstash_via_ui']) if 'agent_api_port' in data and data['agent_api_port'] is not None: try: policy.agent_api_port = int(data['agent_api_port']) @@ -307,6 +325,15 @@ def update_policy(request): if 'log4j2_properties' in data: policy.log4j2_properties = data['log4j2_properties'] + from PipelineManager.agent_versions import resolve_persisted_binary_path + + policy.binary_path = resolve_persisted_binary_path( + source=policy.logstash_source, + version=policy.logstash_version, + download_dir=policy.logstash_download_dir, + binary_path=policy.binary_path, + ) + policy.save() logger.info(f"User '{request.user.username}' updated policy '{policy_name}'") @@ -451,6 +478,7 @@ def clone_policy(request): logstash_version=source_policy.logstash_version, logstash_download_dir=normalize_agent_opt_path(source_policy.logstash_download_dir) or source_policy.logstash_download_dir, + logstash_via_ui=source_policy.logstash_via_ui, logstash_yml=source_policy.logstash_yml, jvm_options=source_policy.jvm_options, log4j2_properties=source_policy.log4j2_properties, diff --git a/src/logstashui/PipelineManager/simulation.py b/src/logstashui/PipelineManager/simulation.py index 130ec6de..31267c4d 100644 --- a/src/logstashui/PipelineManager/simulation.py +++ b/src/logstashui/PipelineManager/simulation.py @@ -225,6 +225,10 @@ def SimulatePipeline(request): or request.build_absolute_uri("/") ).rstrip("/") + from LogstashUI.insecure_http import force_http_url + + logstash_ui_url = force_http_url(logstash_ui_url) + logger.debug("USING THIS URL: %s", logstash_ui_url) # Recursive function to instrument plugins, including nested conditionals step_counter = [0] # Use list to maintain counter across recursive calls diff --git a/src/logstashui/PipelineManager/static/js/agent_policies.js b/src/logstashui/PipelineManager/static/js/agent_policies.js index 46d00d8f..d7526158 100644 --- a/src/logstashui/PipelineManager/static/js/agent_policies.js +++ b/src/logstashui/PipelineManager/static/js/agent_policies.js @@ -6,6 +6,46 @@ let originalFileContents = {}; let changedFiles = new Set(); const DRAFT_POLICY_VALUE = '__draft__'; +const SYSTEM_BINARY_PATH = '/usr/share/logstash/bin'; +const DEFAULT_LS_DOWNLOAD_DIR = '/opt/logstash-agent/logstash-versions'; + +function deriveVersionBinaryPath(downloadDir, version) { + const ver = (version || '').trim(); + if (!ver) return null; + const root = (downloadDir || DEFAULT_LS_DOWNLOAD_DIR).replace(/\/+$/, '') || DEFAULT_LS_DOWNLOAD_DIR; + return `${root}/logstash-${ver}/bin`; +} + +function isDerivedVersionBinaryPath(path, downloadDir) { + const p = (path || '').replace(/\/+$/, ''); + if (!p) return false; + const root = (downloadDir || DEFAULT_LS_DOWNLOAD_DIR).replace(/\/+$/, '') || DEFAULT_LS_DOWNLOAD_DIR; + const prefix = `${root}/logstash-`; + const suffix = '/bin'; + if (!p.startsWith(prefix) || !p.endsWith(suffix)) return false; + const mid = p.slice(prefix.length, p.length - suffix.length); + return mid.length > 0 && !mid.includes('/'); +} + +function syncVersionBinaryPath() { + const sourceEl = document.getElementById('logstashSource'); + const versionEl = document.getElementById('logstashVersion'); + const downloadEl = document.getElementById('logstashDownloadDir'); + const binaryEl = document.getElementById('binaryPath'); + if (!binaryEl) return; + const source = sourceEl?.value || 'SYSTEM'; + const version = versionEl?.value || ''; + const downloadDir = downloadEl?.value || DEFAULT_LS_DOWNLOAD_DIR; + const current = binaryEl.value.trim(); + const derived = deriveVersionBinaryPath(downloadDir, version); + if (source === 'VERSION') { + if (derived && (!current || current === SYSTEM_BINARY_PATH || isDerivedVersionBinaryPath(current, downloadDir))) { + binaryEl.value = derived; + } + } else if (isDerivedVersionBinaryPath(current, downloadDir)) { + binaryEl.value = SYSTEM_BINARY_PATH; + } +} const POLICY_TYPE_INFO = { PACKAGED: { @@ -992,7 +1032,11 @@ document.addEventListener('DOMContentLoaded', function() { document.querySelectorAll('.file-tab').forEach(tab => { tab.addEventListener('click', function() { const file = this.dataset.file; - + + // Every branch below either returns or falls through to a different + // view, so stop here and let the nodes branch restart it. + stopPolicyNodesPolling(); + // Update active tab document.querySelectorAll('.file-tab').forEach(t => { t.classList.remove('active'); @@ -1151,8 +1195,9 @@ document.addEventListener('DOMContentLoaded', function() { if (keystoreView) keystoreView.classList.add('hidden'); if (nodesView) { nodesView.classList.remove('hidden'); - // Load nodes for current policy + // Load nodes for current policy, then keep them current loadPolicyNodes(); + startPolicyNodesPolling(); } return; } @@ -1512,6 +1557,7 @@ document.addEventListener('DOMContentLoaded', function() { setFieldDisabled(document.getElementById('logstashSource'), isEmbedded); setFieldDisabled(document.getElementById('logstashVersion'), isEmbedded); setFieldDisabled(document.getElementById('logstashDownloadDir'), isEmbedded); + setFieldDisabled(document.getElementById('logstashViaUi'), isEmbedded); // Port defaults informational for system multi-instance (enroll formula wins) setFieldDisabled(document.getElementById('agentApiPort'), isEmbedded || isSystemPathLocked); setFieldDisabled(document.getElementById('logstashApiPort'), isEmbedded || isSystemPathLocked); @@ -1599,6 +1645,12 @@ document.addEventListener('DOMContentLoaded', function() { if (hint) { hint.classList.toggle('hidden', !showVersion); } + // Only Managed and Simulate ever download a tarball. Packaged uses the OS + // package and Embedded runs in-process, so the proxy is meaningless there. + const ptype = document.getElementById('policyTypeSelect')?.value || ''; + const viaUiApplies = showVersion && (ptype === 'MANAGED' || ptype === 'SIMULATE'); + document.getElementById('logstashViaUiWrap')?.classList.toggle('hidden', !viaUiApplies); + syncVersionBinaryPath(); } function setPolicyFieldValue(id, value) { @@ -1791,6 +1843,22 @@ document.addEventListener('DOMContentLoaded', function() { toggleVersionFieldsVisibility(); if (typeof detectChanges === 'function') detectChanges(); }); + document.getElementById('logstashVersion')?.addEventListener('input', () => { + syncVersionBinaryPath(); + if (typeof detectChanges === 'function') detectChanges(); + }); + document.getElementById('logstashVersion')?.addEventListener('change', () => { + syncVersionBinaryPath(); + if (typeof detectChanges === 'function') detectChanges(); + }); + document.getElementById('logstashDownloadDir')?.addEventListener('input', () => { + syncVersionBinaryPath(); + if (typeof detectChanges === 'function') detectChanges(); + }); + document.getElementById('logstashDownloadDir')?.addEventListener('change', () => { + syncVersionBinaryPath(); + if (typeof detectChanges === 'function') detectChanges(); + }); // Load policies on page load, auto-selecting a policy if policy_id is in the URL const _urlParams = new URLSearchParams(window.location.search); @@ -1896,6 +1964,9 @@ async function loadPolicyData(policyValue) { if (versionEl) versionEl.value = policy.logstash_version || ''; const downloadEl = document.getElementById('logstashDownloadDir'); if (downloadEl) downloadEl.value = policy.logstash_download_dir || '/opt/logstash-agent/logstash-versions'; + const viaUiEl = document.getElementById('logstashViaUi'); + if (viaUiEl) viaUiEl.checked = !!policy.logstash_via_ui; + syncVersionBinaryPath(); const agentPortEl = document.getElementById('agentApiPort'); if (agentPortEl) agentPortEl.value = policy.agent_api_port ?? 9500; const lsPortEl = document.getElementById('logstashApiPort'); @@ -2187,6 +2258,7 @@ async function savePolicyChanges() { const logstashSource = document.getElementById('logstashSource')?.value || 'SYSTEM'; const logstashVersion = document.getElementById('logstashVersion')?.value || ''; const logstashDownloadDir = document.getElementById('logstashDownloadDir')?.value || ''; + const logstashViaUi = !!document.getElementById('logstashViaUi')?.checked; const agentApiPort = parseInt(document.getElementById('agentApiPort')?.value, 10); const logstashApiPort = parseInt(document.getElementById('logstashApiPort')?.value, 10); @@ -2232,6 +2304,7 @@ async function savePolicyChanges() { logstash_source: logstashSource, logstash_version: logstashVersion, logstash_download_dir: logstashDownloadDir, + logstash_via_ui: logstashViaUi, agent_api_port: Number.isFinite(agentApiPort) ? agentApiPort : undefined, logstash_api_port: Number.isFinite(logstashApiPort) ? logstashApiPort : undefined, logstash_yml: window.policyFileContents ? window.policyFileContents['logstash.yml'] : '', @@ -3211,10 +3284,18 @@ const POLICY_CONFIG_FIELD_IDS = [ 'logstashSource', 'logstashVersion', 'logstashDownloadDir', + 'logstashViaUi', 'agentApiPort', 'logstashApiPort', ]; +// A checkbox reports value "on" whether or not it is checked, so reading .value +// would make logstashViaUi permanently invisible to the change tracker. +function policyFieldValue(el) { + if (!el) return ''; + return el.type === 'checkbox' ? String(el.checked) : (el.value ?? ''); +} + // Store original content when policy loads function storeOriginalContent() { originalFileContents = {}; @@ -3227,7 +3308,7 @@ function storeOriginalContent() { // Store original Policy Config form fields (paths, simulate source, etc.) POLICY_CONFIG_FIELD_IDS.forEach((id) => { - originalFileContents[id] = document.getElementById(id)?.value ?? ''; + originalFileContents[id] = policyFieldValue(document.getElementById(id)); }); // Reset all visual indicators @@ -3293,7 +3374,7 @@ function detectChanges() { let configChanged = false; POLICY_CONFIG_FIELD_IDS.forEach((id) => { const el = document.getElementById(id); - const current = el?.value ?? ''; + const current = policyFieldValue(el); const original = originalFileContents[id] ?? ''; const fieldChanged = current !== original; if (el) { @@ -3479,12 +3560,14 @@ function validateJvmHeapSettings() { // Note: All guide-related functions have been moved to logstashyml_guides.js -// Load nodes for the current policy -function loadPolicyNodes() { +// Load nodes for the current policy. +// `quiet` suppresses toasts, for the background poll below — a flaky network +// would otherwise stack a new error toast every 10 seconds. +function loadPolicyNodes({ quiet = false } = {}) { const policySelect = document.getElementById('policySelect'); const selectedOption = policySelect.options[policySelect.selectedIndex]; const policyId = selectedOption.dataset.policyId; - + if (!policyId) { console.error('No policy ID available'); return; @@ -3497,15 +3580,46 @@ function loadPolicyNodes() { renderPolicyNodes(data.nodes); } else { console.error('Error loading nodes:', data.error); - showToast(data.error || 'Failed to load nodes', 'error'); + if (!quiet) showToast(data.error || 'Failed to load nodes', 'error'); } }) .catch(error => { console.error('Error loading nodes:', error); - showToast('Failed to load nodes', 'error'); + if (!quiet) showToast('Failed to load nodes', 'error'); }); } +// The Agents table has no live channel of its own — unlike the Connections +// page, which has an SSE stream — so poll while it is on screen. That is what +// keeps the LS version pill current after Logstash is upgraded on a host. +// 10s rather than the stream's 5s: GetPolicyNodes does per-node SNMP and CPM +// lookups, and this is a fetch per client rather than work the server is +// already doing. +const NODES_POLL_MS = 10000; +let _nodesPollTimer = null; + +function startPolicyNodesPolling() { + stopPolicyNodesPolling(); + _nodesPollTimer = setInterval(() => { + // A hidden tab still fires intervals. Skip the fetch rather than the + // timer, so polling resumes the moment the tab is focused again. + if (document.hidden) return; + const nodesView = document.getElementById('nodesView'); + if (!nodesView || nodesView.classList.contains('hidden')) { + stopPolicyNodesPolling(); + return; + } + loadPolicyNodes({ quiet: true }); + }, NODES_POLL_MS); +} + +function stopPolicyNodesPolling() { + if (_nodesPollTimer !== null) { + clearInterval(_nodesPollTimer); + _nodesPollTimer = null; + } +} + // Render nodes in the table function renderPolicyNodes(nodes) { const tbody = document.getElementById('nodesTableBody'); @@ -3534,6 +3648,9 @@ function renderPolicyNodes(nodes) { const badges = [ `LogstashAgent` ]; + if (node.logstash_version) { + badges.push(`LS ${node.logstash_version}`); + } if (node.cpm_enabled) { badges.push(`CPM`); } diff --git a/src/logstashui/PipelineManager/static/js/agent_status_sse.js b/src/logstashui/PipelineManager/static/js/agent_status_sse.js index f7f19c4c..f8036e32 100644 --- a/src/logstashui/PipelineManager/static/js/agent_status_sse.js +++ b/src/logstashui/PipelineManager/static/js/agent_status_sse.js @@ -8,10 +8,12 @@ * Connects to /PipelineManager/AgentStatusStream/ and updates the status * badge for each agent connection without a full page reload. * - * Two surfaces are kept in sync: + * Three surfaces are kept in sync: * 1. The list badge — the pill inside .status-container[data-agent-id] * 2. The modal summary badge — [data-modal-badge] inside #agentInspectContent, * only updated when the flyout is currently open for that agent. + * 3. The cyan Logstash version pill inside .ls-version-container[data-agent-id], + * so upgrading Logstash on a host is visible without a page reload. */ const BADGE_SVG = ``; @@ -67,7 +69,16 @@ function _buildModalBadge(cfg) { return span; } -function _applyUpdate(id, name, status) { +function _buildVersionPill(version) { + const span = document.createElement('span'); + span.className = 'inline-flex items-center px-2 py-0.5 rounded-full text-xs font-medium bg-cyan-500/15 text-cyan-300 border border-cyan-500/30'; + // textContent, not innerHTML: the version is agent-reported. + span.title = `Logstash ${version}`; + span.textContent = `LS ${version}`; + return span; +} + +function _applyUpdate(id, name, status, logstashVersion) { const cfg = STATUS_CONFIG[status] || STATUS_CONFIG.offline; // 1. Update list badge @@ -84,6 +95,17 @@ function _applyUpdate(id, name, status) { existing.replaceWith(_buildModalBadge(cfg)); } } + + // 3. Update the Logstash version pill. Only ever written for a truthy value: + // an agent whose Logstash is stopped reports no version, and the last known + // one is more useful than an empty gap next to an offline badge. + if (logstashVersion) { + const versionContainer = document.querySelector(`.ls-version-container[data-agent-id="${id}"]`); + if (versionContainer && versionContainer.textContent.trim() !== `LS ${logstashVersion}`) { + versionContainer.innerHTML = ''; + versionContainer.appendChild(_buildVersionPill(logstashVersion)); + } + } } // ── SSE connection ──────────────────────────────────────────────────────────── @@ -94,7 +116,8 @@ function _connect() { source.onmessage = function (event) { try { const updates = JSON.parse(event.data); - updates.forEach(({ id, name, status }) => _applyUpdate(id, name, status)); + updates.forEach(({ id, name, status, logstash_version }) => + _applyUpdate(id, name, status, logstash_version)); } catch (e) { console.error('[AgentSSE] Failed to parse event data:', e); } diff --git a/src/logstashui/PipelineManager/templates/components/pipeline_manager/agent_policies.html b/src/logstashui/PipelineManager/templates/components/pipeline_manager/agent_policies.html index a9862953..f1972946 100644 --- a/src/logstashui/PipelineManager/templates/components/pipeline_manager/agent_policies.html +++ b/src/logstashui/PipelineManager/templates/components/pipeline_manager/agent_policies.html @@ -834,6 +834,19 @@

Get Started with Agent Policies +