diff --git a/.github/workflows/e2e-nightly.yaml b/.github/workflows/e2e-nightly.yaml index bb51be8..b05244a 100644 --- a/.github/workflows/e2e-nightly.yaml +++ b/.github/workflows/e2e-nightly.yaml @@ -1,8 +1,8 @@ name: Nightly end-to-end # Functions built and executed in the cluster (the executor on -# Docker-in-Docker), with every optional component, which takes longer than -# PR checks should. +# Docker-in-Docker, the built-in object storage), with every optional +# component, which takes longer than PR checks should. on: schedule: - cron: "17 3 * * *" @@ -42,10 +42,8 @@ jobs: kubectl apply --server-side -f "https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/standard-install.yaml" helm upgrade --install traefik traefik --repo https://traefik.github.io/charts --version "${TRAEFIK_CHART_VERSION}" \ -n traefik --create-namespace -f hack/fixtures/traefik-values.yaml --wait --timeout 5m - - name: Start the compute plane - run: hack/compute-plane.sh "${RUNNER_TEMP}/compute-values.yaml" - name: Install with every optional component - run: make install VALUES="charts/appwrite/ci/full-values.yaml ${RUNNER_TEMP}/compute-values.yaml" + run: make install VALUES=charts/appwrite/ci/full-values.yaml - name: helm test run: make helm-test - name: Smoke test (includes a screenshot through the browser service) @@ -73,6 +71,3 @@ jobs: kubectl -n appwrite-executor logs deploy/appwrite-executor --all-containers --prefix --tail=200 || true kubectl -n appwrite-executor exec deploy/appwrite-executor -c dind -- docker ps -a || true echo "::endgroup::" - echo "::group::aw-s3" - docker logs aw-s3 --tail=200 || true - echo "::endgroup::" diff --git a/.github/workflows/test-install.yaml b/.github/workflows/test-install.yaml index 33d3374..dde30fa 100644 --- a/.github/workflows/test-install.yaml +++ b/.github/workflows/test-install.yaml @@ -29,7 +29,7 @@ concurrency: jobs: install: - name: ${{ matrix.scenario }} (k8s ${{ matrix.kubernetes }})${{ matrix.upgrade-from && format(' upgrade from {0}', matrix.upgrade-from) || '' }} + name: ${{ matrix.scenario }} (k8s ${{ matrix.kubernetes }})${{ matrix.upgrade-from && format(' upgrade from chart {0}', matrix.upgrade-from) || '' }} runs-on: >- ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) @@ -52,10 +52,12 @@ jobs: kubernetes: v1.37.0 - scenario: grouped-workers kubernetes: v1.37.0 - # Install the previous Appwrite release, then upgrade to the chart's. + # Install a published chart release with its own defaults and + # Appwrite version, then upgrade to this chart's: the chart upgrade + # and Appwrite's data migration. - scenario: default kubernetes: v1.37.0 - upgrade-from: 2.2.0 + upgrade-from: 1.0.0 # PostgreSQL, Redis and ClickHouse outside the release. - scenario: external-dbs kubernetes: v1.37.0 @@ -106,21 +108,21 @@ jobs: # instead of as a 15-minute install timeout. run: | # shellcheck disable=SC2016 # expanded inside the pod - kubectl -n appwrite run dns-check --rm -i --restart=Never --image=docker.io/appwrite/appwrite:2.3.0 --command -- sh -c ' + kubectl -n appwrite run dns-check --rm -i --restart=Never --image=docker.io/appwrite/appwrite:2.5.0 --command -- sh -c ' cat /etc/resolv.conf ok=0 for host in postgres.external-dbs.svc.cluster.local postgres.external-dbs.svc postgres.external-dbs; do if getent hosts "$host"; then echo "resolved $host"; else echo "FAILED $host"; ok=1; fi done exit $ok' - - name: Install Appwrite ${{ matrix.upgrade-from }} + - name: Install chart ${{ matrix.upgrade-from }} if: matrix.upgrade-from env: SCENARIO: ${{ matrix.scenario }} UPGRADE_FROM: ${{ matrix.upgrade-from }} run: | - helm upgrade --install appwrite charts/appwrite -n appwrite --create-namespace \ - -f "charts/appwrite/ci/${SCENARIO}-values.yaml" --set "image.tag=${UPGRADE_FROM}" --wait --timeout 15m + helm upgrade --install appwrite oci://ghcr.io/appwrite/charts/appwrite --version "${UPGRADE_FROM}" \ + -n appwrite --create-namespace -f "charts/appwrite/ci/${SCENARIO}-values.yaml" --wait --timeout 15m make smoke - name: Install env: diff --git a/.gitignore b/.gitignore index d51b1eb..78e47bd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ charts/*/tests/__snapshot__/*.tmp .idea/ .vscode/ .variables.php +.env-defaults.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index bf66ba3..83a0c77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,46 @@ annotation in `charts/appwrite/Chart.yaml`. ## [Unreleased] +### Changed + +- Functions and Sites are on by default: `compute.enabled` and + `compute.executor.privileged` default to `true`, so a plain install runs + the executor as a privileged pod in a namespace of its own. Set + `compute.enabled: false` on clusters that forbid privileged pods. +- Files are stored in object storage by default: `storage.device` defaults + to `s3`, served by the built-in SeaweedFS. To keep 1.0.0's behaviour, set + `storage.device: local` and `compute.enabled: false`. +- Appwrite 2.5.0, with console 1.2.74, executor 1.1.2, browser 0.3.6 and + geo 0.3.2, and the new trusted-proxy, internal-address and DNS settings in + the environment defaults ([networking.md](docs/networking.md#tls-and-forcehttps)). +- External S3 settings move to `storage.s3.external.endpoint` and + `storage.s3.external.region`, with `storage.s3.mode: external`. Setting + the old keys fails with where they went. +- ClickHouse drops its per-second metric, trace and text logs and gives + tables left behind by a config change a TTL, as Appwrite 2.4.0 does. +- CI upgrades from the published chart 1.0.0 with default values. + +### Added + +- Built-in object storage: SeaweedFS as a single-replica StatefulSet + (`storage.s3.mode: builtin`), with generated keys, the bucket created on + start, and a NetworkPolicy that admits only Appwrite, the orchestrator, + builds and the executor ([storage.md](docs/storage.md#built-in-object-storage)). +- Troubleshooting for deployments that fail with an internal error while + Functions and Sites are off. + +### Removed + +- The assistant: Appwrite removed it in 2.4.0. Setting `assistant.*` fails + with that message. + +### Fixed + +- Site screenshots: the browser gets the API Service in `ALLOWED_HOSTS`, as + browser 0.3.5 and later require for private addresses. +- `make env-defaults` works with Appwrite 2.4.0 and later, and no longer + leaves an error message in `files/env-defaults.yaml` when it fails. + ## [1.0.0] - 2026-10-13 First release of the chart, for Appwrite 2.3.0. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2db945b..a03322d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,12 +80,11 @@ make install VALUES="charts/appwrite/ci/default-values.yaml my-values.yaml" make kind-down ``` -Functions and Sites need S3 and the executor. `hack/compute-plane.sh` starts -an S3 server next to the kind cluster and writes the values for it: +Functions and Sites are on by default, on the built-in object storage. To +build and execute a function as the nightly workflow does: ```console -hack/compute-plane.sh /tmp/compute-values.yaml -make install VALUES="charts/appwrite/ci/full-values.yaml /tmp/compute-values.yaml" +make install VALUES=charts/appwrite/ci/full-values.yaml hack/functions-smoke.sh appwrite appwrite node-22 ``` @@ -157,7 +156,7 @@ also opens a pull request for `appVersion`). To follow it: new version's `app/config/variables.php`, and review the diff. New variables may need values or computed settings in `templates/_env.tpl`. 3. Compare the new version's `docker-compose.yml` with the chart: image tags - of the console, geo, assistant, browser and other components, the + of the console, geo, browser and other components, the orchestrator and executor versions, new workers or queues (`app/config/workers.php`, mirrored by the queue catalog in `templates/workers/_workers.tpl` and by `workers.queues`), and new tasks. @@ -193,9 +192,8 @@ hack/appwrite-e2e.sh ~/source/appwrite appwrite appwrite --filter testCreateUser It reconfigures the release for testing (development mode, open console sign-up, mail and request catchers), so only point it at a throwaway install. -Functions and Sites suites need the compute plane -(`hack/compute-plane.sh` writes the values) and the runtimes they use in -`compute.functionsRuntimes`. Suites that need a fresh instance (the first +Functions and Sites suites need the runtimes they use in +`compute.functionsRuntimes` (as `charts/appwrite/ci/full-values.yaml` sets). Suites that need a fresh instance (the first organization in Teams), GitHub or Gitea, or ClamAV don't pass against a shared install. diff --git a/Makefile b/Makefile index 9606143..6ae8a29 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,9 @@ docs: ## Regenerate the chart README with helm-docs .PHONY: env-defaults env-defaults: ## Regenerate files/env-defaults.yaml from Appwrite's variables.php at APP_VERSION curl -fsSL "https://raw.githubusercontent.com/appwrite/appwrite/$(APP_VERSION)/app/config/variables.php" -o .variables.php - php hack/gen-env-defaults.php .variables.php $(APP_VERSION) > $(CHART)/files/env-defaults.yaml + @# Into a temporary file first: PHP prints fatal errors to stdout. + php hack/gen-env-defaults.php .variables.php $(APP_VERSION) > .env-defaults.yaml || { rm -f .variables.php .env-defaults.yaml; exit 1; } + mv .env-defaults.yaml $(CHART)/files/env-defaults.yaml rm -f .variables.php .PHONY: images diff --git a/README.md b/README.md index 0e34c3e..88fa515 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ to production, swap each of those for a managed service, one at a time. | | | | --- | --- | -| Appwrite | 2.3.0 | +| Appwrite | 2.5.0 | | Kubernetes | 1.29 or later | | Helm | 4, or 3.18 or later | | Chart | `oci://ghcr.io/appwrite/charts/appwrite` | @@ -60,7 +60,7 @@ cluster. Without it, a database backup can't be read: kubectl -n appwrite get secret appwrite -o jsonpath='{.data._APP_OPENSSL_KEY_V1}' | base64 -d ``` -To install a branch, or before the first release is on `ghcr.io`, use +To install a branch, use `./charts/appwrite` from a clone in place of the `oci://` reference ([installing from source](docs/getting-started.md#installing-from-source)). @@ -88,7 +88,7 @@ Every guide is listed in [`docs/`](docs/README.md), and every example in | PostgreSQL | built-in StatefulSet | RDS, Cloud SQL, Azure, CloudNativePG; or MariaDB, MySQL, MongoDB | [External databases](docs/external-databases.md#postgresql) | | Redis | built-in StatefulSet | ElastiCache, Memorystore, Azure Cache, Valkey | [External databases](docs/external-databases.md#redis) | | ClickHouse | built-in StatefulSet | ClickHouse Cloud, Altinity operator | [External databases](docs/external-databases.md#clickhouse) | -| File storage | PersistentVolumeClaim | S3, R2, GCS, Spaces, MinIO, ... | [Storage](docs/storage.md#s3) | +| File storage, deployments and builds | built-in object storage (SeaweedFS) | S3, R2, GCS, Spaces, MinIO, ...; or a PersistentVolumeClaim | [Storage](docs/storage.md#s3) | | Email | off | any SMTP server: SES, SendGrid, Postmark, ... | [Integrations](docs/integrations.md#email-smtp) | | Git deployments, console sign-in, SMS, Sentry | off | GitHub, GitLab, Gitea, Bitbucket, Google, Twilio, ... | [Integrations](docs/integrations.md) | diff --git a/charts/appwrite/Chart.yaml b/charts/appwrite/Chart.yaml index 464c555..dc2ab0b 100644 --- a/charts/appwrite/Chart.yaml +++ b/charts/appwrite/Chart.yaml @@ -3,10 +3,10 @@ name: appwrite description: Appwrite, the open-source backend platform, on Kubernetes. Deploys the API, console, realtime, workers and schedulers with optional built-in PostgreSQL, Redis and ClickHouse. type: application version: 1.0.0 -appVersion: "2.3.0" +appVersion: "2.5.0" kubeVersion: ">=1.29.0-0" home: https://github.com/appwrite/helm-chart -icon: https://raw.githubusercontent.com/appwrite/appwrite/2.3.0/public/images/logos/appwrite-icon.svg +icon: https://raw.githubusercontent.com/appwrite/appwrite/2.5.0/public/images/logos/appwrite-icon.svg sources: - https://github.com/appwrite/helm-chart - https://github.com/appwrite/appwrite @@ -61,13 +61,11 @@ annotations: description: '`values.schema.json`, `helm test` pods, documentation from getting started to backups, GitOps and troubleshooting, and example values files for local, production, external services, AWS, GCP, Azure, and Functions and Sites installs.' artifacthub.io/images: | - name: appwrite - image: docker.io/appwrite/appwrite:2.3.0 - - name: assistant - image: docker.io/appwrite/assistant:0.8.4 + image: docker.io/appwrite/appwrite:2.5.0 - name: autogravity image: ghcr.io/appwrite/autogravity:0.0.11 - name: browser - image: docker.io/appwrite/browser:0.3.4 + image: docker.io/appwrite/browser:0.3.6 - name: builds image: ghcr.io/open-runtimes/orchestrator/job-sidecar:2.2.3 - name: builtin @@ -76,20 +74,22 @@ annotations: image: docker.io/library/redis:7.4.11-alpine - name: builtin image: docker.io/clickhouse/clickhouse-server:26.4.5-alpine + - name: builtin + image: docker.io/chrislusf/seaweedfs:4.48 - name: builtin-exporter image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 - name: builtin-exporter image: docker.io/oliver006/redis_exporter:v1.93.0-alpine - name: console - image: docker.io/appwrite/new:1.1.159-self-hosted + image: docker.io/appwrite/new:1.2.74-self-hosted - name: dind image: docker.io/library/docker:29.9.0-dind - name: embedding image: docker.io/appwrite/embedding:0.3.1 - name: executor - image: docker.io/openruntimes/executor:0.29.0 + image: docker.io/openruntimes/executor:1.1.2 - name: geo - image: docker.io/appwrite/geo:0.3.1 + image: docker.io/appwrite/geo:0.3.2 - name: orchestrator image: ghcr.io/open-runtimes/orchestrator/jobs-service:2.2.3 artifacthub.io/displayName: Appwrite diff --git a/charts/appwrite/README.md b/charts/appwrite/README.md index 1b63934..ec3c825 100644 --- a/charts/appwrite/README.md +++ b/charts/appwrite/README.md @@ -1,6 +1,6 @@ # Appwrite Helm chart -![Version: 1.0.0](https://img.shields.io/badge/Version-1.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.3.0](https://img.shields.io/badge/AppVersion-2.3.0-informational?style=flat-square) +![Version: 1.0.0](https://img.shields.io/badge/Version-1.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.5.0](https://img.shields.io/badge/AppVersion-2.5.0-informational?style=flat-square) Appwrite, the open-source backend platform, on Kubernetes. Deploys the API, console, realtime, workers and schedulers with optional built-in PostgreSQL, Redis and ClickHouse. @@ -16,9 +16,11 @@ Appwrite, the open-source backend platform, on Kubernetes. Deploys the API, cons - Ingress or Gateway API routes for the API, console, realtime, function and site previews and custom domains. - Worker topologies from one combined worker to one Deployment per queue. -- Functions and Sites in the cluster: builds run as unprivileged Jobs in their - own namespace, executions on an executor with its own Docker-in-Docker - daemon in another namespace (a privileged pod you opt into). +- Functions and Sites in the cluster, on by default: builds run as + unprivileged Jobs in their own namespace, executions on an executor with + its own Docker-in-Docker daemon in another namespace (a privileged pod; set + `compute.enabled=false` where that isn't allowed), on built-in S3-compatible + object storage. - Credentials generated once and kept, data migrations on upgrade, and guards against changing the encryption key or downgrading. @@ -28,6 +30,8 @@ Appwrite, the open-source backend platform, on Kubernetes. Deploys the API, cons - Helm 4, or Helm 3.18 or later - An ingress controller or a Gateway API implementation. Appwrite serves the console and the API on one hostname and needs `/v1` routed to the API. +- For Functions and Sites (on by default), a cluster that admits a privileged + pod in the executor's namespace, or `compute.enabled=false`. - For HTTPS: cert-manager, or certificates you provide ## Quick start @@ -60,7 +64,7 @@ kubectl -n appwrite get secret appwrite -o jsonpath='{.data._APP_OPENSSL_KEY_V1} To try Appwrite on a laptop first, or for a step-by-step install, see [Getting started](https://github.com/appwrite/helm-chart/blob/main/docs/getting-started.md). -To install a branch, or before the first release is on `ghcr.io`, use +To install a branch, use `./charts/appwrite` from a clone in place of the `oci://` reference ([installing from source](https://github.com/appwrite/helm-chart/blob/main/docs/getting-started.md#installing-from-source)). @@ -164,7 +168,7 @@ See the [architecture guide](https://github.com/appwrite/helm-chart/blob/main/do - **Highly available datastores.** The built-in ones are single replicas. Use managed services or operators in production ([guide](https://github.com/appwrite/helm-chart/blob/main/docs/external-databases.md)). -- **Metrics.** Appwrite 2.3.0 exposes no Prometheus metrics. +- **Metrics.** Appwrite 2.5.0 exposes no Prometheus metrics. ## Documentation @@ -178,7 +182,7 @@ See the [architecture guide](https://github.com/appwrite/helm-chart/blob/main/do | [Storage](https://github.com/appwrite/helm-chart/blob/main/docs/storage.md) | Volumes, ReadWriteMany, S3, antivirus | | [Email, Git and other integrations](https://github.com/appwrite/helm-chart/blob/main/docs/integrations.md) | SMTP, GitHub App, console sign-in, SMS, Sentry | | [Functions and Sites](https://github.com/appwrite/helm-chart/blob/main/docs/executor.md) | Builds as Jobs and the executor on Docker-in-Docker | -| [Optional components](https://github.com/appwrite/helm-chart/blob/main/docs/optional-components.md) | Browser, assistant, autogravity, embedding, MQTT | +| [Optional components](https://github.com/appwrite/helm-chart/blob/main/docs/optional-components.md) | Browser, autogravity, embedding, MQTT | | [Security](https://github.com/appwrite/helm-chart/blob/main/docs/security.md) | Pod security, credentials, NetworkPolicies, signatures | | [Upgrading](https://github.com/appwrite/helm-chart/blob/main/docs/upgrading.md) | Versions, migrations, rollbacks | | [Backup and restore](https://github.com/appwrite/helm-chart/blob/main/docs/backup-restore.md) | What to back up and how | @@ -457,18 +461,45 @@ Where files go (the volume or S3), and the `/storage` volume. | Key | Type | Default | Description | |-----|------|---------|-------------| -| storage.device | string | `"local"` | Where Appwrite stores files (`_APP_STORAGE_DEVICE`). | +| storage.device | string | `"s3"` | Where Appwrite stores files (`_APP_STORAGE_DEVICE`): `s3`, by default the built-in SeaweedFS (`storage.s3.mode`), or `local`, the `/storage` volume. Functions and Sites need `s3`. | | storage.limit | int | `30000000` | Maximum file size in bytes (`_APP_STORAGE_LIMIT`). | | storage.previewLimit | int | `20000000` | Maximum size in bytes of files Appwrite renders previews for (`_APP_STORAGE_PREVIEW_LIMIT`). | | storage.s3.accessKeyId.secretKey | string | `"s3-access-key"` | Key in the Secret. | | storage.s3.accessKeyId.secretName | string | `""` | Existing Secret holding the access key id. Takes precedence over `value`. | -| storage.s3.accessKeyId.value | string | `""` | Access key ID, set inline. Stored in the chart's Secret. | -| storage.s3.bucket | string | `""` | Bucket (`_APP_STORAGE_S3_BUCKET`). Required with `device: s3`. | -| storage.s3.endpoint | string | `""` | Endpoint of an S3-compatible store (`_APP_STORAGE_S3_ENDPOINT`), e.g. `https://fsn1.your-objectstorage.com` or `https://.r2.cloudflarestorage.com`. Empty uses AWS S3. Endpoints and regions per provider: docs/storage.md#s3. | -| storage.s3.region | string | `"us-east-1"` | Region (`_APP_STORAGE_S3_REGION`). | +| storage.s3.accessKeyId.value | string | `""` | Access key ID, set inline. Stored in the chart's Secret. With `mode: builtin`, generated once when empty. | +| storage.s3.bucket | string | `"appwrite"` | Bucket (`_APP_STORAGE_S3_BUCKET`). The built-in store creates it. | +| storage.s3.builtin.affinity | object | `{}` | Affinity. Empty inherits `defaults.affinity`. | +| storage.s3.builtin.containerSecurityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true}` | Container security context. | +| storage.s3.builtin.extraContainers | list | `[]` | Extra sidecar containers. | +| storage.s3.builtin.extraEnv | list | `[]` | Extra environment variables, as a list of EnvVar objects. | +| storage.s3.builtin.extraInitContainers | list | `[]` | Extra init containers. | +| storage.s3.builtin.extraVolumeMounts | list | `[]` | Extra volume mounts for the main container. | +| storage.s3.builtin.extraVolumes | list | `[]` | Extra volumes. | +| storage.s3.builtin.image.digest | string | `""` | Image digest. | +| storage.s3.builtin.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | +| storage.s3.builtin.image.registry | string | `"docker.io"` | Image registry. | +| storage.s3.builtin.image.repository | string | `"chrislusf/seaweedfs"` | Image repository. | +| storage.s3.builtin.image.tag | string | `"4.48"` | Image tag. | +| storage.s3.builtin.nodeSelector | object | `{}` | Node selector. Empty inherits `defaults.nodeSelector`. | +| storage.s3.builtin.persistence.accessModes | list | `["ReadWriteOnce"]` | Access modes. | +| storage.s3.builtin.persistence.annotations | object | `{}` | Annotations for the PersistentVolumeClaim. | +| storage.s3.builtin.persistence.enabled | bool | `true` | Persist data in a PersistentVolumeClaim. Without it, every restart loses the files and build output. | +| storage.s3.builtin.persistence.size | string | `"20Gi"` | Volume size: uploaded files, deployment sources and build output. | +| storage.s3.builtin.persistence.storageClass | string | `""` | Storage class. Empty uses `global.storageClass`, then the cluster default; `-` requests no class. | +| storage.s3.builtin.podAnnotations | object | `{}` | Pod annotations, merged over `defaults.podAnnotations`. | +| storage.s3.builtin.podLabels | object | `{}` | Pod labels, merged over `defaults.podLabels`. | +| storage.s3.builtin.podSecurityContext | object | `{"fsGroup":1000,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}}` | Pod security context. The image runs as root by default; the chart runs it as UID 1000. | +| storage.s3.builtin.priorityClassName | string | `""` | Priority class. Empty inherits `defaults.priorityClassName`. | +| storage.s3.builtin.resources | object | `{"limits":{"memory":"1Gi"},"requests":{"cpu":"50m","memory":"128Mi"}}` | Resource requests and limits. | +| storage.s3.builtin.terminationGracePeriodSeconds | int | `60` | Seconds to wait on shutdown. | +| storage.s3.builtin.tolerations | list | `[]` | Tolerations. Empty inherits `defaults.tolerations`. | +| storage.s3.builtin.volumeSizeLimitMB | int | `1024` | Size of each SeaweedFS volume file in MB. The store fills the disk with volumes of this size, so keep it well below `persistence.size`. | +| storage.s3.external.endpoint | string | `""` | Endpoint (`_APP_STORAGE_S3_ENDPOINT`), e.g. `https://fsn1.your-objectstorage.com` or `https://.r2.cloudflarestorage.com`. Empty uses AWS S3. Endpoints and regions per provider: docs/storage.md#s3. | +| storage.s3.external.region | string | `"us-east-1"` | Region (`_APP_STORAGE_S3_REGION`). | +| storage.s3.mode | string | `"builtin"` | `builtin` runs SeaweedFS, an S3-compatible store, in the release; `external` uses `storage.s3.external`. Only used with `device: s3`. | | storage.s3.secretAccessKey.secretKey | string | `"s3-secret-key"` | Key in the Secret. | | storage.s3.secretAccessKey.secretName | string | `""` | Existing Secret holding the secret access key. Takes precedence over `value`. | -| storage.s3.secretAccessKey.value | string | `""` | Secret access key, set inline. Stored in the chart's Secret. | +| storage.s3.secretAccessKey.value | string | `""` | Secret access key, set inline. Stored in the chart's Secret. With `mode: builtin`, generated once when empty. | | storage.volume.accessModes | list | `["ReadWriteOnce"]` | Access modes. Use ReadWriteMany to spread API and worker pods across nodes. They mount `/storage` even with S3 storage, so a ReadWriteOnce volume pins them to one node (and the API gets no PodDisruptionBudget, which would block drains). | | storage.volume.annotations | object | `{}` | Annotations for the PersistentVolumeClaim. | | storage.volume.colocateReadWriteOnce | bool | `true` | With a ReadWriteOnce volume, schedule every pod that mounts it on the same node so the volume can be attached. | @@ -553,7 +584,7 @@ The SMTP server Appwrite sends mail through. | console.autoscaling.targetMemoryUtilizationPercentage | string | `""` | Target average memory utilisation, in percent. Empty disables the metric. | | console.containerSecurityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true}` | Container security context. | | console.enabled | bool | `true` | Deploy the console. | -| console.env | object | `{"APPWRITE_ENDPOINT_SAME_ORIGIN":"true","VITE_CONSOLE_PROFILE":"self-hosted","VITE_GROWTH_ENDPOINT":"https://growth.appwrite.io/v1"}` | Environment variables for the console, as a map. | +| console.env | object | `{"APPWRITE_ENDPOINT_SAME_ORIGIN":"true","VITE_CONSOLE_PROFILE":"self-hosted"}` | Environment variables for the console, as a map. | | console.extraContainers | list | `[]` | Extra sidecar containers. | | console.extraEnv | list | `[]` | Extra environment variables, as a list of EnvVar objects. | | console.extraInitContainers | list | `[]` | Extra init containers. | @@ -563,7 +594,7 @@ The SMTP server Appwrite sends mail through. | console.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | | console.image.registry | string | `"docker.io"` | Image registry. | | console.image.repository | string | `"appwrite/new"` | Image repository. | -| console.image.tag | string | `"1.1.159-self-hosted"` | Image tag. Pinned to the console that ships with the chart's appVersion. | +| console.image.tag | string | `"1.2.74-self-hosted"` | Image tag. Pinned to the console that ships with the chart's appVersion. | | console.livenessProbe.enabled | bool | `true` | Enable the liveness probe. | | console.livenessProbe.failureThreshold | int | `3` | Consecutive failures before the probe counts as failed. | | console.livenessProbe.periodSeconds | int | `20` | Seconds between probes. | @@ -606,7 +637,7 @@ The SMTP server Appwrite sends mail through. | geo.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | | geo.image.registry | string | `"docker.io"` | Image registry. | | geo.image.repository | string | `"appwrite/geo"` | Image repository. | -| geo.image.tag | string | `"0.3.1"` | Image tag. | +| geo.image.tag | string | `"0.3.2"` | Image tag. | | geo.nodeSelector | object | `{}` | Node selector. Empty inherits `defaults.nodeSelector`. | | geo.podAnnotations | object | `{}` | Pod annotations, merged over `defaults.podAnnotations`. | | geo.podLabels | object | `{}` | Pod labels, merged over `defaults.podLabels`. | @@ -850,7 +881,7 @@ The build orchestrator and the executor. | compute.builds.sidecarImage.repository | string | `"open-runtimes/orchestrator/job-sidecar"` | Image repository of the sidecar that moves sources, outputs and caches for each build. | | compute.builds.sidecarImage.tag | string | `"2.2.3"` | Image tag. | | compute.builds.tolerations | list | `[]` | Tolerations for build pods. | -| compute.enabled | bool | `false` | Enable Functions and Sites. Needs `storage.device: s3` and `compute.executor.privileged` (see docs/executor.md). | +| compute.enabled | bool | `true` | Enable Functions and Sites. Needs `storage.device: s3`, and runs the executor as a privileged pod in a namespace of its own (see docs/executor.md). Set to false on clusters that forbid privileged pods. | | compute.executor.affinity | object | `{}` | Affinity. Empty inherits `defaults.affinity`. | | compute.executor.createNamespace | bool | `true` | Create the namespace, labelled for the `privileged` Pod Security Standard. Set to false to create it yourself. | | compute.executor.dind.addressPool | string | `"198.18.0.0/15"` | Addresses of the Docker daemon's networks, split into /24s: its bridge and the runtimes' network. They shadow anything the executor and runtimes would reach in the same range, so it must not overlap the S3 endpoint, the cluster or your networks. Docker's own default, 172.17.0.0/16 onwards, often does; this one is reserved for benchmarking (RFC 2544) and routed nowhere. | @@ -866,7 +897,7 @@ The build orchestrator and the executor. | compute.executor.image.digest | string | `""` | Image digest. | | compute.executor.image.registry | string | `"docker.io"` | Image registry. The executor image is pulled by the Docker daemon in the pod, not by the kubelet, so image pull secrets don't apply to it. | | compute.executor.image.repository | string | `"openruntimes/executor"` | Image repository. | -| compute.executor.image.tag | string | `"0.29.0"` | Image tag. | +| compute.executor.image.tag | string | `"1.1.2"` | Image tag. | | compute.executor.images | list | `[]` | Runtime images to pull when the executor starts (`OPR_EXECUTOR_IMAGES`). Empty derives them from `functionsRuntimes` and `sitesRuntimes`, e.g. `node-22` becomes `openruntimes/node:v5-22`; with both of those empty too, nothing is pulled ahead of the first executions. | | compute.executor.inactiveThreshold | int | `60` | Seconds a runtime may stay idle before it's removed (`OPR_EXECUTOR_INACTIVE_THRESHOLD`). The executor removes idle runtimes in its maintenance pass, which the chart runs hourly (`OPR_EXECUTOR_MAINTENANCE_INTERVAL`), so a runtime can stay up to an hour longer. | | compute.executor.namespace | string | `""` | Namespace of the executor. Defaults to `-executor`. Use a different one per release when several releases share a namespace. | @@ -877,14 +908,14 @@ The build orchestrator and the executor. | compute.executor.podAnnotations | object | `{}` | Pod annotations, merged over `defaults.podAnnotations`. | | compute.executor.podLabels | object | `{}` | Pod labels, merged over `defaults.podLabels`. | | compute.executor.priorityClassName | string | `""` | Priority class. Empty inherits `defaults.priorityClassName`. | -| compute.executor.privileged | bool | `false` | Run the Docker daemon as a privileged container, which makes it root on its node. Required unless `runtimeClassName` names a runtime that runs Docker without it, such as Sysbox. Give the executor nodes of its own (`nodeSelector`, `tolerations`). | +| compute.executor.privileged | bool | `true` | Run the Docker daemon as a privileged container, which makes it root on its node. Required unless `runtimeClassName` names a runtime that runs Docker without it, such as Sysbox. For production, give the executor nodes of its own (`nodeSelector`, `tolerations`). | | compute.executor.replicaCount | int | `1` | Executor replicas. Each has its own Docker daemon: a runtime starts on the replica an execution reaches and stays there until it's idle. | | compute.executor.resources | object | `{"limits":{"memory":"4Gi"},"requests":{"cpu":"500m","memory":"1Gi"}}` | Resources of the Docker daemon container, which runs the executor and every runtime: size it for the executions you expect. | | compute.executor.runtimeClassName | string | `""` | RuntimeClass of executor pods, e.g. `sysbox-runc`. | | compute.executor.terminationGracePeriodSeconds | int | `60` | Seconds to wait on shutdown. | | compute.executor.tolerations | list | `[]` | Tolerations. Empty inherits `defaults.tolerations`. | | compute.executor.topologySpreadConstraints | list | `[]` | Topology spread constraints. Empty inherits `defaults.topologySpreadConstraints`, or spreads replicas across nodes. | -| compute.functionsRuntimes | list | `[]` | Function runtimes to enable (`_APP_FUNCTIONS_RUNTIMES`). Empty keeps the Docker installer's list (node-16.0, php-8.0, python-3.9, ruby-3.0). | +| compute.functionsRuntimes | list | `[]` | Function runtimes to enable (`_APP_FUNCTIONS_RUNTIMES`). Empty keeps the Docker installer's list (node-16.0, node-22, php-8.0, python-3.9, ruby-3.0). | | compute.orchestrator.affinity | object | `{}` | Affinity. Empty inherits `defaults.affinity`. | | compute.orchestrator.containerSecurityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true}` | Container security context. | | compute.orchestrator.image.digest | string | `""` | Image digest. | @@ -910,30 +941,6 @@ The build orchestrator and the executor. | Key | Type | Default | Description | |-----|------|---------|-------------| -| assistant.affinity | object | `{}` | Affinity. Empty inherits `defaults.affinity`. | -| assistant.apiKey.secretKey | string | `"openai-api-key"` | Key in the Secret. | -| assistant.apiKey.secretName | string | `""` | Existing Secret holding the OpenAI API key. Takes precedence over `value`. | -| assistant.apiKey.value | string | `""` | OpenAI API key (`_APP_ASSISTANT_OPENAI_API_KEY`), set inline. Stored in the chart's Secret. | -| assistant.containerSecurityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true}` | Container security context. | -| assistant.enabled | bool | `false` | Deploy the assistant. | -| assistant.extraContainers | list | `[]` | Extra sidecar containers. | -| assistant.extraEnv | list | `[]` | Extra environment variables, as a list of EnvVar objects. | -| assistant.extraInitContainers | list | `[]` | Extra init containers. | -| assistant.extraVolumeMounts | list | `[]` | Extra volume mounts for the main container. | -| assistant.extraVolumes | list | `[]` | Extra volumes. | -| assistant.image.digest | string | `""` | Image digest. | -| assistant.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | -| assistant.image.registry | string | `"docker.io"` | Image registry. | -| assistant.image.repository | string | `"appwrite/assistant"` | Image repository. | -| assistant.image.tag | string | `"0.8.4"` | Image tag. | -| assistant.nodeSelector | object | `{}` | Node selector. Empty inherits `defaults.nodeSelector`. | -| assistant.podAnnotations | object | `{}` | Pod annotations, merged over `defaults.podAnnotations`. | -| assistant.podLabels | object | `{}` | Pod labels, merged over `defaults.podLabels`. | -| assistant.podSecurityContext | object | `{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000,"seccompProfile":{"type":"RuntimeDefault"}}` | Pod security context. | -| assistant.priorityClassName | string | `""` | Priority class. Empty inherits `defaults.priorityClassName`. | -| assistant.resources | object | `{"limits":{"memory":"512Mi"},"requests":{"cpu":"20m","memory":"128Mi"}}` | Resource requests and limits. | -| assistant.terminationGracePeriodSeconds | int | `30` | Seconds to wait on shutdown. | -| assistant.tolerations | list | `[]` | Tolerations. Empty inherits `defaults.tolerations`. | | autogravity.affinity | object | `{}` | Affinity. Empty inherits `defaults.affinity`. | | autogravity.containerSecurityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true}` | Container security context. | | autogravity.enabled | bool | `false` | Deploy autogravity. | @@ -968,7 +975,7 @@ The build orchestrator and the executor. | browser.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. | | browser.image.registry | string | `"docker.io"` | Image registry. | | browser.image.repository | string | `"appwrite/browser"` | Image repository. | -| browser.image.tag | string | `"0.3.4"` | Image tag. | +| browser.image.tag | string | `"0.3.6"` | Image tag. | | browser.nodeSelector | object | `{}` | Node selector. Empty inherits `defaults.nodeSelector`. | | browser.podAnnotations | object | `{}` | Pod annotations, merged over `defaults.podAnnotations`. | | browser.podLabels | object | `{}` | Pod labels, merged over `defaults.podLabels`. | @@ -1046,9 +1053,9 @@ Prometheus Operator objects for the datastores' metrics. | Key | Type | Default | Description | |-----|------|---------|-------------| | networkPolicy.egress.dns | list | `[{"namespaceSelector":{},"podSelector":{"matchLabels":{"k8s-app":"kube-dns"}}}]` | DNS servers (NetworkPolicyPeer) allowed on port 53 wherever egress is restricted: here, for the browser, and for the executor and build Jobs, whose policies are on by default with compute. For NodeLocal DNSCache, add `{ipBlock: {cidr: 169.254.20.10/32}}`. DNS on another port (OpenShift: 5353) needs rules in `extraRules`, `compute.executor.networkPolicy.extraEgress` and `compute.builds.networkPolicy.extraEgress`. | -| networkPolicy.egress.enabled | bool | `false` | Restrict egress to DNS, pods of this release and `extraRules` (needs `networkPolicy.enabled`). Off by default: Appwrite calls webhooks, SMTP servers, OAuth providers, S3, and the DNS server it verifies custom domains against (`_APP_DNS`, 8.8.8.8). With compute, the orchestrator also needs the Kubernetes API. Allow what you use in `extraRules`; the executor is allowed already. | +| networkPolicy.egress.enabled | bool | `false` | Restrict egress to DNS, pods of this release and `extraRules` (needs `networkPolicy.enabled`). Off by default: Appwrite calls webhooks, SMTP servers, OAuth providers, S3, and the DNS server it verifies custom domains, migration sources and avatar URLs against (`_APP_DNS` and `_APP_DNS_EXTERNAL`, 8.8.8.8). With compute, the orchestrator also needs the Kubernetes API. Allow what you use in `extraRules`; the executor is allowed already. | | networkPolicy.egress.extraRules | list | `[]` | Extra egress rules (NetworkPolicyEgressRule) when egress is restricted. | -| networkPolicy.enabled | bool | `false` | Create NetworkPolicies in the release namespace. The built-in datastores then accept only Appwrite pods of this release, the internal services (geo, browser, assistant, autogravity, embedding) only pods of this release, and workers and tasks no incoming traffic. Off by default: any pod in the cluster can reach the built-in PostgreSQL, Redis and ClickHouse, which only their passwords protect. Policies only take effect with a CNI that enforces them. | +| networkPolicy.enabled | bool | `false` | Create NetworkPolicies in the release namespace. The built-in datastores then accept only Appwrite pods of this release, the internal services (geo, browser, autogravity, embedding) only pods of this release, and workers and tasks no incoming traffic. Off by default: any pod in the cluster can reach the built-in PostgreSQL, Redis and ClickHouse, which only their passwords protect. Policies only take effect with a CNI that enforces them. | | networkPolicy.ingress.from | list | `[]` | Sources (NetworkPolicyPeer) allowed to reach the API, console, realtime and MQTT, typically your ingress controller or Gateway. Empty allows any source. Pods of this release can always reach the API, console and realtime. | | networkPolicy.metrics.from | list | `[]` | Sources (NetworkPolicyPeer), typically Prometheus, allowed to scrape the datastores' metrics ports (`.builtin.metrics.enabled`). Empty allows none. | diff --git a/charts/appwrite/README.md.gotmpl b/charts/appwrite/README.md.gotmpl index 0cea721..a9f6976 100644 --- a/charts/appwrite/README.md.gotmpl +++ b/charts/appwrite/README.md.gotmpl @@ -16,9 +16,11 @@ - Ingress or Gateway API routes for the API, console, realtime, function and site previews and custom domains. - Worker topologies from one combined worker to one Deployment per queue. -- Functions and Sites in the cluster: builds run as unprivileged Jobs in their - own namespace, executions on an executor with its own Docker-in-Docker - daemon in another namespace (a privileged pod you opt into). +- Functions and Sites in the cluster, on by default: builds run as + unprivileged Jobs in their own namespace, executions on an executor with + its own Docker-in-Docker daemon in another namespace (a privileged pod; set + `compute.enabled=false` where that isn't allowed), on built-in S3-compatible + object storage. - Credentials generated once and kept, data migrations on upgrade, and guards against changing the encryption key or downgrading. @@ -28,6 +30,8 @@ - Helm 4, or Helm 3.18 or later - An ingress controller or a Gateway API implementation. Appwrite serves the console and the API on one hostname and needs `/v1` routed to the API. +- For Functions and Sites (on by default), a cluster that admits a privileged + pod in the executor's namespace, or `compute.enabled=false`. - For HTTPS: cert-manager, or certificates you provide ## Quick start @@ -60,7 +64,7 @@ kubectl -n appwrite get secret appwrite -o jsonpath='{.data._APP_OPENSSL_KEY_V1} To try Appwrite on a laptop first, or for a step-by-step install, see [Getting started](https://github.com/appwrite/helm-chart/blob/main/docs/getting-started.md). -To install a branch, or before the first release is on `ghcr.io`, use +To install a branch, use `./charts/appwrite` from a clone in place of the `oci://` reference ([installing from source](https://github.com/appwrite/helm-chart/blob/main/docs/getting-started.md#installing-from-source)). @@ -164,7 +168,7 @@ See the [architecture guide](https://github.com/appwrite/helm-chart/blob/main/do - **Highly available datastores.** The built-in ones are single replicas. Use managed services or operators in production ([guide](https://github.com/appwrite/helm-chart/blob/main/docs/external-databases.md)). -- **Metrics.** Appwrite 2.3.0 exposes no Prometheus metrics. +- **Metrics.** Appwrite 2.5.0 exposes no Prometheus metrics. ## Documentation @@ -178,7 +182,7 @@ See the [architecture guide](https://github.com/appwrite/helm-chart/blob/main/do | [Storage](https://github.com/appwrite/helm-chart/blob/main/docs/storage.md) | Volumes, ReadWriteMany, S3, antivirus | | [Email, Git and other integrations](https://github.com/appwrite/helm-chart/blob/main/docs/integrations.md) | SMTP, GitHub App, console sign-in, SMS, Sentry | | [Functions and Sites](https://github.com/appwrite/helm-chart/blob/main/docs/executor.md) | Builds as Jobs and the executor on Docker-in-Docker | -| [Optional components](https://github.com/appwrite/helm-chart/blob/main/docs/optional-components.md) | Browser, assistant, autogravity, embedding, MQTT | +| [Optional components](https://github.com/appwrite/helm-chart/blob/main/docs/optional-components.md) | Browser, autogravity, embedding, MQTT | | [Security](https://github.com/appwrite/helm-chart/blob/main/docs/security.md) | Pod security, credentials, NetworkPolicies, signatures | | [Upgrading](https://github.com/appwrite/helm-chart/blob/main/docs/upgrading.md) | Versions, migrations, rollbacks | | [Backup and restore](https://github.com/appwrite/helm-chart/blob/main/docs/backup-restore.md) | What to back up and how | @@ -220,12 +224,12 @@ prints them with their comments. {{ template "appwrite.valuesTable" (dict "rows" .Values "title" "API, console, realtime and geo" "description" "" "prefixes" (list "api" "console" "realtime" "geo")) }} {{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Workers and tasks" "description" "" "prefixes" (list "workers" "tasks")) }} {{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Functions and Sites" "description" "The build orchestrator and the executor." "prefixes" (list "compute")) }} -{{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Optional components" "description" "" "prefixes" (list "browser" "assistant" "autogravity" "embedding" "mqtt")) }} +{{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Optional components" "description" "" "prefixes" (list "browser" "autogravity" "embedding" "mqtt")) }} {{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Monitoring" "description" "Prometheus Operator objects for the datastores' metrics." "prefixes" (list "metrics")) }} {{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Network policies" "description" "" "prefixes" (list "networkPolicy")) }} {{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Migrations and tests" "description" "" "prefixes" (list "migrations" "tests")) }} {{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Global settings and defaults" "description" "Image, security contexts, scheduling defaults, names, labels and extra objects." "prefixes" (list "global" "image" "defaults" "podSecurityContext" "containerSecurityContext" "serviceAccount" "nameOverride" "fullnameOverride" "commonLabels" "commonAnnotations" "extraObjects")) }} -{{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Other values" "description" "" "prefixes" (list "appwrite" "ingress" "httpRoute" "redis" "clickhouse" "database" "documentsDb" "vectorsDb" "storage" "smtp" "api" "console" "realtime" "geo" "workers" "tasks" "compute" "browser" "assistant" "autogravity" "embedding" "mqtt" "metrics" "networkPolicy" "migrations" "tests" "global" "image" "defaults" "podSecurityContext" "containerSecurityContext" "serviceAccount" "nameOverride" "fullnameOverride" "commonLabels" "commonAnnotations" "extraObjects") "exclude" true) }} +{{ template "appwrite.valuesTable" (dict "rows" .Values "title" "Other values" "description" "" "prefixes" (list "appwrite" "ingress" "httpRoute" "redis" "clickhouse" "database" "documentsDb" "vectorsDb" "storage" "smtp" "api" "console" "realtime" "geo" "workers" "tasks" "compute" "browser" "autogravity" "embedding" "mqtt" "metrics" "networkPolicy" "migrations" "tests" "global" "image" "defaults" "podSecurityContext" "containerSecurityContext" "serviceAccount" "nameOverride" "fullnameOverride" "commonLabels" "commonAnnotations" "extraObjects") "exclude" true) }} ## License diff --git a/charts/appwrite/ci/executor-values.yaml b/charts/appwrite/ci/executor-values.yaml index a4e3b2f..97c90ab 100644 --- a/charts/appwrite/ci/executor-values.yaml +++ b/charts/appwrite/ci/executor-values.yaml @@ -7,7 +7,9 @@ appwrite: storage: device: s3 s3: - endpoint: http://10.0.0.30:8333 + mode: external + external: + endpoint: http://10.0.0.30:8333 bucket: appwrite accessKeyId: value: appwrite diff --git a/charts/appwrite/ci/full-values.yaml b/charts/appwrite/ci/full-values.yaml index 3a664ee..8945435 100644 --- a/charts/appwrite/ci/full-values.yaml +++ b/charts/appwrite/ci/full-values.yaml @@ -1,10 +1,10 @@ -# Every optional component but embedding, used by the nightly workflow together -# with the compute values hack/compute-plane.sh writes (S3 next to the cluster, -# Functions and Sites). Also linted and rendered on its own like the other +# Every optional component but embedding, with Functions and Sites on the +# built-in object storage: the nightly workflow installs it and builds and +# executes a function. Also linted and rendered on its own like the other # scenarios. # -# CPU requests are cut down so everything fits on a single kind node on a -# 2-vCPU GitHub-hosted runner, next to the control plane and Traefik. +# CPU requests are cut down so everything fits on a single kind node, next to +# the control plane and Traefik. appwrite: domain: appwrite.ci.local certificatesEmail: ci@example.com @@ -50,14 +50,6 @@ browser: resources: requests: cpu: 50m -assistant: - enabled: true - # Not a real key: the assistant only needs one to start. - apiKey: - value: sk-ci-placeholder - resources: - requests: - cpu: 10m autogravity: enabled: true resources: @@ -91,6 +83,15 @@ compute: # Builds get an 8-CPU limit; the default overcommit requests 1 CPU, # which doesn't fit on the runner next to everything else. cpuOvercommit: 40 + functionsRuntimes: [node-22] + sitesRuntimes: [static-1, node-22] + executor: + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + memory: 3Gi migrations: mode: job networkPolicy: diff --git a/charts/appwrite/ci/s3-values.yaml b/charts/appwrite/ci/s3-values.yaml index 720a3ba..09eb83a 100644 --- a/charts/appwrite/ci/s3-values.yaml +++ b/charts/appwrite/ci/s3-values.yaml @@ -8,8 +8,10 @@ appwrite: storage: device: s3 s3: - endpoint: https://s3.example.com - region: eu-central + mode: external + external: + endpoint: https://s3.example.com + region: eu-central bucket: appwrite accessKeyId: secretName: appwrite-s3 diff --git a/charts/appwrite/files/clickhouse/init.sh b/charts/appwrite/files/clickhouse/init.sh new file mode 100644 index 0000000..705ab59 --- /dev/null +++ b/charts/appwrite/files/clickhouse/init.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# The ClickHouse entrypoint sources this file, so a subshell keeps these options out of it. +( + set -euo pipefail + + client() { + clickhouse-client --host 127.0.0.1 --user "$CLICKHOUSE_USER" --password "$CLICKHOUSE_PASSWORD" "$@" + } + + # Flushing replaces log tables whose definition changed, keeping each old table as _. + client --query 'SYSTEM FLUSH LOGS' + + # Tables left behind never expire on their own: take the TTL of the log that + # replaced this one, or a week when the log is no longer written at all. + # The named logs must match the ones system-log-ttl.xml removes: a log that + # is still written gets replaced on the next restart if its TTL is altered here. + client --format TSVRaw <<'SQL' | client --multiquery +SELECT concat('ALTER TABLE system.', stale.name, ' MODIFY TTL ', if(active.name = '', 'event_date + INTERVAL 7 DAY DELETE', splitByString(' SETTINGS ', splitByString(' TTL ', active.engine_full)[2])[1]), ';') +FROM ( + SELECT name, splitByChar('_', name)[-1] AS suffix, arrayStringConcat(arrayPopBack(splitByChar('_', name)), '_') AS log + FROM system.tables + WHERE database = 'system' + AND position(engine_full, ' TTL ') = 0 + AND endsWith(engine, 'MergeTree') + AND name IN (SELECT table FROM system.columns WHERE database = 'system' AND name = 'event_date' AND type = 'Date') +) AS stale +LEFT JOIN system.tables AS active + ON active.database = 'system' + AND active.name = stale.log + AND position(active.engine_full, ' TTL ') > 0 +WHERE (endsWith(stale.log, '_log') AND toUInt64OrNull(stale.suffix) IS NOT NULL) + OR stale.name IN ('asynchronous_metric_log', 'metric_log', 'histogram_metric_log', 'query_metric_log', 'processors_profile_log', 'trace_log', 'text_log') +SQL +) diff --git a/charts/appwrite/files/clickhouse/system-log-ttl.xml b/charts/appwrite/files/clickhouse/system-log-ttl.xml index 113578b..5ad6e1b 100644 --- a/charts/appwrite/files/clickhouse/system-log-ttl.xml +++ b/charts/appwrite/files/clickhouse/system-log-ttl.xml @@ -7,38 +7,29 @@ whether or not the instance serves traffic. Left alone they outgrow the usage data they sit next to and fill the volume. - ClickHouse applies a window when it creates the table and alters the table - on the next restart when the window here changes, so an install that has - been running without one shrinks on upgrade. + When a window here changes, ClickHouse replaces the table on the next + restart and keeps the old one, rows and all, as _. + files/clickhouse/init.sh gives those old tables the same window + (Appwrite 2.4.0's clickhouse-config.xml and clickhouse-init.sh). --> - - event_date + INTERVAL 3 DAY DELETE - event_date + INTERVAL 3 DAY DELETE - event_date + INTERVAL 3 DAY DELETE - - - - event_date + INTERVAL 7 DAY DELETE - 10000 - - - event_date + INTERVAL 7 DAY DELETE - 10000 - - - event_date + INTERVAL 7 DAY DELETE - 10000 - + + + + + + + + + - - event_date + INTERVAL 7 DAY DELETE + + event_date + INTERVAL 3 DAY DELETE event_date + INTERVAL 7 DAY DELETE event_date + INTERVAL 7 DAY DELETE event_date + INTERVAL 7 DAY DELETE - event_date + INTERVAL 7 DAY DELETE event_date + INTERVAL 7 DAY DELETE event_date + INTERVAL 7 DAY DELETE diff --git a/charts/appwrite/files/env-defaults.yaml b/charts/appwrite/files/env-defaults.yaml index 57d5cf8..7e12f1e 100644 --- a/charts/appwrite/files/env-defaults.yaml +++ b/charts/appwrite/files/env-defaults.yaml @@ -1,4 +1,4 @@ -# Generated by hack/gen-env-defaults.php from Appwrite 2.3.0 +# Generated by hack/gen-env-defaults.php from Appwrite 2.5.0 # app/config/variables.php. Do not edit by hand; run `make env-defaults`. # # These are the defaults the Appwrite Docker installer writes to .env. @@ -25,6 +25,8 @@ _APP_DOMAIN_TARGET_AAAA: "::1" _APP_DOMAIN_TARGET_A: "127.0.0.1" _APP_DOMAIN_TARGET_CAA: "" _APP_DNS: "8.8.8.8" +_APP_DNS_EXTERNAL: "" +_APP_ALLOWED_INTERNAL_ADDRESSES: "" _APP_CONSOLE_WHITELIST_ROOT: "enabled" _APP_CONSOLE_WHITELIST_EMAILS: "" _APP_CONSOLE_WHITELIST_IPS: "" @@ -47,6 +49,7 @@ _APP_CONSOLE_SESSION_ALERTS: "disabled" _APP_COMPRESSION_ENABLED: "enabled" _APP_COMPRESSION_MIN_SIZE_BYTES: "1024" _APP_TRUSTED_HEADERS: "x-forwarded-for" +_APP_TRUSTED_PROXIES: "127.0.0.1,::1,10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16" _APP_GEO_ENDPOINT: "http://appwrite-geo/v1" # Database @@ -107,7 +110,7 @@ _APP_FUNCTIONS_TIMEOUT: "900" _APP_COMPUTE_BUILD_TIMEOUT: "2700" _APP_COMPUTE_CPUS: "0" _APP_COMPUTE_MEMORY: "0" -_APP_FUNCTIONS_RUNTIMES: "node-16.0,php-8.0,python-3.9,ruby-3.0" +_APP_FUNCTIONS_RUNTIMES: "node-16.0,node-22,php-8.0,python-3.9,ruby-3.0" _APP_EXECUTOR_HOST: "http://exc1/v1" _APP_BUILDS_VOLUME: "appwrite-builds" _APP_JOBS_HOST: "http://orchestrator:8080" diff --git a/charts/appwrite/templates/NOTES.txt b/charts/appwrite/templates/NOTES.txt index 4000684..156aaf4 100644 --- a/charts/appwrite/templates/NOTES.txt +++ b/charts/appwrite/templates/NOTES.txt @@ -51,6 +51,7 @@ First steps {{- if (eq .Values.database.mode "builtin") }}{{ $builtin = append $builtin "PostgreSQL" }}{{ end }} {{- if (eq .Values.redis.mode "builtin") }}{{ $builtin = append $builtin "Redis" }}{{ end }} {{- if (eq .Values.clickhouse.mode "builtin") }}{{ $builtin = append $builtin "ClickHouse" }}{{ end }} +{{- if include "appwrite.s3.builtin" . }}{{ $builtin = append $builtin "SeaweedFS (object storage)" }}{{ end }} {{- with $builtin }} The built-in {{ if eq (len .) 1 }}{{ first . }} is a single replica{{ else }}{{ join ", " (initial .) }} and {{ last . }} are single replicas{{ end }}, fine for trying Appwrite. @@ -62,11 +63,13 @@ Before real users depend on this install, move to replicated services: {{- if .Values.compute.enabled }} Functions and Sites build as Jobs in namespace "{{ include "appwrite.builds.namespace" . }}" and execute in -namespace "{{ include "appwrite.executor.namespace" . }}", on Docker-in-Docker{{ if .Values.compute.executor.privileged }} in a privileged container: -keep the executor on nodes of its own{{ end }}. +namespace "{{ include "appwrite.executor.namespace" . }}", on Docker-in-Docker{{ if .Values.compute.executor.privileged }} in a privileged container. +For production, keep the executor on nodes of its own. Where privileged pods +aren't allowed, set compute.enabled=false{{ end }}: {{ $docs }}/executor.md {{- else }} -Functions and Sites are off. To turn them on: {{ $docs }}/executor.md +Functions and Sites are off (compute.enabled=false): creating a function or +site fails until they're on. To turn them on: {{ $docs }}/executor.md {{- end }} {{- if not .Values.smtp.host }} diff --git a/charts/appwrite/templates/_datastore.tpl b/charts/appwrite/templates/_datastore.tpl index d7d8a34..254aa6b 100644 --- a/charts/appwrite/templates/_datastore.tpl +++ b/charts/appwrite/templates/_datastore.tpl @@ -10,6 +10,7 @@ Usage: include "appwrite.datastore" (dict "env" "" "ports" (list ...) "probes" "" + "lifecycle" "" optional (postStart, preStop) "dataPath" "/var/lib/postgresql" "volumeMounts" "" extra mounts "volumes" "" extra volumes @@ -91,6 +92,10 @@ spec: {{- with .probes }} {{- . | trim | nindent 10 }} {{- end }} + {{- with .lifecycle }} + lifecycle: + {{- . | trim | nindent 12 }} + {{- end }} {{- with $v.containerSecurityContext }} securityContext: {{- toYaml . | nindent 12 }} diff --git a/charts/appwrite/templates/_datastores.tpl b/charts/appwrite/templates/_datastores.tpl index ca573ca..4dadbf1 100644 --- a/charts/appwrite/templates/_datastores.tpl +++ b/charts/appwrite/templates/_datastores.tpl @@ -94,3 +94,28 @@ secret: {{- . | nindent 2 }} {{- end }} {{- end }} + +{{/* +S3 settings for `storage.device: s3`, built-in or external, as YAML: builtin, +endpoint (a URL; empty means AWS S3), region and bucket. +*/}} +{{- define "appwrite.s3" -}} +{{- $s := .Values.storage.s3 }} +{{- if eq $s.mode "builtin" }} +builtin: true +endpoint: {{ printf "http://%s.%s.svc:8333" (include "appwrite.componentName" (dict "root" . "component" "seaweedfs")) .Release.Namespace | quote }} +region: us-east-1 +{{- else }} +builtin: false +endpoint: {{ $s.external.endpoint | quote }} +region: {{ $s.external.region | quote }} +{{- end }} +bucket: {{ $s.bucket | quote }} +{{- end }} + +{{/* +"true" when the release runs the built-in SeaweedFS, else empty. +*/}} +{{- define "appwrite.s3.builtin" -}} +{{- if and (eq .Values.storage.device "s3") (eq .Values.storage.s3.mode "builtin") }}true{{ end }} +{{- end }} diff --git a/charts/appwrite/templates/_deprecations.tpl b/charts/appwrite/templates/_deprecations.tpl index 0d3166b..3df314b 100644 --- a/charts/appwrite/templates/_deprecations.tpl +++ b/charts/appwrite/templates/_deprecations.tpl @@ -6,7 +6,9 @@ permissive stub for the old key in values.schema.json, for at least one minor release (CONTRIBUTING.md, "Deprecating a value"). */}} {{- define "appwrite.deprecations.renamed" -}} -{{- dict | toYaml }} +assistant: Appwrite removed the assistant in 2.4.0. Delete the assistant block from your values. +storage.s3.endpoint: "moved to storage.s3.external.endpoint, with storage.s3.mode: external (the default, builtin, runs SeaweedFS in the release)." +storage.s3.region: "moved to storage.s3.external.region, with storage.s3.mode: external." {{- end }} {{/* diff --git a/charts/appwrite/templates/_env.tpl b/charts/appwrite/templates/_env.tpl index a81d0f4..68af4aa 100644 --- a/charts/appwrite/templates/_env.tpl +++ b/charts/appwrite/templates/_env.tpl @@ -98,9 +98,10 @@ on each container (see appwrite.env). {{- $_ := set $computed "_APP_STORAGE_LIMIT" $storage.limit }} {{- $_ := set $computed "_APP_STORAGE_PREVIEW_LIMIT" $storage.previewLimit }} {{- if eq $storage.device "s3" }} -{{- $_ := set $computed "_APP_STORAGE_S3_ENDPOINT" $storage.s3.endpoint }} -{{- $_ := set $computed "_APP_STORAGE_S3_REGION" $storage.s3.region }} -{{- $_ := set $computed "_APP_STORAGE_S3_BUCKET" $storage.s3.bucket }} +{{- $s3 := include "appwrite.s3" . | fromYaml }} +{{- $_ := set $computed "_APP_STORAGE_S3_ENDPOINT" $s3.endpoint }} +{{- $_ := set $computed "_APP_STORAGE_S3_REGION" $s3.region }} +{{- $_ := set $computed "_APP_STORAGE_S3_BUCKET" $s3.bucket }} {{- end }} {{- /* Compute plane (Functions and Sites): the orchestrator runs builds as Jobs, the executor runs executions; build Jobs call back to the API Service */}} @@ -232,14 +233,6 @@ Usage: include "appwrite.env" (dict "root" $ "cpuNum" "1" "extra" (dict "_APP_WO valueFrom: {{- include "appwrite.secretRef" (dict "root" $root "existingSecret" $cs.existingSecret "key" $cs.orchestratorSecretKey "defaultKey" "orchestrator-secret") | nindent 4 }} {{- end }} -{{- if $v.assistant.enabled }} -{{- /* The console shows the assistant when Appwrite has the key. */}} -{{- with include "appwrite.credential.valueFrom" (dict "root" $root "ref" $v.assistant.apiKey "chartKey" "openai-api-key") | fromYaml }} -- name: _APP_ASSISTANT_OPENAI_API_KEY - valueFrom: - {{- toYaml . | nindent 4 }} -{{- end }} -{{- end }} {{- with include "appwrite.credential.valueFrom" (dict "root" $root "ref" $v.smtp.password "chartKey" "smtp-password") | fromYaml }} - name: _APP_SMTP_PASSWORD valueFrom: diff --git a/charts/appwrite/templates/_secrets.tpl b/charts/appwrite/templates/_secrets.tpl index 2725488..421e862 100644 --- a/charts/appwrite/templates/_secrets.tpl +++ b/charts/appwrite/templates/_secrets.tpl @@ -28,16 +28,16 @@ One with a secretName lives in that Secret; the chart stores the others. */}} {{- $v := .Values }} {{- $s3 := eq $v.storage.device "s3" }} +{{- $s3Builtin := and $s3 (eq $v.storage.s3.mode "builtin") }} {{- $credentials := list (list $v.database.password "db-password" true (eq $v.database.mode "builtin")) (list $v.redis.password "redis-password" true (and (eq $v.redis.mode "builtin") $v.redis.builtin.requirePassword)) (list $v.clickhouse.password "clickhouse-password" true (eq $v.clickhouse.mode "builtin")) (list $v.documentsDb.password "documentsdb-password" $v.documentsDb.enabled false) (list $v.vectorsDb.password "vectorsdb-password" $v.vectorsDb.enabled false) - (list $v.storage.s3.accessKeyId "s3-access-key" $s3 false) - (list $v.storage.s3.secretAccessKey "s3-secret-key" $s3 false) - (list $v.smtp.password "smtp-password" true false) - (list $v.assistant.apiKey "openai-api-key" $v.assistant.enabled false) }} + (list $v.storage.s3.accessKeyId "s3-access-key" $s3 $s3Builtin) + (list $v.storage.s3.secretAccessKey "s3-secret-key" $s3 $s3Builtin) + (list $v.smtp.password "smtp-password" true false) }} {{- range $credentials }} {{- $ref := index . 0 }} {{- $key := index . 1 }} diff --git a/charts/appwrite/templates/_validate.tpl b/charts/appwrite/templates/_validate.tpl index 9ed91f8..36e470f 100644 --- a/charts/appwrite/templates/_validate.tpl +++ b/charts/appwrite/templates/_validate.tpl @@ -12,7 +12,7 @@ configmap-env.yaml so it also runs under `helm template`. {{- $errors = append $errors "appwrite.certificatesEmail is required: the certificates worker refuses to start without it." }} {{- end }} {{- $hasValue := dict }} -{{- range $name, $ref := dict "database.password" $v.database.password "redis.password" $v.redis.password "clickhouse.password" $v.clickhouse.password "documentsDb.password" $v.documentsDb.password "vectorsDb.password" $v.vectorsDb.password "smtp.password" $v.smtp.password "storage.s3.accessKeyId" $v.storage.s3.accessKeyId "storage.s3.secretAccessKey" $v.storage.s3.secretAccessKey "assistant.apiKey" $v.assistant.apiKey }} +{{- range $name, $ref := dict "database.password" $v.database.password "redis.password" $v.redis.password "clickhouse.password" $v.clickhouse.password "documentsDb.password" $v.documentsDb.password "vectorsDb.password" $v.vectorsDb.password "smtp.password" $v.smtp.password "storage.s3.accessKeyId" $v.storage.s3.accessKeyId "storage.s3.secretAccessKey" $v.storage.s3.secretAccessKey }} {{- $_ := set $hasValue $name (or $ref.value $ref.secretName) }} {{- if and $ref.secretName (not $ref.secretKey) }} {{- $errors = append $errors (printf "%s.secretKey is required with %s.secretName: name the key in the Secret." $name $name) }} @@ -99,8 +99,8 @@ configmap-env.yaml so it also runs under `helm template`. {{- if not $storage.s3.bucket }} {{- $errors = append $errors "storage.s3.bucket is required when storage.device is s3." }} {{- end }} -{{- if not (and (get $hasValue "storage.s3.accessKeyId") (get $hasValue "storage.s3.secretAccessKey")) }} -{{- $errors = append $errors "storage.s3.accessKeyId and storage.s3.secretAccessKey need a value or a secretName when storage.device is s3." }} +{{- if and (eq $storage.s3.mode "external") (not (and (get $hasValue "storage.s3.accessKeyId") (get $hasValue "storage.s3.secretAccessKey"))) }} +{{- $errors = append $errors "storage.s3.accessKeyId and storage.s3.secretAccessKey need a value or a secretName with storage.s3.mode external." }} {{- end }} {{- end }} {{- if $v.compute.enabled }} @@ -122,12 +122,9 @@ configmap-env.yaml so it also runs under `helm template`. {{- $errors = append $errors "compute.executor.existingSecret is required when compute.secrets.existingSecret or an S3 secretName (storage.s3.accessKeyId, storage.s3.secretAccessKey) is set: the executor runs in another namespace and can't read them. Create a Secret there with the keys executor-secret, s3-access-key and s3-secret-key." }} {{- end }} {{- if ne $storage.device "s3" }} -{{- $errors = append $errors "compute.enabled needs storage.device=s3: builds and the executor exchange code and build output through object storage." }} +{{- $errors = append $errors "compute.enabled needs storage.device=s3: builds and the executor exchange code and build output through object storage. Keep the default (s3, with the built-in object store), or set compute.enabled=false to store files on the local volume without Functions and Sites." }} {{- end }} {{- end }} -{{- if and $v.assistant.enabled (not (get $hasValue "assistant.apiKey")) }} -{{- $errors = append $errors "assistant.enabled needs assistant.apiKey: a value or a secretName." }} -{{- end }} {{- if and $v.mqtt.enabled $v.redis.username }} {{- $errors = append $errors "mqtt.enabled doesn't work with redis.username: Appwrite's MQTT server ignores Redis ACL users. Use the default user." }} {{- end }} @@ -144,9 +141,10 @@ configmap-env.yaml so it also runs under `helm template`. {{- if and (eq $v.database.mode "builtin") $v.database.builtin.maxConnections (lt (int $v.database.builtin.maxConnections) (int $v.database.connectionsMax)) }} {{- $errors = append $errors (printf "database.builtin.maxConnections (%v) is below database.connectionsMax (%v): Appwrite would open more connections than PostgreSQL accepts." $v.database.builtin.maxConnections $v.database.connectionsMax) }} {{- end }} -{{- range $store, $service := dict "postgresql" "database" "redis" "redis" "clickhouse" "clickhouse" }} +{{- $statefulSets := dict "postgresql" (eq $v.database.mode "builtin") "redis" (eq $v.redis.mode "builtin") "clickhouse" (eq $v.clickhouse.mode "builtin") "seaweedfs" (include "appwrite.s3.builtin" . | eq "true") }} +{{- range $store, $builtin := $statefulSets }} {{- $name := include "appwrite.componentName" (dict "root" $ "component" $store) }} -{{- if and (eq (get $v $service).mode "builtin") (gt (len $name) 52) }} +{{- if and $builtin (gt (len $name) 52) }} {{- $errors = append $errors (printf "StatefulSet name %q is longer than 52 characters, which breaks its pods' controller-revision-hash label: use a shorter release name or fullnameOverride." $name) }} {{- end }} {{- end }} diff --git a/charts/appwrite/templates/assistant/deployment.yaml b/charts/appwrite/templates/assistant/deployment.yaml deleted file mode 100644 index df9512e..0000000 --- a/charts/appwrite/templates/assistant/deployment.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.assistant.enabled }} -{{- $v := .Values.assistant }} -{{- $env := list (dict "name" "_APP_ASSISTANT_OPENAI_API_KEY" "valueFrom" (include "appwrite.credential.valueFrom" (dict "root" . "ref" $v.apiKey "chartKey" "openai-api-key") | fromYaml)) }} -{{- $handler := dict "tcpSocket" (dict "port" "http") }} -{{- $probes := dict "readinessProbe" (dict "enabled" true "periodSeconds" 10 "timeoutSeconds" 5) "livenessProbe" (dict "enabled" true "periodSeconds" 20 "timeoutSeconds" 5 "failureThreshold" 3) }} -{{- include "appwrite.deployment" (dict - "root" . - "component" "assistant" - "values" $v - "image" (include "appwrite.image" (dict "root" . "image" $v.image)) - "pullPolicy" $v.image.pullPolicy - "env" (toYaml $env) - "ports" (list (dict "name" "http" "containerPort" 3003 "protocol" "TCP")) - "probes" (include "appwrite.probes" (dict "values" $probes "handler" $handler)) - "replicas" 1 - "podAnnotations" (dict "checksum/secret" (include "appwrite.secret.checksum" (dict "root" . "keys" (list "openai-api-key")))) - "volumeMounts" (toYaml (list (dict "name" "tmp" "mountPath" "/tmp"))) - "volumes" (toYaml (list (dict "name" "tmp" "emptyDir" dict))) -) }} -{{- end }} diff --git a/charts/appwrite/templates/assistant/service.yaml b/charts/appwrite/templates/assistant/service.yaml deleted file mode 100644 index e85b064..0000000 --- a/charts/appwrite/templates/assistant/service.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if .Values.assistant.enabled }} -{{- /* Appwrite calls the assistant at the hard-coded http://appwrite-assistant:3003. */}} -apiVersion: v1 -kind: Service -metadata: - name: appwrite-assistant - namespace: {{ .Release.Namespace }} - labels: - {{- include "appwrite.componentLabels" (dict "root" . "component" "assistant") | nindent 4 }} - {{- with include "appwrite.annotations" (dict "root" .) | trim }}{{ . | nindent 2 }}{{ end }} -spec: - type: ClusterIP - ports: - - name: http - port: 3003 - targetPort: http - protocol: TCP - appProtocol: http - selector: - {{- include "appwrite.componentSelectorLabels" (dict "root" . "component" "assistant") | nindent 4 }} -{{- end }} diff --git a/charts/appwrite/templates/browser/deployment.yaml b/charts/appwrite/templates/browser/deployment.yaml index 509cb8d..9f21a86 100644 --- a/charts/appwrite/templates/browser/deployment.yaml +++ b/charts/appwrite/templates/browser/deployment.yaml @@ -2,13 +2,16 @@ {{- $v := .Values.browser }} {{- $handler := dict "tcpSocket" (dict "port" "http") }} {{- $probes := dict "startupProbe" (dict "enabled" true "periodSeconds" 5 "failureThreshold" 30 "timeoutSeconds" 5) "readinessProbe" (dict "enabled" true "periodSeconds" 10 "timeoutSeconds" 5) "livenessProbe" (dict "enabled" true "periodSeconds" 20 "timeoutSeconds" 5 "failureThreshold" 3) }} +{{- /* Browser 0.3.5 and later refuse hosts that resolve to private addresses +unless ALLOWED_HOSTS lists them. Screenshots of sites go through the API +Service (_APP_WORKER_SCREENSHOTS_ROUTER), which is one. */}} {{- include "appwrite.deployment" (dict "root" . "component" "browser" "values" $v "image" (include "appwrite.image" (dict "root" . "image" $v.image)) "pullPolicy" $v.image.pullPolicy - "env" (toYaml (list (dict "name" "PORT" "value" "3000") (dict "name" "HOME" "value" "/tmp"))) + "env" (toYaml (list (dict "name" "PORT" "value" "3000") (dict "name" "HOME" "value" "/tmp") (dict "name" "ALLOWED_HOSTS" "value" (include "appwrite.api.internalHost" .)))) "ports" (list (dict "name" "http" "containerPort" 3000 "protocol" "TCP")) "probes" (include "appwrite.probes" (dict "values" $probes "handler" $handler)) "replicas" $v.replicaCount diff --git a/charts/appwrite/templates/builds/networkpolicy.yaml b/charts/appwrite/templates/builds/networkpolicy.yaml index 55d2262..c2c043a 100644 --- a/charts/appwrite/templates/builds/networkpolicy.yaml +++ b/charts/appwrite/templates/builds/networkpolicy.yaml @@ -1,5 +1,6 @@ {{- if and .Values.compute.enabled .Values.compute.builds.networkPolicy.enabled }} -{{- $s3 := urlParse (.Values.storage.s3.endpoint | default "https://s3.amazonaws.com") }} +{{- $conn := include "appwrite.s3" . | fromYaml }} +{{- $s3 := urlParse ($conn.endpoint | default "https://s3.amazonaws.com") }} {{- $s3Host := regexReplaceAll ":[0-9]+$" $s3.host "" }} {{- $s3Port := regexFind "[0-9]+$" (regexFind ":[0-9]+$" $s3.host) | default (ternary "443" "80" (eq $s3.scheme "https")) }} {{- /* @@ -37,7 +38,17 @@ spec: ports: - port: 8080 protocol: TCP - {{- if regexMatch "^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$" $s3Host }} + {{- if $conn.builtin }} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + {{- include "appwrite.networkPolicy.selector" (dict "root" . "components" (list "seaweedfs")) | nindent 12 }} + ports: + - port: 8333 + protocol: TCP + {{- else if regexMatch "^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$" $s3Host }} - to: - ipBlock: cidr: {{ $s3Host }}/32 diff --git a/charts/appwrite/templates/clickhouse/_config.tpl b/charts/appwrite/templates/clickhouse/_config.tpl index 22f5a28..8bed6c9 100644 --- a/charts/appwrite/templates/clickhouse/_config.tpl +++ b/charts/appwrite/templates/clickhouse/_config.tpl @@ -1,6 +1,8 @@ {{- define "appwrite.clickhouse.config" -}} system-log-ttl.xml: | {{- .Files.Get "files/clickhouse/system-log-ttl.xml" | nindent 2 }} +init.sh: | + {{- .Files.Get "files/clickhouse/init.sh" | nindent 2 }} logging.xml: | {{- .Files.Get "files/clickhouse/logging.xml" | nindent 2 }} {{- if .Values.clickhouse.builtin.metrics.enabled }} diff --git a/charts/appwrite/templates/clickhouse/statefulset.yaml b/charts/appwrite/templates/clickhouse/statefulset.yaml index c01f624..b35f534 100644 --- a/charts/appwrite/templates/clickhouse/statefulset.yaml +++ b/charts/appwrite/templates/clickhouse/statefulset.yaml @@ -5,10 +5,12 @@ (dict "name" "CLICKHOUSE_DB" "value" $c.database) (dict "name" "CLICKHOUSE_USER" "value" $c.username) (dict "name" "CLICKHOUSE_PASSWORD" "valueFrom" (include "appwrite.credential.valueFrom" (dict "root" . "ref" $c.password "chartKey" "clickhouse-password") | fromYaml)) - (dict "name" "CLICKHOUSE_SKIP_USER_SETUP" "value" "1") }} + (dict "name" "CLICKHOUSE_SKIP_USER_SETUP" "value" "1") + (dict "name" "CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS" "value" "1") }} {{- $handler := dict "httpGet" (dict "path" "/ping" "port" "http") }} {{- $mounts := list (dict "name" "config" "mountPath" "/etc/clickhouse-server/config.d/appwrite-system-log-ttl.xml" "subPath" "system-log-ttl.xml") + (dict "name" "config" "mountPath" "/docker-entrypoint-initdb.d/appwrite-init.sh" "subPath" "init.sh") (dict "name" "config" "mountPath" "/etc/clickhouse-server/config.d/appwrite-logging.xml" "subPath" "logging.xml") (dict "name" "config" "mountPath" "/etc/clickhouse-server/users.d/appwrite.xml" "subPath" "users.xml") }} {{- $ports := list (dict "name" "http" "containerPort" 8123 "protocol" "TCP") (dict "name" "native" "containerPort" 9000 "protocol" "TCP") }} diff --git a/charts/appwrite/templates/executor/deployment.yaml b/charts/appwrite/templates/executor/deployment.yaml index fff2a84..53533ec 100644 --- a/charts/appwrite/templates/executor/deployment.yaml +++ b/charts/appwrite/templates/executor/deployment.yaml @@ -4,10 +4,11 @@ {{- $name := include "appwrite.componentName" (dict "root" . "component" "executor") }} {{- $secret := $e.existingSecret | default $name }} {{- /* Where the executor finds build output. With an endpoint, Appwrite's paths start with the bucket; on AWS the bucket is in the hostname (docs/executor.md). */}} -{{- $location := printf "%s.s3.%s.amazonaws.com?region=%s" $s3.bucket $s3.region $s3.region }} -{{- with $s3.endpoint }} +{{- $conn := include "appwrite.s3" . | fromYaml }} +{{- $location := printf "%s.s3.%s.amazonaws.com?region=%s" $conn.bucket $conn.region $conn.region }} +{{- with $conn.endpoint }} {{- $u := urlParse . }} -{{- $location = printf "%s?region=%s%s" $u.host $s3.region (ternary "&insecure=true" "" (eq $u.scheme "http")) }} +{{- $location = printf "%s?region=%s%s" $u.host $conn.region (ternary "&insecure=true" "" (eq $u.scheme "http")) }} {{- end }} {{- $images := include "appwrite.executor.images" . }} apiVersion: apps/v1 diff --git a/charts/appwrite/templates/executor/networkpolicy.yaml b/charts/appwrite/templates/executor/networkpolicy.yaml index 7d6167d..1405f25 100644 --- a/charts/appwrite/templates/executor/networkpolicy.yaml +++ b/charts/appwrite/templates/executor/networkpolicy.yaml @@ -1,6 +1,7 @@ {{- if and .Values.compute.enabled .Values.compute.executor.networkPolicy.enabled }} {{- $np := .Values.compute.executor.networkPolicy }} -{{- $s3 := urlParse (.Values.storage.s3.endpoint | default "https://s3.amazonaws.com") }} +{{- $conn := include "appwrite.s3" . | fromYaml }} +{{- $s3 := urlParse ($conn.endpoint | default "https://s3.amazonaws.com") }} {{- $s3Host := regexReplaceAll ":[0-9]+$" $s3.host "" }} {{- $s3Port := regexFind "[0-9]+$" (regexFind ":[0-9]+$" $s3.host) | default (ternary "443" "80" (eq $s3.scheme "https")) }} {{- /* @@ -40,7 +41,17 @@ spec: protocol: UDP - port: 53 protocol: TCP - {{- if regexMatch "^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$" $s3Host }} + {{- if $conn.builtin }} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + {{- include "appwrite.networkPolicy.selector" (dict "root" . "components" (list "seaweedfs")) | nindent 12 }} + ports: + - port: 8333 + protocol: TCP + {{- else if regexMatch "^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$" $s3Host }} - to: - ipBlock: cidr: {{ $s3Host }}/32 diff --git a/charts/appwrite/templates/networkpolicy/internal.yaml b/charts/appwrite/templates/networkpolicy/internal.yaml index 5627ec6..42e14e1 100644 --- a/charts/appwrite/templates/networkpolicy/internal.yaml +++ b/charts/appwrite/templates/networkpolicy/internal.yaml @@ -5,7 +5,7 @@ kind: NetworkPolicy {{ include "appwrite.networkPolicy.metadata" (dict "root" . "name" "internal") }} spec: podSelector: - {{- include "appwrite.networkPolicy.selector" (dict "root" . "components" (list "geo" "browser" "assistant" "autogravity" "embedding")) | nindent 4 }} + {{- include "appwrite.networkPolicy.selector" (dict "root" . "components" (list "geo" "browser" "autogravity" "embedding")) | nindent 4 }} policyTypes: [Ingress] ingress: - from: diff --git a/charts/appwrite/templates/networkpolicy/object-storage.yaml b/charts/appwrite/templates/networkpolicy/object-storage.yaml new file mode 100644 index 0000000..5c27c2e --- /dev/null +++ b/charts/appwrite/templates/networkpolicy/object-storage.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.networkPolicy.enabled (include "appwrite.s3.builtin" .) }} +{{- /* +The built-in object store: Appwrite pods and the orchestrator of this release, +build pods and the executor may connect to its S3 port. +*/}} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +{{ include "appwrite.networkPolicy.metadata" (dict "root" . "name" "object-storage") }} +spec: + podSelector: + {{- include "appwrite.networkPolicy.selector" (dict "root" . "components" (list "seaweedfs")) | nindent 4 }} + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + {{- include "appwrite.selectorLabels" . | nindent 14 }} + appwrite/datastore-client: "true" + - podSelector: + {{- include "appwrite.networkPolicy.selector" (dict "root" . "components" (list "orchestrator")) | nindent 12 }} + {{- if .Values.compute.enabled }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "appwrite.builds.namespace" . }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ include "appwrite.executor.namespace" . }} + {{- end }} + ports: + - port: s3 + protocol: TCP +{{- end }} diff --git a/charts/appwrite/templates/orchestrator/deployment.yaml b/charts/appwrite/templates/orchestrator/deployment.yaml index a0133f5..d8339d8 100644 --- a/charts/appwrite/templates/orchestrator/deployment.yaml +++ b/charts/appwrite/templates/orchestrator/deployment.yaml @@ -3,6 +3,7 @@ {{- $b := .Values.compute.builds }} {{- $cs := .Values.compute.secrets }} {{- $s3 := .Values.storage.s3 }} +{{- $conn := include "appwrite.s3" . | fromYaml }} {{- $name := include "appwrite.componentName" (dict "root" . "component" "orchestrator") }} {{- $env := list (dict "name" "ORCHESTRATOR_BACKEND" "value" "kubernetes") @@ -16,10 +17,10 @@ (dict "name" "KUBE_MEMORY_OVERCOMMIT" "value" (include "appwrite.envValue" $b.memoryOvercommit)) (dict "name" "SHUTDOWN_DRAIN_WAIT" "value" "0s") (dict "name" "API_KEY_FILE" "value" "/var/run/secrets/orchestrator/api-key") - (dict "name" "S3_REGION" "value" $s3.region) + (dict "name" "S3_REGION" "value" $conn.region) (dict "name" "S3_ACCESS_KEY_ID" "valueFrom" (include "appwrite.credential.valueFrom" (dict "root" . "ref" $s3.accessKeyId "chartKey" "s3-access-key") | fromYaml)) (dict "name" "S3_SECRET_ACCESS_KEY" "valueFrom" (include "appwrite.credential.valueFrom" (dict "root" . "ref" $s3.secretAccessKey "chartKey" "s3-secret-key") | fromYaml)) }} -{{- with $s3.endpoint }} +{{- with $conn.endpoint }} {{- $env = append $env (dict "name" "S3_ENDPOINT" "value" .) }} {{- end }} {{- with $b.nodeSelector }} diff --git a/charts/appwrite/templates/seaweedfs/service-headless.yaml b/charts/appwrite/templates/seaweedfs/service-headless.yaml new file mode 100644 index 0000000..7a58e3c --- /dev/null +++ b/charts/appwrite/templates/seaweedfs/service-headless.yaml @@ -0,0 +1,3 @@ +{{- if include "appwrite.s3.builtin" . }} +{{- include "appwrite.service" (dict "root" . "component" "seaweedfs" "headless" true "service" (dict) "ports" (list (dict "name" "s3" "port" 8333 "targetPort" "s3"))) }} +{{- end }} diff --git a/charts/appwrite/templates/seaweedfs/service.yaml b/charts/appwrite/templates/seaweedfs/service.yaml new file mode 100644 index 0000000..2932be6 --- /dev/null +++ b/charts/appwrite/templates/seaweedfs/service.yaml @@ -0,0 +1,3 @@ +{{- if include "appwrite.s3.builtin" . }} +{{- include "appwrite.service" (dict "root" . "component" "seaweedfs" "service" (dict) "ports" (list (dict "name" "s3" "port" 8333 "targetPort" "s3" "appProtocol" "http"))) }} +{{- end }} diff --git a/charts/appwrite/templates/seaweedfs/statefulset.yaml b/charts/appwrite/templates/seaweedfs/statefulset.yaml new file mode 100644 index 0000000..fdb6bd1 --- /dev/null +++ b/charts/appwrite/templates/seaweedfs/statefulset.yaml @@ -0,0 +1,40 @@ +{{- if include "appwrite.s3.builtin" . }} +{{- $s := .Values.storage.s3 }} +{{- $v := $s.builtin }} +{{- $env := list + (dict "name" "S3_ACCESS_KEY" "valueFrom" (include "appwrite.credential.valueFrom" (dict "root" . "ref" $s.accessKeyId "chartKey" "s3-access-key") | fromYaml)) + (dict "name" "S3_SECRET_KEY" "valueFrom" (include "appwrite.credential.valueFrom" (dict "root" . "ref" $s.secretAccessKey "chartKey" "s3-secret-key") | fromYaml)) + (dict "name" "S3_BUCKET" "value" $s.bucket) + (dict "name" "HOME" "value" "/tmp") }} +{{- /* +The S3 gateway reads its users from a file, written from the Secret on every +start. Master, volume server and filer talk over loopback inside the pod; only +the S3 port is served. +*/}} +{{- $script := printf `set -eu +esc() { printf '%%s' "$1" | sed 's/[\\"]/\\&/g'; } +printf '{"identities":[{"name":"appwrite","credentials":[{"accessKey":"%%s","secretKey":"%%s"}],"actions":["Admin","Read","Write","List","Tagging"]}]}' \ + "$(esc "$S3_ACCESS_KEY")" "$(esc "$S3_SECRET_KEY")" > /tmp/s3.json +exec weed server -dir=/data -ip=127.0.0.1 -ip.bind=0.0.0.0 -s3 -s3.config=/tmp/s3.json -master.volumeSizeLimitMB=%d -volume.max=0` (int $v.volumeSizeLimitMB) }} +{{- /* Creates the bucket before the pod reports ready; "already exists" is fine. */}} +{{- $postStart := `for i in $(seq 1 60); do + out=$(echo "s3.bucket.create -name $S3_BUCKET" | weed shell -master=localhost:9333 2>&1) || true + case "$out" in *"created bucket"*|*"already exists"*) exit 0;; esac + sleep 2 +done +echo "$out" >&2 +exit 1` }} +{{- $handler := dict "httpGet" (dict "path" "/healthz" "port" "s3") }} +{{- include "appwrite.datastore" (dict + "root" . + "component" "seaweedfs" + "values" $v + "command" (list "sh" "-c" $script) + "env" (toYaml $env) + "ports" (list (dict "name" "s3" "containerPort" 8333 "protocol" "TCP")) + "probes" (include "appwrite.datastore.probes" (dict "handler" $handler)) + "lifecycle" (toYaml (dict "postStart" (dict "exec" (dict "command" (list "sh" "-c" $postStart))))) + "dataPath" "/data" + "secretKeys" (list "s3-access-key" "s3-secret-key") +) }} +{{- end }} diff --git a/charts/appwrite/tests/builds_test.yaml b/charts/appwrite/tests/builds_test.yaml index fab40ae..ac0f03e 100644 --- a/charts/appwrite/tests/builds_test.yaml +++ b/charts/appwrite/tests/builds_test.yaml @@ -17,7 +17,8 @@ values: - values/required.yaml set: storage.device: s3 - storage.s3.endpoint: http://10.1.2.3:9000 + storage.s3.mode: external + storage.s3.external.endpoint: http://10.1.2.3:9000 storage.s3.bucket: files storage.s3.accessKeyId.secretName: s3 storage.s3.secretAccessKey.secretName: s3 @@ -181,7 +182,8 @@ tests: - it: adds no S3 rule for a hostname endpoint template: templates/builds/networkpolicy.yaml set: - storage.s3.endpoint: https://s3.example.com + storage.s3.mode: external + storage.s3.external.endpoint: https://s3.example.com asserts: - lengthEqual: path: spec.egress diff --git a/charts/appwrite/tests/compute_test.yaml b/charts/appwrite/tests/compute_test.yaml index 174f2ec..7cc7f19 100644 --- a/charts/appwrite/tests/compute_test.yaml +++ b/charts/appwrite/tests/compute_test.yaml @@ -8,8 +8,11 @@ release: values: - values/required.yaml set: + # Compute is on by default; the tests that need it turn it back on. + compute.enabled: false storage.device: s3 - storage.s3.endpoint: https://s3.example.com + storage.s3.mode: external + storage.s3.external.endpoint: https://s3.example.com storage.s3.bucket: files storage.s3.accessKeyId.secretName: s3 storage.s3.secretAccessKey.secretName: s3 diff --git a/charts/appwrite/tests/executor_test.yaml b/charts/appwrite/tests/executor_test.yaml index a6da7fe..90f0269 100644 --- a/charts/appwrite/tests/executor_test.yaml +++ b/charts/appwrite/tests/executor_test.yaml @@ -16,7 +16,8 @@ values: - values/required.yaml set: storage.device: s3 - storage.s3.endpoint: http://10.1.2.3:9000 + storage.s3.mode: external + storage.s3.external.endpoint: http://10.1.2.3:9000 storage.s3.bucket: files storage.s3.accessKeyId.value: key/+ storage.s3.secretAccessKey.value: secret @@ -195,7 +196,7 @@ tests: asserts: - contains: path: spec.template.spec.containers[0].env - content: {name: EXECUTOR_IMAGE, value: docker.io/openruntimes/executor:0.29.0} + content: {name: EXECUTOR_IMAGE, value: docker.io/openruntimes/executor:1.1.2} - contains: path: spec.template.spec.containers[0].env content: {name: OPR_EXECUTOR_IMAGES, value: "openruntimes/node:v5-22,openruntimes/python-ml:v5-3.11,openruntimes/static:v5-1"} @@ -228,8 +229,9 @@ tests: - it: reaches an HTTPS S3 endpoint template: templates/executor/deployment.yaml set: - storage.s3.endpoint: https://s3.example.com - storage.s3.region: eu-central + storage.s3.mode: external + storage.s3.external.endpoint: https://s3.example.com + storage.s3.external.region: eu-central asserts: - contains: path: spec.template.spec.containers[0].env @@ -238,8 +240,9 @@ tests: - it: names the bucket in the hostname on AWS S3 template: templates/executor/deployment.yaml set: - storage.s3.endpoint: "" - storage.s3.region: eu-central-1 + storage.s3.mode: external + storage.s3.external.endpoint: "" + storage.s3.external.region: eu-central-1 asserts: - contains: path: spec.template.spec.containers[0].env diff --git a/charts/appwrite/tests/migrations_test.yaml b/charts/appwrite/tests/migrations_test.yaml index 81310e5..87050f5 100644 --- a/charts/appwrite/tests/migrations_test.yaml +++ b/charts/appwrite/tests/migrations_test.yaml @@ -19,7 +19,7 @@ tests: asserts: - equal: path: metadata.annotations["appwrite/version"] - value: "2.3.0" + value: "2.5.0" - it: doesn't migrate on install templates: @@ -80,7 +80,7 @@ tests: name: aw-appwrite-api namespace: appwrite annotations: - appwrite/version: 2.3.0 + appwrite/version: 2.5.0 asserts: - hasDocuments: count: 0 @@ -99,10 +99,10 @@ tests: name: aw-appwrite-api namespace: appwrite annotations: - appwrite/version: 2.4.0 + appwrite/version: 2.6.0 asserts: - failedTemplate: - errorPattern: Appwrite 2.4.0 is installed and this would downgrade it to 2.3.0 + errorPattern: Appwrite 2.6.0 is installed and this would downgrade it to 2.5.0 - it: retries a hook migration that failed template: templates/hooks/migrate-hook.yaml @@ -123,7 +123,7 @@ tests: name: aw-appwrite-api namespace: appwrite annotations: - appwrite/version: 2.3.0 + appwrite/version: 2.5.0 - kind: Job apiVersion: batch/v1 metadata: @@ -140,11 +140,11 @@ tests: asserts: - matchRegex: path: metadata.name - pattern: ^aw-appwrite-migrate-2-3-0-[0-9a-f]{8}$ + pattern: ^aw-appwrite-migrate-2-5-0-[0-9a-f]{8}$ # Pinned so the rename test below compares against this exact render. - equal: path: metadata.name - value: aw-appwrite-migrate-2-3-0-48c408a6 + value: aw-appwrite-migrate-2-5-0-ea2a7473 - notExists: path: metadata.annotations["helm.sh/hook"] - notExists: @@ -158,10 +158,10 @@ tests: asserts: - notEqual: path: metadata.name - value: aw-appwrite-migrate-2-3-0-48c408a6 + value: aw-appwrite-migrate-2-5-0-ea2a7473 - matchRegex: path: metadata.name - pattern: ^aw-appwrite-migrate-2-3-0-[0-9a-f]{8}$ + pattern: ^aw-appwrite-migrate-2-5-0-[0-9a-f]{8}$ - it: renders nothing in none mode templates: diff --git a/charts/appwrite/tests/networkpolicy_test.yaml b/charts/appwrite/tests/networkpolicy_test.yaml index 2f7b41a..0e79a3b 100644 --- a/charts/appwrite/tests/networkpolicy_test.yaml +++ b/charts/appwrite/tests/networkpolicy_test.yaml @@ -76,6 +76,7 @@ tests: - it: restricts egress only when asked template: templates/networkpolicy/egress.yaml set: + compute.enabled: false networkPolicy.enabled: true networkPolicy.egress.enabled: true networkPolicy.egress.extraRules: diff --git a/charts/appwrite/tests/objectstorage_test.yaml b/charts/appwrite/tests/objectstorage_test.yaml new file mode 100644 index 0000000..f61cc56 --- /dev/null +++ b/charts/appwrite/tests/objectstorage_test.yaml @@ -0,0 +1,163 @@ +suite: built-in object storage +templates: + - templates/seaweedfs/statefulset.yaml + - templates/seaweedfs/service.yaml + - templates/secret.yaml + - templates/configmap-env.yaml + - templates/networkpolicy/object-storage.yaml + - templates/executor/networkpolicy.yaml + - templates/builds/networkpolicy.yaml +release: + name: aw + namespace: appwrite +values: + - values/required.yaml +set: + storage.device: s3 +tests: + - it: runs nothing with local storage + templates: + - templates/seaweedfs/statefulset.yaml + - templates/seaweedfs/service.yaml + set: + storage.device: local + compute.enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: runs nothing with an external store + templates: + - templates/seaweedfs/statefulset.yaml + - templates/seaweedfs/service.yaml + set: + storage.s3.mode: external + storage.s3.accessKeyId.value: key + storage.s3.secretAccessKey.value: secret + asserts: + - hasDocuments: + count: 0 + + - it: writes the S3 gateway's user from the chart Secret and serves only S3 + template: templates/seaweedfs/statefulset.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'exec weed server -dir=/data -ip=127.0.0.1 -ip.bind=0.0.0.0 -s3 -s3.config=/tmp/s3.json -master.volumeSizeLimitMB=1024 -volume.max=0' + - contains: + path: spec.template.spec.containers[0].env + content: + name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: {name: aw-appwrite, key: s3-access-key} + - equal: + path: spec.template.spec.containers[0].ports + value: [{name: s3, containerPort: 8333, protocol: TCP}] + - equal: + path: spec.volumeClaimTemplates[0].spec.resources.requests.storage + value: 20Gi + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 1000 + - exists: + path: spec.template.metadata.annotations["checksum/secret"] + + - it: creates the bucket before the pod is ready, accepting one that exists + template: templates/seaweedfs/statefulset.yaml + set: + storage.s3.bucket: files + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: {name: S3_BUCKET, value: files} + - matchRegex: + path: spec.template.spec.containers[0].lifecycle.postStart.exec.command[2] + pattern: 's3.bucket.create -name \$S3_BUCKET[\s\S]*already exists' + + - it: generates the access keys once, like the other built-in credentials + template: templates/secret.yaml + asserts: + - exists: + path: data["s3-access-key"] + - exists: + path: data["s3-secret-key"] + + - it: points Appwrite at the built-in store + template: templates/configmap-env.yaml + asserts: + - equal: + path: data._APP_STORAGE_S3_ENDPOINT + value: http://aw-appwrite-seaweedfs.appwrite.svc:8333 + - equal: + path: data._APP_STORAGE_S3_BUCKET + value: appwrite + + - it: requires credentials for an external store only + template: templates/configmap-env.yaml + set: + storage.s3.mode: external + asserts: + - failedTemplate: + errorPattern: storage.s3.accessKeyId and storage.s3.secretAccessKey need a value or a secretName with storage.s3.mode external + + - it: fails on the moved endpoint with where it went + template: templates/configmap-env.yaml + set: + storage.s3.endpoint: https://s3.example.com + asserts: + - failedTemplate: + errorPattern: "storage.s3.endpoint is no longer supported: moved to storage.s3.external.endpoint" + + - it: lets Appwrite, the orchestrator, builds and the executor reach the store + template: templates/networkpolicy/object-storage.yaml + set: + networkPolicy.enabled: true + compute.enabled: true + compute.executor.privileged: true + asserts: + - equal: + path: spec.ingress[0].ports + value: [{port: s3, protocol: TCP}] + - contains: + path: spec.ingress[0].from + content: + namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: appwrite-builds} + - contains: + path: spec.ingress[0].from + content: + namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: appwrite-executor} + + - it: lets the executor and build pods out to the built-in store + templates: + - templates/executor/networkpolicy.yaml + - templates/builds/networkpolicy.yaml + set: + compute.enabled: true + compute.executor.privileged: true + asserts: + - contains: + path: spec.egress + content: + to: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: appwrite} + podSelector: + matchLabels: + app.kubernetes.io/name: appwrite + app.kubernetes.io/instance: aw + matchExpressions: + - {key: app.kubernetes.io/component, operator: In, values: [seaweedfs]} + ports: [{port: 8333, protocol: TCP}] + + - it: refuses a StatefulSet name its pods' labels can't hold, with only the store built in + template: templates/configmap-env.yaml + set: + fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + database.mode: external + redis.mode: external + clickhouse.mode: external + asserts: + - failedTemplate: + errorPattern: StatefulSet name "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-seaweedfs" is longer than 52 characters diff --git a/charts/appwrite/tests/optional_test.yaml b/charts/appwrite/tests/optional_test.yaml index 0a36d6a..675759a 100644 --- a/charts/appwrite/tests/optional_test.yaml +++ b/charts/appwrite/tests/optional_test.yaml @@ -1,8 +1,6 @@ suite: optional components templates: - templates/browser/deployment.yaml - - templates/assistant/deployment.yaml - - templates/assistant/service.yaml - templates/autogravity/deployment.yaml - templates/embedding/deployment.yaml - templates/embedding/pvc.yaml @@ -18,7 +16,6 @@ tests: - it: deploys none of them by default templates: - templates/browser/deployment.yaml - - templates/assistant/deployment.yaml - templates/autogravity/deployment.yaml - templates/embedding/deployment.yaml - templates/mqtt/deployment.yaml @@ -65,39 +62,24 @@ tests: path: spec.template.spec.securityContext.runAsUser value: 999 - - it: names the assistant Service as Appwrite expects - template: templates/assistant/service.yaml - set: - assistant.enabled: true - assistant.apiKey.value: sk - asserts: - - equal: - path: metadata.name - value: appwrite-assistant - - equal: - path: spec.ports[0].port - value: 3003 - - - it: gives Appwrite the assistant key so the console shows it - template: templates/api/deployment.yaml + - it: lets the browser load sites through the API Service + template: templates/browser/deployment.yaml set: - assistant.enabled: true - assistant.apiKey.secretName: openai + browser.enabled: true asserts: - contains: path: spec.template.spec.containers[0].env content: - name: _APP_ASSISTANT_OPENAI_API_KEY - valueFrom: - secretKeyRef: {name: openai, key: openai-api-key} + name: ALLOWED_HOSTS + value: aw-appwrite-api.NAMESPACE.svc - - it: requires an OpenAI key for the assistant + - it: fails on the removed assistant with what to do instead template: templates/configmap-env.yaml set: assistant.enabled: true asserts: - failedTemplate: - errorPattern: assistant.enabled needs + errorPattern: "assistant is no longer supported: Appwrite removed the assistant in 2.4.0" - it: keeps embedding models on a volume and recreates the pod on updates template: templates/embedding/deployment.yaml diff --git a/charts/appwrite/tests/secret_test.yaml b/charts/appwrite/tests/secret_test.yaml index 65c4bac..a5d6cfc 100644 --- a/charts/appwrite/tests/secret_test.yaml +++ b/charts/appwrite/tests/secret_test.yaml @@ -132,6 +132,8 @@ tests: - it: skips credentials that come from existing Secrets set: + compute.enabled: false + storage.device: local appwrite.secrets.existingSecret: core database.password.secretName: pg redis.password.secretName: redis diff --git a/charts/appwrite/tests/serviceaccount_test.yaml b/charts/appwrite/tests/serviceaccount_test.yaml index f46f401..fcf68e6 100644 --- a/charts/appwrite/tests/serviceaccount_test.yaml +++ b/charts/appwrite/tests/serviceaccount_test.yaml @@ -28,7 +28,7 @@ tests: app.kubernetes.io/instance: my-release app.kubernetes.io/managed-by: Helm app.kubernetes.io/part-of: appwrite - app.kubernetes.io/version: "2.3.0" + app.kubernetes.io/version: "2.5.0" - matchRegex: path: metadata.labels["helm.sh/chart"] pattern: ^appwrite-\d+\.\d+\.\d+ diff --git a/charts/appwrite/values.schema.json b/charts/appwrite/values.schema.json index 8de3001..f653ba1 100644 --- a/charts/appwrite/values.schema.json +++ b/charts/appwrite/values.schema.json @@ -541,180 +541,6 @@ }, "additionalProperties": false }, - "assistant": { - "type": "object", - "properties": { - "affinity": { - "type": "object", - "description": "Affinity. Empty inherits `defaults.affinity`." - }, - "apiKey": { - "type": "object", - "properties": { - "secretKey": { - "description": "Key in the Secret.", - "type": "string" - }, - "secretName": { - "description": "Existing Secret holding the OpenAI API key. Takes precedence over `value`.", - "type": "string" - }, - "value": { - "description": "OpenAI API key (`_APP_ASSISTANT_OPENAI_API_KEY`), set inline. Stored in the chart's Secret.", - "type": "string" - } - }, - "additionalProperties": false - }, - "containerSecurityContext": { - "type": "object", - "description": "Container security context." - }, - "enabled": { - "description": "Deploy the assistant.", - "type": "boolean" - }, - "extraContainers": { - "type": "array", - "items": { - "type": "object" - }, - "description": "Extra sidecar containers." - }, - "extraEnv": { - "type": "array", - "items": { - "type": "object" - }, - "description": "Extra environment variables, as a list of EnvVar objects." - }, - "extraInitContainers": { - "type": "array", - "items": { - "type": "object" - }, - "description": "Extra init containers." - }, - "extraVolumeMounts": { - "type": "array", - "items": { - "type": "object" - }, - "description": "Extra volume mounts for the main container." - }, - "extraVolumes": { - "type": "array", - "items": { - "type": "object" - }, - "description": "Extra volumes." - }, - "image": { - "type": "object", - "properties": { - "digest": { - "type": "string", - "pattern": "^$|^sha256:[a-f0-9]{64}$", - "description": "Image digest." - }, - "pullPolicy": { - "description": "Image pull policy.", - "type": "string", - "enum": [ - "Always", - "IfNotPresent", - "Never" - ] - }, - "registry": { - "description": "Image registry.", - "type": "string" - }, - "repository": { - "description": "Image repository.", - "type": "string" - }, - "tag": { - "description": "Image tag.", - "type": "string" - } - }, - "additionalProperties": false - }, - "nodeSelector": { - "description": "Node selector. Empty inherits `defaults.nodeSelector`.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "podAnnotations": { - "description": "Pod annotations, merged over `defaults.podAnnotations`.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "podLabels": { - "description": "Pod labels, merged over `defaults.podLabels`.", - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "podSecurityContext": { - "type": "object", - "description": "Pod security context." - }, - "priorityClassName": { - "description": "Priority class. Empty inherits `defaults.priorityClassName`.", - "type": "string" - }, - "resources": { - "type": "object", - "properties": { - "requests": { - "type": "object", - "additionalProperties": { - "type": [ - "string", - "number" - ] - } - }, - "limits": { - "type": "object", - "additionalProperties": { - "type": [ - "string", - "number" - ] - } - }, - "claims": { - "type": "array", - "items": { - "type": "object" - } - } - }, - "additionalProperties": false, - "description": "Resource requests and limits." - }, - "terminationGracePeriodSeconds": { - "description": "Seconds to wait on shutdown.", - "type": "integer" - }, - "tolerations": { - "type": "array", - "items": { - "type": "object" - }, - "description": "Tolerations. Empty inherits `defaults.tolerations`." - } - }, - "additionalProperties": false - }, "autogravity": { "type": "object", "properties": { @@ -1387,7 +1213,7 @@ "additionalProperties": false }, "enabled": { - "description": "Enable Functions and Sites. Needs `storage.device: s3` and `compute.executor.privileged` (see docs/executor.md).", + "description": "Enable Functions and Sites. Needs `storage.device: s3`, and runs the executor as a privileged pod in a namespace of its own (see docs/executor.md). Set to false on clusters that forbid privileged pods.", "type": "boolean" }, "executor": { @@ -1553,7 +1379,7 @@ "type": "string" }, "privileged": { - "description": "Run the Docker daemon as a privileged container, which makes it root on its node. Required unless `runtimeClassName` names a runtime that runs Docker without it, such as Sysbox. Give the executor nodes of its own (`nodeSelector`, `tolerations`).", + "description": "Run the Docker daemon as a privileged container, which makes it root on its node. Required unless `runtimeClassName` names a runtime that runs Docker without it, such as Sysbox. For production, give the executor nodes of its own (`nodeSelector`, `tolerations`).", "type": "boolean" }, "replicaCount": { @@ -1618,7 +1444,7 @@ "additionalProperties": false }, "functionsRuntimes": { - "description": "Function runtimes to enable (`_APP_FUNCTIONS_RUNTIMES`). Empty keeps the Docker installer's list (node-16.0, php-8.0, python-3.9, ruby-3.0).", + "description": "Function runtimes to enable (`_APP_FUNCTIONS_RUNTIMES`). Empty keeps the Docker installer's list (node-16.0, node-22, php-8.0, python-3.9, ruby-3.0).", "type": "array", "items": { "type": "string" @@ -1837,9 +1663,6 @@ }, "VITE_CONSOLE_PROFILE": { "type": "string" - }, - "VITE_GROWTH_ENDPOINT": { - "type": "string" } }, "additionalProperties": { @@ -3531,7 +3354,7 @@ "description": "DNS servers (NetworkPolicyPeer) allowed on port 53 wherever egress is restricted: here, for the browser, and for the executor and build Jobs, whose policies are on by default with compute. For NodeLocal DNSCache, add `{ipBlock: {cidr: 169.254.20.10/32}}`. DNS on another port (OpenShift: 5353) needs rules in `extraRules`, `compute.executor.networkPolicy.extraEgress` and `compute.builds.networkPolicy.extraEgress`." }, "enabled": { - "description": "Restrict egress to DNS, pods of this release and `extraRules` (needs `networkPolicy.enabled`). Off by default: Appwrite calls webhooks, SMTP servers, OAuth providers, S3, and the DNS server it verifies custom domains against (`_APP_DNS`, 8.8.8.8). With compute, the orchestrator also needs the Kubernetes API. Allow what you use in `extraRules`; the executor is allowed already.", + "description": "Restrict egress to DNS, pods of this release and `extraRules` (needs `networkPolicy.enabled`). Off by default: Appwrite calls webhooks, SMTP servers, OAuth providers, S3, and the DNS server it verifies custom domains, migration sources and avatar URLs against (`_APP_DNS` and `_APP_DNS_EXTERNAL`, 8.8.8.8). With compute, the orchestrator also needs the Kubernetes API. Allow what you use in `extraRules`; the executor is allowed already.", "type": "boolean" }, "extraRules": { @@ -3545,7 +3368,7 @@ "additionalProperties": false }, "enabled": { - "description": "Create NetworkPolicies in the release namespace. The built-in datastores then accept only Appwrite pods of this release, the internal services (geo, browser, assistant, autogravity, embedding) only pods of this release, and workers and tasks no incoming traffic. Off by default: any pod in the cluster can reach the built-in PostgreSQL, Redis and ClickHouse, which only their passwords protect. Policies only take effect with a CNI that enforces them.", + "description": "Create NetworkPolicies in the release namespace. The built-in datastores then accept only Appwrite pods of this release, the internal services (geo, browser, autogravity, embedding) only pods of this release, and workers and tasks no incoming traffic. Off by default: any pod in the cluster can reach the built-in PostgreSQL, Redis and ClickHouse, which only their passwords protect. Policies only take effect with a CNI that enforces them.", "type": "boolean" }, "ingress": { @@ -4297,7 +4120,7 @@ "type": "object", "properties": { "device": { - "description": "Where Appwrite stores files (`_APP_STORAGE_DEVICE`).", + "description": "Where Appwrite stores files (`_APP_STORAGE_DEVICE`): `s3`, by default the built-in SeaweedFS (`storage.s3.mode`), or `local`, the `/storage` volume. Functions and Sites need `s3`.", "type": "string", "enum": [ "local", @@ -4329,23 +4152,226 @@ "type": "string" }, "value": { - "description": "Access key ID, set inline. Stored in the chart's Secret.", + "description": "Access key ID, set inline. Stored in the chart's Secret. With `mode: builtin`, generated once when empty.", "type": "string" } }, "additionalProperties": false }, "bucket": { - "description": "Bucket (`_APP_STORAGE_S3_BUCKET`). Required with `device: s3`.", + "description": "Bucket (`_APP_STORAGE_S3_BUCKET`). The built-in store creates it.", "type": "string" }, - "endpoint": { - "description": "Endpoint of an S3-compatible store (`_APP_STORAGE_S3_ENDPOINT`), e.g. `https://fsn1.your-objectstorage.com` or `https://.r2.cloudflarestorage.com`. Empty uses AWS S3. Endpoints and regions per provider: docs/storage.md#s3.", - "type": "string" + "builtin": { + "type": "object", + "properties": { + "affinity": { + "type": "object", + "description": "Affinity. Empty inherits `defaults.affinity`." + }, + "containerSecurityContext": { + "type": "object", + "description": "Container security context." + }, + "extraContainers": { + "type": "array", + "items": { + "type": "object" + }, + "description": "Extra sidecar containers." + }, + "extraEnv": { + "type": "array", + "items": { + "type": "object" + }, + "description": "Extra environment variables, as a list of EnvVar objects." + }, + "extraInitContainers": { + "type": "array", + "items": { + "type": "object" + }, + "description": "Extra init containers." + }, + "extraVolumeMounts": { + "type": "array", + "items": { + "type": "object" + }, + "description": "Extra volume mounts for the main container." + }, + "extraVolumes": { + "type": "array", + "items": { + "type": "object" + }, + "description": "Extra volumes." + }, + "image": { + "type": "object", + "properties": { + "digest": { + "type": "string", + "pattern": "^$|^sha256:[a-f0-9]{64}$", + "description": "Image digest." + }, + "pullPolicy": { + "description": "Image pull policy.", + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + }, + "registry": { + "description": "Image registry.", + "type": "string" + }, + "repository": { + "description": "Image repository.", + "type": "string" + }, + "tag": { + "description": "Image tag.", + "type": "string" + } + }, + "additionalProperties": false + }, + "nodeSelector": { + "description": "Node selector. Empty inherits `defaults.nodeSelector`.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "persistence": { + "type": "object", + "properties": { + "accessModes": { + "description": "Access modes.", + "type": "array", + "items": { + "type": "string" + } + }, + "annotations": { + "description": "Annotations for the PersistentVolumeClaim.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "enabled": { + "description": "Persist data in a PersistentVolumeClaim. Without it, every restart loses the files and build output.", + "type": "boolean" + }, + "size": { + "description": "Volume size: uploaded files, deployment sources and build output.", + "type": "string" + }, + "storageClass": { + "description": "Storage class. Empty uses `global.storageClass`, then the cluster default; `-` requests no class.", + "type": "string" + } + }, + "additionalProperties": false + }, + "podAnnotations": { + "description": "Pod annotations, merged over `defaults.podAnnotations`.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "podLabels": { + "description": "Pod labels, merged over `defaults.podLabels`.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "podSecurityContext": { + "type": "object", + "description": "Pod security context. The image runs as root by default; the chart runs it as UID 1000." + }, + "priorityClassName": { + "description": "Priority class. Empty inherits `defaults.priorityClassName`.", + "type": "string" + }, + "resources": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number" + ] + } + }, + "limits": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number" + ] + } + }, + "claims": { + "type": "array", + "items": { + "type": "object" + } + } + }, + "additionalProperties": false, + "description": "Resource requests and limits." + }, + "terminationGracePeriodSeconds": { + "description": "Seconds to wait on shutdown.", + "type": "integer" + }, + "tolerations": { + "type": "array", + "items": { + "type": "object" + }, + "description": "Tolerations. Empty inherits `defaults.tolerations`." + }, + "volumeSizeLimitMB": { + "description": "Size of each SeaweedFS volume file in MB. The store fills the disk with volumes of this size, so keep it well below `persistence.size`.", + "type": "integer", + "minimum": 1 + } + }, + "additionalProperties": false }, - "region": { - "description": "Region (`_APP_STORAGE_S3_REGION`).", - "type": "string" + "external": { + "type": "object", + "properties": { + "endpoint": { + "description": "Endpoint (`_APP_STORAGE_S3_ENDPOINT`), e.g. `https://fsn1.your-objectstorage.com` or `https://.r2.cloudflarestorage.com`. Empty uses AWS S3. Endpoints and regions per provider: docs/storage.md#s3.", + "type": "string" + }, + "region": { + "description": "Region (`_APP_STORAGE_S3_REGION`).", + "type": "string" + } + }, + "additionalProperties": false + }, + "mode": { + "description": "`builtin` runs SeaweedFS, an S3-compatible store, in the release; `external` uses `storage.s3.external`. Only used with `device: s3`.", + "type": "string", + "enum": [ + "builtin", + "external" + ] }, "secretAccessKey": { "type": "object", @@ -4359,11 +4385,17 @@ "type": "string" }, "value": { - "description": "Secret access key, set inline. Stored in the chart's Secret.", + "description": "Secret access key, set inline. Stored in the chart's Secret. With `mode: builtin`, generated once when empty.", "type": "string" } }, "additionalProperties": false + }, + "endpoint": { + "description": "No longer supported: setting storage.s3.endpoint fails with what to do instead." + }, + "region": { + "description": "No longer supported: setting storage.s3.region fails with what to do instead." } }, "additionalProperties": false @@ -6019,6 +6051,9 @@ } }, "additionalProperties": false + }, + "assistant": { + "description": "No longer supported: setting assistant fails with what to do instead." } }, "additionalProperties": false diff --git a/charts/appwrite/values.yaml b/charts/appwrite/values.yaml index b85c7c7..49f3101 100644 --- a/charts/appwrite/values.yaml +++ b/charts/appwrite/values.yaml @@ -427,7 +427,7 @@ vectorsDb: # Redis, for queues, cache, pub/sub and locks. `builtin` runs a # single-replica StatefulSet; `external` connects to a single endpoint -# without TLS (Appwrite 2.3.0 has no Redis TLS, Sentinel or Cluster). +# without TLS (Appwrite 2.5.0 has no Redis TLS, Sentinel or Cluster). # See docs/external-databases.md#redis. redis: # -- `builtin` deploys Redis in the release; `external` uses `redis.external`. @@ -650,36 +650,124 @@ clickhouse: # the `/storage` volume, `s3` in an S3-compatible bucket (required for # Functions and Sites). See docs/storage.md. storage: - # -- Where Appwrite stores files (`_APP_STORAGE_DEVICE`). - device: local # @schema enum: [local, s3] + # -- Where Appwrite stores files (`_APP_STORAGE_DEVICE`): `s3`, by default + # the built-in SeaweedFS (`storage.s3.mode`), or `local`, the `/storage` + # volume. Functions and Sites need `s3`. + device: s3 # @schema enum: [local, s3] # -- Maximum file size in bytes (`_APP_STORAGE_LIMIT`). limit: 30000000 # @schema minimum: 0 # -- Maximum size in bytes of files Appwrite renders previews for (`_APP_STORAGE_PREVIEW_LIMIT`). previewLimit: 20000000 # @schema minimum: 0 s3: - # -- Endpoint of an S3-compatible store (`_APP_STORAGE_S3_ENDPOINT`), e.g. - # `https://fsn1.your-objectstorage.com` or - # `https://.r2.cloudflarestorage.com`. Empty uses AWS S3. - # Endpoints and regions per provider: docs/storage.md#s3. - endpoint: "" - # -- Region (`_APP_STORAGE_S3_REGION`). - region: us-east-1 - # -- Bucket (`_APP_STORAGE_S3_BUCKET`). Required with `device: s3`. - bucket: "" + # -- `builtin` runs SeaweedFS, an S3-compatible store, in the release; + # `external` uses `storage.s3.external`. Only used with `device: s3`. + mode: builtin # @schema enum: [builtin, external] + # -- Bucket (`_APP_STORAGE_S3_BUCKET`). The built-in store creates it. + bucket: appwrite accessKeyId: - # -- Access key ID, set inline. Stored in the chart's Secret. + # -- Access key ID, set inline. Stored in the chart's Secret. With + # `mode: builtin`, generated once when empty. value: "" # -- Existing Secret holding the access key id. Takes precedence over `value`. secretName: "" # -- Key in the Secret. secretKey: s3-access-key secretAccessKey: - # -- Secret access key, set inline. Stored in the chart's Secret. + # -- Secret access key, set inline. Stored in the chart's Secret. With + # `mode: builtin`, generated once when empty. value: "" # -- Existing Secret holding the secret access key. Takes precedence over `value`. secretName: "" # -- Key in the Secret. secretKey: s3-secret-key + # Your own S3-compatible store (`mode: external`). + external: + # -- Endpoint (`_APP_STORAGE_S3_ENDPOINT`), e.g. + # `https://fsn1.your-objectstorage.com` or + # `https://.r2.cloudflarestorage.com`. Empty uses AWS S3. + # Endpoints and regions per provider: docs/storage.md#s3. + endpoint: "" + # -- Region (`_APP_STORAGE_S3_REGION`). + region: us-east-1 + # Settings of the built-in SeaweedFS (`mode: builtin`): a single-replica + # StatefulSet running master, volume server, filer and S3 gateway in one + # process. See docs/storage.md#built-in-object-storage. + builtin: + image: + # -- Image registry. + registry: docker.io + # -- Image repository. + repository: chrislusf/seaweedfs + # -- Image tag. + tag: "4.48" + # -- Image digest. + digest: "" + # -- Image pull policy. + pullPolicy: IfNotPresent # @schema enum: [Always, IfNotPresent, Never] + # -- Size of each SeaweedFS volume file in MB. The store fills the disk + # with volumes of this size, so keep it well below `persistence.size`. + volumeSizeLimitMB: 1024 # @schema minimum: 1 + # -- Resource requests and limits. + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 1Gi + persistence: + # -- Persist data in a PersistentVolumeClaim. Without it, every + # restart loses the files and build output. + enabled: true + # -- Storage class. Empty uses `global.storageClass`, then the cluster + # default; `-` requests no class. + storageClass: "" + # -- Access modes. + accessModes: [ReadWriteOnce] # @schema item: string + # -- Volume size: uploaded files, deployment sources and build output. + size: 20Gi + # -- Annotations for the PersistentVolumeClaim. + annotations: {} # @schema additionalProperties: {type: string} + # -- Pod security context. The image runs as root by default; the + # chart runs it as UID 1000. + podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + # -- Container security context. + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + privileged: false + capabilities: + drop: [ALL] + # -- Pod annotations, merged over `defaults.podAnnotations`. + podAnnotations: {} # @schema additionalProperties: {type: string} + # -- Pod labels, merged over `defaults.podLabels`. + podLabels: {} # @schema additionalProperties: {type: string} + # -- Node selector. Empty inherits `defaults.nodeSelector`. + nodeSelector: {} # @schema additionalProperties: {type: string} + # -- Tolerations. Empty inherits `defaults.tolerations`. + tolerations: [] # @schema item: object + # -- Affinity. Empty inherits `defaults.affinity`. + affinity: {} # @schema additionalProperties: true + # -- Priority class. Empty inherits `defaults.priorityClassName`. + priorityClassName: "" + # -- Seconds to wait on shutdown. + terminationGracePeriodSeconds: 60 + # -- Extra environment variables, as a list of EnvVar objects. + extraEnv: [] # @schema item: object + # -- Extra volumes. + extraVolumes: [] # @schema item: object + # -- Extra volume mounts for the main container. + extraVolumeMounts: [] # @schema item: object + # -- Extra init containers. + extraInitContainers: [] # @schema item: object + # -- Extra sidecar containers. + extraContainers: [] # @schema item: object # The `/storage` volume. Appwrite uses it for files with `device: local`, # and for certificates and scratch files with either device. volume: @@ -731,18 +819,19 @@ smtp: # of its own, and the executor runs executions on Docker-in-Docker in another. # See docs/executor.md. compute: - # -- Enable Functions and Sites. Needs `storage.device: s3` and - # `compute.executor.privileged` (see docs/executor.md). - enabled: false + # -- Enable Functions and Sites. Needs `storage.device: s3`, and runs the + # executor as a privileged pod in a namespace of its own (see + # docs/executor.md). Set to false on clusters that forbid privileged pods. + enabled: true # The open-runtimes executor, which starts a container per function or site # runtime and executes requests in it. It needs a Docker daemon, so it runs # on Docker-in-Docker, in a namespace of its own. executor: # -- Run the Docker daemon as a privileged container, which makes it root # on its node. Required unless `runtimeClassName` names a runtime that runs - # Docker without it, such as Sysbox. Give the executor nodes of its own - # (`nodeSelector`, `tolerations`). - privileged: false + # Docker without it, such as Sysbox. For production, give the executor + # nodes of its own (`nodeSelector`, `tolerations`). + privileged: true # -- RuntimeClass of executor pods, e.g. `sysbox-runc`. runtimeClassName: "" # -- Namespace of the executor. Defaults to `-executor`. @@ -761,7 +850,7 @@ compute: # -- Image repository. repository: openruntimes/executor # -- Image tag. - tag: 0.29.0 + tag: 1.1.2 # -- Image digest. digest: "" dind: @@ -947,7 +1036,7 @@ compute: # ranges, IPv6 clusters). deniedCIDRs: [] # @schema item: string # -- Function runtimes to enable (`_APP_FUNCTIONS_RUNTIMES`). Empty keeps the - # Docker installer's list (node-16.0, php-8.0, python-3.9, ruby-3.0). + # Docker installer's list (node-16.0, node-22, php-8.0, python-3.9, ruby-3.0). functionsRuntimes: [] # @schema item: string # -- Site runtimes to enable (`_APP_SITES_RUNTIMES`). Empty keeps the # Docker installer's list (static-1, node-22, flutter-3.29). Sites build @@ -1091,7 +1180,7 @@ console: # -- Image repository. repository: appwrite/new # -- Image tag. Pinned to the console that ships with the chart's appVersion. - tag: 1.1.159-self-hosted + tag: 1.2.74-self-hosted # -- Image digest. digest: "" # -- Image pull policy. @@ -1100,7 +1189,6 @@ console: env: VITE_CONSOLE_PROFILE: self-hosted APPWRITE_ENDPOINT_SAME_ORIGIN: "true" - VITE_GROWTH_ENDPOINT: https://growth.appwrite.io/v1 # -- Number of console replicas. Ignored when autoscaling is enabled. replicaCount: 1 # @schema minimum: 0 # -- Resource requests and limits. @@ -1738,7 +1826,7 @@ geo: # -- Image repository. repository: appwrite/geo # -- Image tag. - tag: 0.3.1 + tag: 0.3.2 # -- Image digest. digest: "" # -- Image pull policy. @@ -1807,7 +1895,7 @@ browser: # -- Image repository. repository: appwrite/browser # -- Image tag. - tag: 0.3.4 + tag: 0.3.6 # -- Image digest. digest: "" # -- Image pull policy. @@ -1861,78 +1949,6 @@ browser: # -- Extra sidecar containers. extraContainers: [] # @schema item: object -# AI assistant in the console, backed by OpenAI. Appwrite reaches it at the -# fixed address `appwrite-assistant:3003`, so its Service is named -# `appwrite-assistant` and only one release per namespace can enable it. -# See docs/optional-components.md#assistant. -assistant: - # -- Deploy the assistant. - enabled: false - image: - # -- Image registry. - registry: docker.io - # -- Image repository. - repository: appwrite/assistant - # -- Image tag. - tag: 0.8.4 - # -- Image digest. - digest: "" - # -- Image pull policy. - pullPolicy: IfNotPresent # @schema enum: [Always, IfNotPresent, Never] - apiKey: - # -- OpenAI API key (`_APP_ASSISTANT_OPENAI_API_KEY`), set inline. Stored in the chart's Secret. - value: "" - # -- Existing Secret holding the OpenAI API key. Takes precedence over `value`. - secretName: "" - # -- Key in the Secret. - secretKey: openai-api-key - # -- Resource requests and limits. - resources: - requests: - cpu: 20m - memory: 128Mi - limits: - memory: 512Mi - # -- Pod security context. - podSecurityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - # -- Container security context. - containerSecurityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - privileged: false - capabilities: - drop: [ALL] - # -- Pod annotations, merged over `defaults.podAnnotations`. - podAnnotations: {} # @schema additionalProperties: {type: string} - # -- Pod labels, merged over `defaults.podLabels`. - podLabels: {} # @schema additionalProperties: {type: string} - # -- Node selector. Empty inherits `defaults.nodeSelector`. - nodeSelector: {} # @schema additionalProperties: {type: string} - # -- Tolerations. Empty inherits `defaults.tolerations`. - tolerations: [] # @schema item: object - # -- Affinity. Empty inherits `defaults.affinity`. - affinity: {} # @schema additionalProperties: true - # -- Priority class. Empty inherits `defaults.priorityClassName`. - priorityClassName: "" - # -- Seconds to wait on shutdown. - terminationGracePeriodSeconds: 30 - # -- Extra environment variables, as a list of EnvVar objects. - extraEnv: [] # @schema item: object - # -- Extra volumes. - extraVolumes: [] # @schema item: object - # -- Extra volume mounts for the main container. - extraVolumeMounts: [] # @schema item: object - # -- Extra init containers. - extraInitContainers: [] # @schema item: object - # -- Extra sidecar containers. - extraContainers: [] # @schema item: object - # Image analysis that picks the focal point of images for smart cropping # (`_APP_AUTOGRAVITY_HOST`). See docs/optional-components.md#autogravity. autogravity: @@ -2225,7 +2241,7 @@ httpRoute: networkPolicy: # -- Create NetworkPolicies in the release namespace. The built-in # datastores then accept only Appwrite pods of this release, the internal - # services (geo, browser, assistant, autogravity, embedding) only pods of + # services (geo, browser, autogravity, embedding) only pods of # this release, and workers and tasks no incoming traffic. Off by default: # any pod in the cluster can reach the built-in PostgreSQL, Redis and # ClickHouse, which only their passwords protect. Policies only take effect @@ -2245,7 +2261,8 @@ networkPolicy: egress: # -- Restrict egress to DNS, pods of this release and `extraRules` (needs # `networkPolicy.enabled`). Off by default: Appwrite calls webhooks, SMTP servers, OAuth providers, S3, and - # the DNS server it verifies custom domains against (`_APP_DNS`, 8.8.8.8). + # the DNS server it verifies custom domains, migration sources and avatar + # URLs against (`_APP_DNS` and `_APP_DNS_EXTERNAL`, 8.8.8.8). # With compute, the orchestrator also needs the Kubernetes API. Allow what # you use in `extraRules`; the executor is allowed already. enabled: false diff --git a/docs/README.md b/docs/README.md index 0b19761..c2db61a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -27,7 +27,7 @@ others: | PostgreSQL | built-in StatefulSet | RDS, Cloud SQL, Azure, CloudNativePG; or MariaDB, MySQL, MongoDB | [External databases](external-databases.md#postgresql) | | Redis | built-in StatefulSet | ElastiCache, Memorystore, Azure Cache, Valkey | [External databases](external-databases.md#redis) | | ClickHouse | built-in StatefulSet | ClickHouse Cloud, Altinity operator | [External databases](external-databases.md#clickhouse) | -| File storage | PersistentVolumeClaim | S3, R2, GCS, Spaces, MinIO, ... | [Storage](storage.md#s3) | +| File storage, deployments and builds | built-in object storage (SeaweedFS) | S3, R2, GCS, Spaces, MinIO, ...; or a PersistentVolumeClaim | [Storage](storage.md#s3) | | Email | off | any SMTP server: SES, SendGrid, Postmark, ... | [Integrations](integrations.md#email-smtp) | | Git deployments, console sign-in, SMS, Sentry | off | GitHub, GitLab, Gitea, Bitbucket, Google, Twilio, ... | [Integrations](integrations.md) | | Antivirus | off | ClamAV | [Storage](storage.md#antivirus) | @@ -49,7 +49,7 @@ built-in services, see | [Storage](storage.md) | Volumes, ReadWriteMany, S3, antivirus | | [Email, Git and other integrations](integrations.md) | SMTP, GitHub App, console sign-in, SMS, Sentry | | [Functions and Sites](executor.md) | Builds as Jobs and the executor on Docker-in-Docker | -| [Optional components](optional-components.md) | Browser, assistant, autogravity, embedding, MQTT | +| [Optional components](optional-components.md) | Browser, autogravity, embedding, MQTT | | [Security](security.md) | Pod security, credentials, NetworkPolicies, chart signatures | | [Every value](../charts/appwrite/README.md#values) | The generated reference of `values.yaml` | diff --git a/docs/architecture.md b/docs/architecture.md index c7a2004..ea230bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,14 +43,15 @@ flowchart LR | `postgresql` | `appwrite/postgres` | `postgres` | 5432 | One replica | Built-in database. Optional. | | `redis` | `redis` | `redis-server` | 6379 | One replica | Cache, queues, pub/sub and locks. Optional. | | `clickhouse` | `clickhouse/clickhouse-server` | image default | 8123, 9000 | One replica | Usage statistics and execution logs. Optional. | -| `orchestrator` | `open-runtimes/orchestrator/jobs-service` | image default | 8080 | One replica | Runs builds of functions and sites as Jobs in the build namespace. With `compute.enabled`. | +| `seaweedfs` | `chrislusf/seaweedfs` | `weed server -s3` | 8333 | One replica | Built-in S3-compatible object storage for files, deployments and build output. Optional (`storage.s3.mode`). | +| `orchestrator` | `open-runtimes/orchestrator/jobs-service` | image default | 8080 | One replica | Runs builds of functions and sites as Jobs in the build namespace. With `compute.enabled`, the default. | | `executor` | `openruntimes/executor`, run by `docker:dind` | see [executor.md](executor.md#the-executor) | 9900 | `compute.executor.replicaCount` | Runs functions and sites in runtime containers of its own Docker daemon, in the executor namespace. With `compute.enabled`. | Every component has a ClusterIP Service named `-`, except workers and tasks, which nothing connects to. The API, console, realtime and geo Services listen on port 80; the others use the container port. The datastores also get a headless Service for their StatefulSet. -Optional components such as the browser, assistant and MQTT broker are +Optional components such as the browser and the MQTT broker are described in [optional-components.md](optional-components.md). The API and the builds worker submit builds to the orchestrator, and the API diff --git a/docs/executor.md b/docs/executor.md index 9a8f7ff..4ecf3a1 100644 --- a/docs/executor.md +++ b/docs/executor.md @@ -13,10 +13,10 @@ unprivileged Jobs. The executor creates a container per runtime through a Docker daemon, and Kubernetes has no Docker daemon to offer, so the chart gives each executor pod its own (Docker-in-Docker). That daemon runs as a **privileged container: root on its node**. That is why the executor gets a -namespace of its own, why you have to accept it explicitly, and why it should -get nodes of its own. Where privileged pods aren't allowed (GKE Autopilot, -EKS on Fargate, a policy that forbids them), [Sysbox](#without-a-privileged-container) -runs the daemon without it. +namespace of its own, and why it should get nodes of its own. Where +privileged pods aren't allowed (GKE Autopilot, EKS on Fargate, a policy that +forbids them), [Sysbox](#without-a-privileged-container) runs the daemon +without it, or set `compute.enabled: false`. ```mermaid flowchart LR @@ -41,13 +41,16 @@ flowchart LR executor -- "build output" --> s3 ``` -Functions and Sites are off by default (`compute.enabled: false`). The rest -of Appwrite works without them; deployments of functions and sites stay in -`waiting` and executions fail. +Functions and Sites are on by default, on the +[built-in object storage](storage.md#built-in-object-storage). With +`compute.enabled: false`, the rest of Appwrite works, but creating a function +or site fails +([troubleshooting.md](troubleshooting.md#deployments-fail-with-an-internal-error)). ## Requirements -- **S3 storage** (`storage.device: s3`, see [storage.md](storage.md)). +- **S3 storage** (`storage.device: s3`, the default, see + [storage.md](storage.md)): the built-in object storage or your own store. Builds and the executor exchange code and build output through the bucket. The chart refuses to render `compute.enabled` without it. - **Nodes that may run a privileged pod** for the executor, ideally @@ -67,6 +70,10 @@ of Appwrite works without them; deployments of functions and sites stay in ## 1. Configure the chart +A plain install already runs both, with the built-in object storage and the +executor on any node. For production, pin the executor to nodes of its own, +use an S3 store with redundancy, and pick the runtimes you need: + ```yaml appwrite: domain: appwrite.example.com @@ -74,8 +81,10 @@ appwrite: storage: device: s3 s3: - endpoint: https://s3.example.com # empty for AWS S3 - region: us-east-1 + mode: external + external: + endpoint: https://s3.example.com # empty for AWS S3 + region: us-east-1 bucket: appwrite accessKeyId: value: @@ -83,9 +92,8 @@ storage: value: compute: - enabled: true executor: - privileged: true # Docker-in-Docker: root on the node + privileged: true # the default; Docker-in-Docker: root on the node nodeSelector: workload: executor tolerations: @@ -115,7 +123,7 @@ instead ([existing Secrets](#existing-secrets)). | --- | --- | --- | | `compute.enabled` | `false` | Deploys the orchestrator and the executor and turns on Functions and Sites. | | `compute.executor.privileged` | `false` | Accepts the privileged Docker daemon. Required unless `compute.executor.runtimeClassName` is set. | -| `compute.functionsRuntimes` | `[]` | Function runtimes (`_APP_FUNCTIONS_RUNTIMES`). Empty keeps the Docker installer's list (`node-16.0`, `php-8.0`, `python-3.9`, `ruby-3.0`). | +| `compute.functionsRuntimes` | `[]` | Function runtimes (`_APP_FUNCTIONS_RUNTIMES`). Empty keeps the Docker installer's list (`node-16.0`, `node-22`, `php-8.0`, `python-3.9`, `ruby-3.0`). | | `compute.sitesRuntimes` | `[]` | Site runtimes (`_APP_SITES_RUNTIMES`). Empty keeps Appwrite's default list. | | `compute.executor.nodeSelector`, `tolerations` | inherit `defaults` | Where executor pods run. | @@ -498,9 +506,9 @@ compute: The same applies to the executor: add the same ranges to `compute.executor.networkPolicy.deniedCIDRs`. -The build's sidecar uploads the output straight to S3. When -`storage.s3.endpoint` is an IP address, the chart allows it; when -it's a hostname on a private network, add a rule (and the same rule under +The build's sidecar uploads the output straight to S3. The chart allows the +built-in object store, and an external `storage.s3.external.endpoint` that +is an IP address; when it's a hostname on a private network, add a rule (and the same rule under `compute.executor.networkPolicy.extraEgress`, which reads build output from S3): diff --git a/docs/external-databases.md b/docs/external-databases.md index 4797449..8c826ff 100644 --- a/docs/external-databases.md +++ b/docs/external-databases.md @@ -90,7 +90,6 @@ set `secretKey` when yours differ: | `smtp.password` | `smtp-password` | | `documentsDb.password` | `documentsdb-password` | | `vectorsDb.password` | `vectorsdb-password` | -| `assistant.apiKey` | `openai-api-key` | `secretName` wins over `value`. Secrets from External Secrets, Sealed Secrets or SOPS work the same way ([gitops.md](gitops.md#credentials)). @@ -142,7 +141,7 @@ Set any that differ on yours. CREATE EXTENSION IF NOT EXISTS pg_trgm; ``` -- **Extensions.** Appwrite 2.3.0 uses three, which Appwrite's own image +- **Extensions.** Appwrite 2.5.0 uses three, which Appwrite's own image (`appwrite/postgres`, PostgreSQL 18) ships: | Extension | Used for | @@ -310,7 +309,7 @@ database: - **One host, no TLS.** Appwrite connects to a single host with a username and password. It doesn't take connection strings, so SRV records - (`mongodb+srv://`) and TLS aren't supported in 2.3.0. That rules out + (`mongodb+srv://`) and TLS aren't supported in 2.5.0. That rules out MongoDB Atlas. ## Connections @@ -397,7 +396,7 @@ redis: ### Requirements -- **One endpoint, no TLS.** Appwrite 2.3.0 connects to a single Redis +- **One endpoint, no TLS.** Appwrite 2.5.0 connects to a single Redis endpoint in plain text. TLS, Sentinel and Redis Cluster aren't supported. Valkey works as a drop-in replacement. - **Memory that never runs out.** Redis holds the job queues as well as the diff --git a/docs/getting-started.md b/docs/getting-started.md index 5dfa59b..c785c9e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -12,12 +12,6 @@ Both use the chart's built-in PostgreSQL, Redis and ClickHouse; you can [switch to your own](external-databases.md) at any point. Functions and Sites need extra infrastructure and come [afterwards](#functions-and-sites). -> [!NOTE] -> Until the first release is on `ghcr.io`, the -> `oci://ghcr.io/appwrite/charts/appwrite` commands below fail because the -> chart isn't there yet. Clone the repository and use `./charts/appwrite` -> instead ([installing from source](#installing-from-source)). - ## What you need - Kubernetes 1.29 or later, with a default StorageClass @@ -95,8 +89,8 @@ sudo kubectl --kubeconfig "${KUBECONFIG:-$HOME/.kube/config}" \ -n traefik port-forward svc/traefik 80:80 ``` -Functions and Sites work locally too once you add S3 storage and -`compute.enabled` ([below](#functions-and-sites)), with two limits: previews +Functions and Sites work locally too: they're on by default +([below](#functions-and-sites)), with two limits: previews at `*.functions.appwrite.localhost` open in Chrome and Firefox, which resolve `*.localhost` themselves, but not in every browser; and function code can't call Appwrite back at `appwrite.localhost`, which inside a runtime is the @@ -131,7 +125,7 @@ These are hard to change once Appwrite holds data: | **The encryption key** | The chart generates `_APP_OPENSSL_KEY_V1` once. Without it, a database backup can't be read. Back it up after installing ([step 5](#5-back-up-the-encryption-key)), or create it yourself and pass `appwrite.secrets.existingSecret`. | [Bringing your own Secrets](security.md#bringing-your-own-secrets) | | **`appwrite.domain`** | It ends up in OAuth redirect URLs you register with providers, Git webhooks, the CNAME target of custom domains, and the function and site preview domains stored in the database. Changing it breaks all of them. | [Custom domains](networking.md#custom-domains) | | **Database engine** | PostgreSQL, built-in or external, is recommended and the one CI tests. Moving to another engine later is a data migration, not a values change. | [External databases](external-databases.md) | -| **Storage device** | `storage.device` is `local` (the `/storage` volume) or `s3`. Files stay where they were written, so switching later leaves them behind. Functions and Sites need `s3`. | [Storage](storage.md) | +| **Storage device** | `storage.device` is `s3` (by default the built-in object storage) or `local` (the `/storage` volume). Files stay where they were written, so switching later leaves them behind. Functions and Sites need `s3`. | [Storage](storage.md) | | **`/storage` access mode** | ReadWriteOnce keeps every pod that mounts `/storage` on one node; ReadWriteMany lets them spread. A claim's access mode can't be changed, only replaced with a new claim. | [ReadWriteOnce vs ReadWriteMany](storage.md#readwriteonce-vs-readwritemany) | ### 1. Write a values file @@ -225,24 +219,33 @@ kubectl -n appwrite get secret appwrite -o jsonpath='{.data._APP_OPENSSL_KEY_V1} ## Functions and Sites -Functions and Sites are off by default. They need: +Functions and Sites are on by default. A plain install runs: + +- **The built-in object storage**, through which builds and the executor + exchange code and build output ([storage.md](storage.md#built-in-object-storage)). +- **Builds** as ordinary Kubernetes Jobs, in the namespace `-builds`. +- **Executions** on the open-runtimes executor, in the namespace + `-executor`. It starts runtime containers through a Docker daemon + of its own, which runs as a privileged pod: root on its node. + +That's enough to try them. Before real users depend on them, give the +executor nodes of its own, or run the daemon without privileges through +Sysbox ([executor.md](executor.md)); move files and build output to an S3 +store with redundancy ([storage.md](storage.md#your-own-s3-store)); and set +up wildcard DNS and certificates for previews, at `*.functions.` and +`*.sites.`. -- **S3 storage.** Builds and the executor exchange code and build output - through a bucket ([storage.md](storage.md#s3)). -- **A privileged pod.** Builds run as ordinary Kubernetes Jobs, but - executions run on the open-runtimes executor, which starts runtime - containers through a Docker daemon of its own. That daemon is privileged, - root on its node: you accept it with `compute.executor.privileged` and give - it nodes of its own. -- **Wildcard DNS and certificates** for previews, at `*.functions.` - and `*.sites.`. +Where privileged pods aren't allowed (GKE Autopilot, OpenShift's default +policies, policy engines that forbid them), set `compute.enabled: false`. +The console still shows Functions and Sites, but creating one then fails +([troubleshooting.md](troubleshooting.md#deployments-fail-with-an-internal-error)). -[executor.md](executor.md) walks through all of it. Two example files have -it ready; add one to your install with another `-f`: +Two example files set up a production executor, on nodes of its own with +your own S3 store; add one to your install with another `-f`: | Your other values | Add | | --- | --- | -| `minimal.yaml`, or your own file without S3 | [`examples/functions.yaml`](../examples/functions.yaml): S3 with the keys inline, and everything else generated by the chart | +| `minimal.yaml`, or your own file without S3 | [`examples/functions.yaml`](../examples/functions.yaml): your S3 store with the keys inline, and everything else generated by the chart | | S3 keys in a Secret you manage: `production.yaml`, `external-services.yaml`, `aws.yaml`, `gcp.yaml` | [`examples/functions-existing-secrets.yaml`](../examples/functions-existing-secrets.yaml): the Secrets the executor needs in its own namespace, which you create first | ```console diff --git a/docs/gitops.md b/docs/gitops.md index b91ba78..dbe7d42 100644 --- a/docs/gitops.md +++ b/docs/gitops.md @@ -88,7 +88,6 @@ change it. | `storage.s3.accessKeyId.secretName`, `storage.s3.secretAccessKey.secretName` | `s3-access-key`, `s3-secret-key` (their `secretKey`) | S3 storage | | `documentsDb.password.secretName`, `vectorsDb.password.secretName` | `documentsdb-password`, `vectorsdb-password` (their `secretKey`) | DocumentsDB or VectorsDB with a password other than the main database's | | `smtp.password.secretName` | `smtp-password` (`smtp.password.secretKey`) | SMTP with a password | -| `assistant.apiKey.secretName` | `openai-api-key` (`assistant.apiKey.secretKey`) | `assistant.enabled` | The executor runs in a namespace of its own and can't read Secrets in the release namespace, so it needs its own copy of `executor-secret` and the S3 diff --git a/docs/integrations.md b/docs/integrations.md index 576bc8d..522ef73 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -12,7 +12,6 @@ sign-in, Sentry for errors. This guide shows how to connect each one. | [SMS](#sms) | phone sign-in and SMS verification | `_APP_SMS_*` | | [Error tracking](#error-tracking) | sending server errors to Sentry | `_APP_LOGGING_CONFIG` | | [Antivirus](storage.md#antivirus) | scanning uploads with ClamAV | `_APP_STORAGE_ANTIVIRUS*` | -| [Assistant](optional-components.md#assistant) | the console's AI assistant (OpenAI) | `assistant` | Some settings live in the Appwrite console, per project, not in the chart: sign-in providers for your apps' users (OAuth2), Messaging providers @@ -217,7 +216,8 @@ appwrite: Upgrade, then open a function or site in the console and choose *Connect Git*. If GitHub shows a delivery error under the app's *Advanced* tab, check that `https:///v1/vcs/github/events` is reachable from outside and -that the webhook secret matches. +that the webhook secret matches. The secret is required: Appwrite 2.4.0 and +later refuse GitHub's events without `_APP_VCS_GITHUB_WEBHOOK_SECRET`. ### GitLab, Gitea and Bitbucket @@ -309,7 +309,7 @@ appwrite: _APP_LOGGING_CONFIG: sentry://:@/ ``` -Sentry is the only provider Appwrite 2.3.0 supports. When the DSN is +Sentry is the only provider Appwrite 2.5.0 supports. When the DSN is invalid, the pods log `Invalid _APP_LOGGING_CONFIG, error reporting is disabled` at start-up. Logs go to stdout regardless, as JSON by default (`appwrite.logFormat`), ready for your log pipeline. diff --git a/docs/migrating-from-docker.md b/docs/migrating-from-docker.md index 0f55b48..128db20 100644 --- a/docs/migrating-from-docker.md +++ b/docs/migrating-from-docker.md @@ -5,8 +5,8 @@ keeping all data. Before you start: -- **Same Appwrite version.** The chart deploys Appwrite 2.3.0. Upgrade the - Docker install to 2.3.0 first, including its data migration. A Docker +- **Same Appwrite version.** The chart deploys Appwrite 2.5.0. Upgrade the + Docker install to 2.5.0 first, including its data migration. A Docker install on 2.2.0 can also move as it is: install the chart with `--set image.tag=2.2.0` in step 3 and drop it in step 5, which then runs the migration; give that upgrade a `--timeout` long enough for it @@ -72,8 +72,8 @@ only carry over what you changed. Most variables have a value of their own: | `_APP_SYSTEM_EMAIL_NAME`, `_APP_SYSTEM_EMAIL_ADDRESS`, `_APP_EMAIL_SECURITY` | `appwrite.system.emailName`, `.emailAddress`, `.securityEmailAddress` | | `_APP_SMTP_HOST`, `_PORT`, `_SECURE`, `_USERNAME` | `smtp.host`, `.port`, `.secure`, `.username` (defaults `587` and `tls`) | | `_APP_SMTP_PASSWORD` | `smtp.password.value`, or `smtp.password.secretName` | -| `_APP_STORAGE_DEVICE` | `storage.device`: `local` or `s3` | -| `_APP_STORAGE_S3_ENDPOINT`, `_REGION`, `_BUCKET` | `storage.s3.endpoint`, `.region`, `.bucket` | +| `_APP_STORAGE_DEVICE` | `storage.device`: `s3` (the chart's default) or `local`, the Docker default: set it explicitly, see below | +| `_APP_STORAGE_S3_ENDPOINT`, `_REGION`, `_BUCKET` | `storage.s3.mode: external` with `storage.s3.external.endpoint`, `.region`, and `storage.s3.bucket` | | `_APP_STORAGE_S3_ACCESS_KEY`, `_APP_STORAGE_S3_SECRET` | `storage.s3.accessKeyId`, `storage.s3.secretAccessKey`, each with a `value` or a `secretName` | | `_APP_STORAGE_LIMIT`, `_APP_STORAGE_PREVIEW_LIMIT` | `storage.limit`, `.previewLimit` | | `_APP_STORAGE_ANTIVIRUS*` | `appwrite.config` ([storage.md](storage.md#antivirus)) | @@ -81,7 +81,6 @@ only carry over what you changed. Most variables have a value of their own: | `_APP_DB_SCHEMA` | `database.name`, built-in or external. Keep the same name (default `appwrite`): it's also the name of the schema inside the dump. | | `_APP_FUNCTIONS_RUNTIMES`, `_APP_SITES_RUNTIMES` | `compute.functionsRuntimes`, `compute.sitesRuntimes` (lists) | | `_APP_DOCKER_HUB_USERNAME`, `_APP_DOCKER_HUB_PASSWORD` | `OPR_EXECUTOR_DOCKER_HUB_USERNAME` and `_PASSWORD` in `compute.executor.extraEnv` ([executor.md](executor.md#registries)) | -| `_APP_ASSISTANT_OPENAI_API_KEY` | `assistant.enabled` and `assistant.apiKey` ([optional-components.md](optional-components.md#assistant)) | | `_APP_DOCUMENTSDB`, `_APP_VECTORSDB` and their `_APP_DB_*` variables | `documentsDb.*`, `vectorsDb.*` ([external-databases.md](external-databases.md#optional-databases)) | | OAuth, Git (`_APP_VCS_*`), SMS and other secrets, `_APP_LOGGING_CONFIG` | `appwrite.secretConfig` or `appwrite.extraEnvFrom` ([integrations.md](integrations.md)) | | other settings you changed: retention, GraphQL limits, function timeouts and sizes, ... | `appwrite.config` | @@ -113,12 +112,16 @@ as `s3` with the provider's endpoint ([storage.md](storage.md#s3)). We haven't tested that existing files stay readable after the switch; check before you rely on it. -**Functions and Sites need S3.** The chart's compute plane exchanges code and -build output through a bucket and refuses `compute.enabled` with `local` -storage. A Docker install on local storage can move its files as described -below, but its functions and sites won't build or run in the chart until -storage moves to S3. That move is a migration of its own (the paths stored -in the database change with the device), so plan it separately. +**Keep the storage device.** The chart stores files in its built-in object +storage by default, while a Docker install keeps them on its volume +(`_APP_STORAGE_DEVICE=local`). Set `storage.device: local` to carry those +files over as described below, and with it `compute.enabled: false`: the +chart's compute plane exchanges code and build output through a bucket and +refuses `local` storage, so functions and sites won't build or run until +storage moves to S3. That move is a migration of its own (the paths stored in +the database change with the device), so plan it separately. A Docker install +already on S3 keeps its bucket (`storage.s3.mode: external`) and Functions +and Sites. ## 2. Stop the Docker install and dump the data diff --git a/docs/networking.md b/docs/networking.md index 2a8a9fc..bc928c5 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -169,8 +169,44 @@ redirect HTTP to HTTPS at the edge for unknown hostnames, keeping `/.well-known/acme-challenge/` on HTTP for custom-domain certificates. Appwrite takes the client IP from the headers in `_APP_TRUSTED_HEADERS` -(default `x-forwarded-for`). Make sure your controller sets it, and that -clients can't spoof it. +(default `x-forwarded-for`), but only on connections from a proxy in +`_APP_TRUSTED_PROXIES`, and then the rightmost address in the header that +isn't itself a trusted proxy. The default trusts loopback, the RFC 1918 +ranges and `100.64.0.0/10`, which covers the pod network of most clusters, so +the ingress controller or Gateway in front of Appwrite is trusted as it is. +Set your own list through `appwrite.config` when pods get other addresses +(IPv6 or dual-stack clusters), or when a load balancer or CDN in front of the +controller adds its own address to the header: + +```yaml +appwrite: + config: + _APP_TRUSTED_PROXIES: "10.0.0.0/8,fd00::/8" # replaces the default +``` + +Make sure your controller sets the header, and that clients can't spoof it. + +### Requests to private addresses + +Appwrite refuses to fetch a URL that resolves to a private or reserved +address: OAuth2 and OIDC providers, webhooks, messaging webhooks, migration +sources and avatar URLs. That includes every pod and Service in the cluster. +To use an identity provider or webhook receiver that runs in the cluster, +list its addresses (IPs or CIDRs, comma-separated) in +`_APP_ALLOWED_INTERNAL_ADDRESSES`, for example your Service range: + +```yaml +appwrite: + config: + _APP_ALLOWED_INTERNAL_ADDRESSES: "10.96.0.0/12" +``` + +Migrations from this Appwrite itself are allowed already +(`_APP_MIGRATION_HOST`). Before fetching a migration source or an avatar +URL, Appwrite also checks its host through the resolvers in +`_APP_DNS_EXTERNAL` (by default `_APP_DNS`, `8.8.8.8`) rather than the +cluster's DNS. With `networkPolicy.egress.enabled`, that resolver needs the +same egress rule as custom domains below. ### cert-manager @@ -244,9 +280,11 @@ httpRoute: ``` Step 1 also needs DNS: Appwrite resolves the domain through `_APP_DNS` -(default `8.8.8.8`). With `networkPolicy.egress.enabled`, allow that resolver -on port 53 in `networkPolicy.egress.extraRules`, or point `_APP_DNS` at a -resolver the policy already allows (through `appwrite.config`): +(default `8.8.8.8`), and the hosts of migration sources and avatar URLs +through `_APP_DNS_EXTERNAL` (by default the same). With +`networkPolicy.egress.enabled`, allow that resolver on port 53 in +`networkPolicy.egress.extraRules`, or point both at a resolver the policy +already allows (through `appwrite.config`): ```yaml networkPolicy: diff --git a/docs/optional-components.md b/docs/optional-components.md index 0cf60c8..2007f64 100644 --- a/docs/optional-components.md +++ b/docs/optional-components.md @@ -7,7 +7,6 @@ each of them. Geo is on by default; the others are off until you enable them. | --- | --- | --- | --- | | [Geo](#geo) | `geo.enabled` | on | Country and location for sessions and logs. | | [Browser](#browser) | `browser.enabled` | off | Site screenshots and `/v1/avatars/screenshots`. | -| [Assistant](#assistant) | `assistant.enabled` | off | The console's AI assistant. Needs an OpenAI API key. | | [Autogravity](#autogravity) | `autogravity.enabled` | off | Image previews cropped around the subject (`gravity=auto`). | | [Embedding](#embedding) | `embedding.enabled` | off | The embeddings API, which turns text into vectors. | | [MQTT](#mqtt) | `mqtt.enabled` | off | Push notifications over MQTT, through Messaging's Appwrite provider. | @@ -53,34 +52,6 @@ It runs as UID 999 with a read-only root filesystem, a writable `/tmp`, and a 256 MiB in-memory `/dev/shm` for Chromium, which counts towards its memory limit. Scale it with `browser.replicaCount`. -## Assistant - -The AI assistant in the console. It answers through OpenAI's API with your -key, so prompts leave the cluster. - -```console -kubectl -n appwrite create secret generic openai \ - --from-literal=openai-api-key='sk-...' -``` - -```yaml -assistant: - enabled: true - apiKey: - secretName: openai # key openai-api-key; or value: sk-... inline -``` - -The chart refuses to install without a key. The key is also passed to every -Appwrite pod (`_APP_ASSISTANT_OPENAI_API_KEY`), which is how the console -knows to show the assistant. - -Appwrite reaches the assistant at the fixed address -`http://appwrite-assistant:3003`, so its Service is named -`appwrite-assistant` regardless of the release name. Only one release per -namespace can enable it. It runs as a single pod. With -`networkPolicy.egress.enabled`, allow it to reach `api.openai.com` on port -443 in `networkPolicy.egress.extraRules`. - ## Autogravity Finds the subject of an image, so that previews requested with @@ -166,9 +137,8 @@ with `mqtt.replicaCount`. `_APP_MQTT_REPLAY_DEPTH` (5) and | Component | Image | Requests | Memory limit | Replicas | | --- | --- | --- | --- | --- | -| Geo | `docker.io/appwrite/geo:0.3.1` | 20m CPU, 128Mi | 512Mi | `geo.replicaCount` (1) | -| Browser | `docker.io/appwrite/browser:0.3.4` | 100m CPU, 512Mi | 2Gi | `browser.replicaCount` (1) | -| Assistant | `docker.io/appwrite/assistant:0.8.4` | 20m CPU, 128Mi | 512Mi | 1 | +| Geo | `docker.io/appwrite/geo:0.3.2` | 20m CPU, 128Mi | 512Mi | `geo.replicaCount` (1) | +| Browser | `docker.io/appwrite/browser:0.3.6` | 100m CPU, 512Mi | 2Gi | `browser.replicaCount` (1) | | Autogravity | `ghcr.io/appwrite/autogravity:0.0.11` | 100m CPU, 512Mi | 2Gi | 1 | | Embedding | `docker.io/appwrite/embedding:0.3.1` | 250m CPU, 1Gi | 4Gi | 1 | | MQTT | the Appwrite image | 50m CPU, 128Mi | 512Mi | `mqtt.replicaCount` (1) | diff --git a/docs/production.md b/docs/production.md index 5af259c..f262cff 100644 --- a/docs/production.md +++ b/docs/production.md @@ -70,10 +70,15 @@ to the guide that explains it. none, so nothing limits how many of them a drain evicts at once. Drain one node at a time ([architecture.md](architecture.md#probes-and-disruption-budgets)). -- [ ] **Functions and Sites:** the executor on nodes of its own - ([executor.md](executor.md#executor-isolation)), with replicas and +- [ ] **Functions and Sites** (on by default): the executor on nodes of its + own ([executor.md](executor.md#executor-isolation)), with replicas and resources for the executions you expect - ([executor.md](executor.md#sizing)). + ([executor.md](executor.md#sizing)); or `compute.enabled: false` if you + don't use them, which removes the privileged executor pod. +- [ ] **Object storage with redundancy:** your own S3 store + ([storage.md](storage.md#your-own-s3-store)), or backups of the + built-in store's volume, which is a single replica + ([storage.md](storage.md#built-in-object-storage)). ## Operations @@ -97,7 +102,7 @@ to the guide that explains it. ## Monitoring -Appwrite 2.3.0 exposes no Prometheus metrics. You have its logs, its health +Appwrite 2.5.0 exposes no Prometheus metrics. You have its logs, its health API, and the metrics of Kubernetes and your datastores. | Watch | How | Alert when | diff --git a/docs/security.md b/docs/security.md index de0e560..aae277e 100644 --- a/docs/security.md +++ b/docs/security.md @@ -33,9 +33,9 @@ What that means for each pod: | Geo | `www-data` (82) | Its port 80 is hard-coded, so the pod sets the namespaced sysctl `net.ipv4.ip_unprivileged_port_start=0`, which `restricted` allows. | | PostgreSQL | `postgres` (999) | `/var/run/postgresql` is an `emptyDir`. | | Redis | `redis` (999) | | +| SeaweedFS | 1000 | Runs as root in its own image; the chart sets UID 1000. The S3 users file is written to `/tmp` from the chart's Secret on start. | | ClickHouse | `clickhouse` (101) | Users and logging come from mounted files, because the image's entrypoint would otherwise write them to `/etc`. Logs go to stdout. | | Browser | 999 | `/dev/shm` is a memory `emptyDir`. | -| Assistant | 1000 | | | Autogravity | 65532 | | | Embedding | 10001 | Models are kept on their own PersistentVolumeClaim. | | Orchestrator | `nonroot` (65532) | The only pod with a ServiceAccount token, for its Role in the build namespace. | @@ -109,11 +109,10 @@ The chart keeps credentials in a Secret named after the release: | `clickhouse-password` | `clickhouse.password` | ClickHouse, built-in or external. | | `documentsdb-password`, `vectorsdb-password` | `documentsDb.password`, `vectorsDb.password` | DocumentsDB and VectorsDB, when they have a password of their own. | | `executor-secret`, `orchestrator-secret` | `compute.secrets.executorSecret`, `orchestratorSecret` | The executor and the orchestrator (`compute.enabled`). The in-cluster executor gets a copy of `executor-secret` and the S3 keys in a Secret in its namespace. | -| `s3-access-key`, `s3-secret-key` | `storage.s3.accessKeyId`, `storage.s3.secretAccessKey` | S3 storage. | +| `s3-access-key`, `s3-secret-key` | `storage.s3.accessKeyId`, `storage.s3.secretAccessKey` | S3 storage, built-in or external. Generated once for the built-in store. | | `smtp-password` | `smtp.password` | SMTP. | -| `openai-api-key` | `assistant.apiKey` | The assistant. | -The datastore, S3, SMTP and assistant credentials all take the same three +The datastore, S3 and SMTP credentials all take the same three fields: `value` sets the credential inline and stores it in this Secret; `secretName` reads it from a Secret you manage instead, under `secretKey` (the key in the table by default), and leaves it out of this one. These are @@ -170,7 +169,7 @@ clickhouse: ``` Key names are configurable (`appwrite.secrets.keys.*`, and `secretKey` next -to each `secretName`). External databases, S3, SMTP and the assistant take a +to each `secretName`). External databases, S3 and SMTP take a `secretName` the same way ([external-databases.md](external-databases.md#credentials) lists their keys), and integrations can read a Secret through `appwrite.extraEnvFrom` @@ -196,7 +195,7 @@ kubectl -n appwrite patch secret appwrite --type merge \ `helm upgrade` then restarts every pod that reads a changed value of the chart's Secret: the Appwrite pods (API, realtime, workers, tasks, MQTT), the -orchestrator, the executor, and geo, Redis, ClickHouse and the assistant, +orchestrator, the executor, and geo, Redis and ClickHouse, which each carry a checksum of just the keys they read. PostgreSQL doesn't need a restart. @@ -266,8 +265,9 @@ which carry the label `appwrite/datastore-client: "true"`. | --- | --- | --- | | `-edge` | API, console, realtime | `networkPolicy.ingress.from` on the HTTP port (any source when empty); pods of the release on any port | | `-builds-to-api` | API (`compute.enabled`) | build pods, on the HTTP port | -| `-internal` | geo, browser, assistant, autogravity, embedding | pods of the release | +| `-internal` | geo, browser, autogravity, embedding | pods of the release | | `-datastores` | PostgreSQL, Redis, ClickHouse | Appwrite pods of the release; on the metrics ports (`database.builtin.metrics.enabled` and the same under `redis` and `clickhouse`), also `networkPolicy.metrics.from` | +| `-object-storage` | SeaweedFS (`storage.s3.mode: builtin` with `device: s3`) | Appwrite pods and the orchestrator of the release, build pods and the executor, on the S3 port | | `-background` | workers and tasks | nothing | | `-mqtt` | MQTT broker (`mqtt.enabled`) | `networkPolicy.ingress.from` (any source when empty), on the MQTT ports | | `-orchestrator` | orchestrator (`compute.enabled`) | pods of the release and build pods, on the HTTP port | @@ -289,8 +289,8 @@ in the other namespaces, whether or not `networkPolicy.enabled` is set: | Policy | Namespace | Allows in | Allows out | Toggle | | --- | --- | --- | --- | --- | -| `-build` | build | nothing | DNS, the API and orchestrator, the S3 endpoint, public addresses, `extraEgress` | `compute.builds.networkPolicy.enabled` | -| `-executor` | executor | Appwrite pods of the release, on port 9900 | DNS, the S3 endpoint, public addresses, `extraEgress` | `compute.executor.networkPolicy.enabled` | +| `-build` | build | nothing | DNS, the API and orchestrator, the built-in object store or the S3 endpoint, public addresses, `extraEgress` | `compute.builds.networkPolicy.enabled` | +| `-executor` | executor | Appwrite pods of the release, on port 9900 | DNS, the built-in object store or the S3 endpoint, public addresses, `extraEgress` | `compute.executor.networkPolicy.enabled` | Both are on by default. [executor.md](executor.md#isolation) explains which ranges count as public and how to add rules. diff --git a/docs/storage.md b/docs/storage.md index 5d2e589..c4acaea 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -4,19 +4,23 @@ Appwrite stores files (bucket uploads, deployment code, build output, certificates) either on a volume under `/storage` or in S3-compatible object storage. `storage.device` picks one: -| | `local` (default) | `s3` | +| | `s3` (default) | `local` | | --- | --- | --- | -| Files live on | a PersistentVolumeClaim | a bucket | -| Spreading the API and workers across nodes | needs a ReadWriteMany volume | still needs a small ReadWriteMany volume ([why](#the-volume-with-s3)) | -| [Functions and Sites](executor.md) | not supported | required | -| [Antivirus](#antivirus) scanning | supported | not supported | +| Files live on | a bucket: the [built-in object storage](#built-in-object-storage) or [your own](#your-own-s3-store) | a PersistentVolumeClaim | +| Spreading the API and workers across nodes | still needs a small ReadWriteMany volume ([why](#the-volume-with-s3)) | needs a ReadWriteMany volume | +| [Functions and Sites](executor.md) | required | not supported: set `compute.enabled: false` | +| [Antivirus](#antivirus) scanning | not supported | supported | Choose before going live: changing `device` later doesn't move files that are already stored. They stay where they were written. -## Local storage (default) +## Local storage + +Files on the `/storage` volume, without Functions and Sites: ```yaml +compute: + enabled: false storage: device: local volume: @@ -84,6 +88,43 @@ The chart then creates no claim of its own, and ignores `size` and Object storage takes files off the volume, lets pods share them without a ReadWriteMany claim, and is required for [Functions and Sites](executor.md). +`storage.s3.mode` picks where the bucket lives: in the release (`builtin`, +the default), or in a store you run or buy (`external`). + +### Built-in object storage + +```yaml +storage: + device: s3 +``` + +That's all. The chart runs [SeaweedFS](https://github.com/seaweedfs/seaweedfs), +an Apache-2.0 S3-compatible store, as a single-replica StatefulSet +(`-seaweedfs`) on a 20Gi volume +(`storage.s3.builtin.persistence.size`). It creates the bucket +(`storage.s3.bucket`, `appwrite` by default) when it starts, and the chart +generates its access keys once and keeps them in its Secret (`s3-access-key`, +`s3-secret-key`), like the other built-in credentials. Appwrite, the +orchestrator, build Jobs and the executor reach it at +`http://-seaweedfs..svc:8333`; it has no route from +outside the cluster. With `networkPolicy.enabled`, the +`-object-storage` policy lets only those pods in. + +It's one pod and one volume, without replication: back the volume up +([backup-restore.md](backup-restore.md)), or use your own store with +redundancy once real users depend on the install. The volume grows like the +[datastore volumes](#datastore-volumes) (claim `data--seaweedfs-0`). +SeaweedFS fills it with volume files of `storage.s3.builtin.volumeSizeLimitMB` +(1024 MB), so keep the claim at a few GB at least. + +### Your own S3 store + +```yaml +storage: + device: s3 + s3: + mode: external +``` **1. Create a bucket** and an access key that can read, write, list and delete objects in it. Appwrite doesn't create the bucket. On AWS, an IAM @@ -121,8 +162,10 @@ kubectl -n appwrite create secret generic appwrite-s3 \ storage: device: s3 s3: - endpoint: https://fsn1.your-objectstorage.com # empty for AWS S3 - region: fsn1 # default us-east-1 + mode: external + external: + endpoint: https://fsn1.your-objectstorage.com # empty for AWS S3 + region: fsn1 # default us-east-1 bucket: appwrite accessKeyId: secretName: appwrite-s3 # or value: ... inline @@ -207,12 +250,13 @@ Your ingress controller's request body limit matters too ## Datastore volumes -The built-in PostgreSQL, Redis and ClickHouse each keep their data on a claim +The built-in PostgreSQL, Redis, ClickHouse and SeaweedFS each keep their data on a claim created from their StatefulSet's `volumeClaimTemplates`, named `data---0` (for example `data-appwrite-postgresql-0`). Kubernetes doesn't allow changing a StatefulSet's claim templates, so changing -`.builtin.persistence.size` (`database`, `redis` or `clickhouse`) -after installing makes `helm upgrade` fail. +`.builtin.persistence.size` (`database`, `redis` or `clickhouse`, +and `storage.s3.builtin.persistence.size`) after installing makes +`helm upgrade` fail. To grow a volume, with a StorageClass that has `allowVolumeExpansion: true`: @@ -249,7 +293,7 @@ appwrite: ``` Run ClamAV separately (for example with its official image as a Deployment). -Appwrite 2.3.0 scans only: +Appwrite 2.5.0 scans only: - with `device: local`. Files stored in S3 aren't scanned. - files up to 20 MB. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 941305b..52db768 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -206,7 +206,7 @@ The server requires TLS, which Appwrite's MySQL client doesn't use. Turn off ### Redis: connection resets or timeouts against a managed Redis The service expects TLS (ElastiCache with in-transit encryption, ElastiCache -Serverless, Azure's TLS port, Upstash). Appwrite 2.3.0 connects in plain text +Serverless, Azure's TLS port, Upstash). Appwrite 2.5.0 connects in plain text only; use a plain-text endpoint on a private network ([Redis](external-databases.md#redis)). @@ -320,6 +320,24 @@ Deployments that stay `waiting`, builds that fail, executor pods that don't start, executions that time out: see [executor.md](executor.md#troubleshooting). +### Deployments fail with an internal error + +Every deployment fails with "An internal error occurred while building", and +creating a function or site from a template fails with a 500. Retrying then +fails with a 409, "Domain is already used", because the first attempt already +created the function and its domain. The API's log names the cause: + +```console +kubectl -n appwrite logs deploy/appwrite-api | grep -i 'resolve host' +``` + +`Could not resolve host: orchestrator` (or `exc1`) means Functions and Sites +are off (`compute.enabled: false`): Appwrite falls back to the Docker +install's hostnames, which don't exist in the cluster, and the console offers +Functions and Sites anyway. Turn compute on ([executor.md](executor.md)), or +hide Functions and Sites in each project's settings (Settings → Services). +Delete the functions the failed attempts left behind. + ## FAQ ### Is it highly available? @@ -340,11 +358,12 @@ failover. For high availability, use managed services or operators The open-runtimes executor runs each function and site in a container of its own, started by a Docker daemon inside the executor pod. That daemon needs a -privileged container, which is root on its node. Functions and Sites are off -by default, and the chart only runs the executor once you accept that with -`compute.executor.privileged: true`. Give it nodes of its own -([executor.md](executor.md#executor-isolation)). Builds don't need it: they -run as ordinary Jobs. +privileged container, which is root on its node. Functions and Sites are on +by default, so a plain install runs one, in a namespace of its own. For +production, give it nodes of its own +([executor.md](executor.md#executor-isolation)); where privileged pods aren't +allowed, set `compute.enabled: false`. Builds don't need it: they run as +ordinary Jobs. The alternative is [Sysbox](https://github.com/nestybox/sysbox), which runs the daemon without `privileged`: install it on the executor's nodes and set @@ -354,7 +373,7 @@ it. ### Does it expose Prometheus metrics? -Appwrite 2.3.0 has no metrics endpoint. Watch its health API, its queues and +Appwrite 2.5.0 has no metrics endpoint. Watch its health API, its queues and its logs, plus the metrics of Kubernetes and the datastores. The built-in datastores can run exporters (`database.builtin.metrics`, `redis.builtin.metrics`, `clickhouse.builtin.metrics`) with ServiceMonitors @@ -372,8 +391,9 @@ see [upgrading.md](upgrading.md#helm-3-to-helm-4). Appwrite's Docker Compose install runs everything on one server and is the simpler choice there. The chart runs the same images on Kubernetes: use it when you already run Kubernetes and want managed datastores, replicas across -nodes, rolling upgrades or GitOps. Functions and Sites need more here: S3 -storage and a privileged executor ([executor.md](executor.md)). To move an +nodes, rolling upgrades or GitOps. Functions and Sites run on a privileged +executor here too, and on object storage, built-in or your own +([executor.md](executor.md)). To move an existing Docker install, see [migrating-from-docker.md](migrating-from-docker.md). diff --git a/docs/upgrading.md b/docs/upgrading.md index 4d4bac3..c1563ef 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -14,9 +14,8 @@ The [changelog](../CHANGELOG.md) lists what changed in each chart version, and [version notes](#version-notes) what you have to do about it. What CI tests: a fresh install of the `appVersion`, and an upgrade from the -previous Appwrite release (2.2.0, installed with this chart through -`image.tag`) to the `appVersion`, including the data migration. Upgrades from -an earlier chart version aren't tested yet. +published chart 1.0.0 (Appwrite 2.3.0) with default values, including the +data migration. ## What happens during an upgrade @@ -52,11 +51,11 @@ the Job as soon as the manifests are applied. for breaking API changes your apps depend on. 4. **Don't skip a data migration.** `migrate` applies only the migration of the version it runs and doesn't replay those of the versions in between. - Several releases share a migration: every version from 1.9.6 to 2.3.0 - uses the same one, so 2.0 → 2.3 is a single step. 1.8 → 2.3 isn't: go to - a release from 1.9.0 to 1.9.5 first, whose migration 2.3 doesn't repeat. + Several releases share a migration: every version from 1.9.6 to 2.5.0 + uses the same one, so 2.0 → 2.5 is a single step. 1.8 → 2.5 isn't: go to + a release from 1.9.0 to 1.9.5 first, whose migration 2.5 doesn't repeat. The table is in Appwrite's - [`src/Appwrite/Migration/Migration.php`](https://github.com/appwrite/appwrite/blob/2.3.0/src/Appwrite/Migration/Migration.php). + [`src/Appwrite/Migration/Migration.php`](https://github.com/appwrite/appwrite/blob/2.5.0/src/Appwrite/Migration/Migration.php). ```console helm upgrade appwrite oci://ghcr.io/appwrite/charts/appwrite \ diff --git a/examples/aws.yaml b/examples/aws.yaml index 9ec9e53..114e6c2 100644 --- a/examples/aws.yaml +++ b/examples/aws.yaml @@ -15,7 +15,7 @@ # CREATE EXTENSION postgis; CREATE EXTENSION vector; CREATE EXTENSION pg_trgm; # rds.force_ssl can stay on: Appwrite connects with TLS when offered. # - ElastiCache for Valkey or Redis OSS, node-based, cluster mode disabled, -# in-transit encryption off (Appwrite 2.3.0 has no Redis TLS). ElastiCache +# in-transit encryption off (Appwrite 2.5.0 has no Redis TLS). ElastiCache # only allows an AUTH token with in-transit encryption, so there's no # password: restrict it with a security group. # - An S3 bucket and an IAM user with an access key for it. Appwrite doesn't @@ -69,8 +69,10 @@ clickhouse: storage: device: s3 s3: - endpoint: "" # empty: AWS S3 - region: eu-west-1 + mode: external + external: + endpoint: "" # empty: AWS S3 + region: eu-west-1 bucket: appwrite-example-com accessKeyId: secretName: appwrite-external @@ -81,6 +83,12 @@ storage: accessModes: [ReadWriteMany] # EFS lets the API and workers spread across nodes storageClass: efs-sc +# The executor runs in a namespace of its own and can't read the S3 Secret +# above, so Functions and Sites stay off here: add +# functions-existing-secrets.yaml to turn them on. +compute: + enabled: false + smtp: host: email-smtp.eu-west-1.amazonaws.com port: 587 diff --git a/examples/azure.yaml b/examples/azure.yaml index 1a7f913..2e8e0b1 100644 --- a/examples/azure.yaml +++ b/examples/azure.yaml @@ -9,9 +9,10 @@ # # Files are stored on the Azure Files volume, not in object storage: Azure # Blob Storage has no S3 API, and Appwrite only speaks S3. Functions and -# Sites need S3, so for them set storage.device: s3 with another -# S3-compatible store (docs/storage.md#s3), then add -# functions-existing-secrets.yaml. A translating proxy such as S3Proxy can put +# Sites need S3, so this file turns them off. For them, drop storage.device +# and compute.enabled below to use the chart's built-in object storage, or +# set storage.device: s3 with another S3-compatible store +# (docs/storage.md#s3), then add functions-existing-secrets.yaml. A translating proxy such as S3Proxy can put # an S3 API in front of Blob Storage; it isn't tested with Appwrite. MinIO's # gateway mode, which did the same, was removed in 2022. Decide before going # live: switching the device later doesn't move stored files. @@ -27,7 +28,7 @@ # require_secure_transport can stay on: Appwrite connects with TLS when # offered. # - Azure Cache for Redis, Standard tier or higher for a replica, with the -# non-TLS port (6379) turned on: it's off by default, and Appwrite 2.3.0 has +# non-TLS port (6379) turned on: it's off by default, and Appwrite 2.5.0 has # no Redis TLS. An access key is the password. Keep it on a private # endpoint. Azure Cache for Redis retires on 30 September 2028; for Azure # Managed Redis, choose the Non-clustered policy, turn on Non-TLS access and @@ -82,6 +83,9 @@ clickhouse: external: url: https://abc123.germanywestcentral.azure.clickhouse.cloud:8443 +compute: + enabled: false # needs S3: see the top of this file + storage: device: local # Azure Files below; S3 for Functions and Sites volume: diff --git a/examples/external-services.yaml b/examples/external-services.yaml index 8234e9f..928e2c5 100644 --- a/examples/external-services.yaml +++ b/examples/external-services.yaml @@ -68,8 +68,10 @@ clickhouse: storage: device: s3 s3: - endpoint: https://s3.example.com - region: us-east-1 + mode: external + external: + endpoint: https://s3.example.com + region: us-east-1 bucket: appwrite accessKeyId: secretName: appwrite-external @@ -79,6 +81,12 @@ storage: volume: size: 5Gi +# The executor runs in a namespace of its own and can't read the S3 Secret +# above, so Functions and Sites stay off here: add +# functions-existing-secrets.yaml to turn them on. +compute: + enabled: false + smtp: host: smtp.example.com port: 587 diff --git a/examples/functions.yaml b/examples/functions.yaml index c2cca31..4fafd29 100644 --- a/examples/functions.yaml +++ b/examples/functions.yaml @@ -1,5 +1,7 @@ -# Functions and Sites on top of minimal.yaml, with the S3 keys in this file -# and every other secret generated by the chart. When your S3 keys are in a +# Functions and Sites for production on top of minimal.yaml: your own S3 +# store instead of the built-in object storage, and the executor on nodes of +# its own. Functions and Sites are on by default; this file hardens them. The +# S3 keys are in this file and every other secret is generated by the chart. When your S3 keys are in a # Secret you manage (production.yaml, external-services.yaml, aws.yaml), use # functions-existing-secrets.yaml instead. # @@ -29,8 +31,10 @@ storage: device: s3 s3: - endpoint: https://s3.example.com # empty for AWS S3 - region: us-east-1 + mode: external + external: + endpoint: https://s3.example.com # empty for AWS S3 + region: us-east-1 bucket: appwrite accessKeyId: value: @@ -40,7 +44,7 @@ storage: compute: enabled: true executor: - privileged: true # acknowledges Docker-in-Docker: root on the node + privileged: true # the default: Docker-in-Docker, root on the node nodeSelector: workload: executor tolerations: diff --git a/examples/gcp.yaml b/examples/gcp.yaml index 381859f..11561f7 100644 --- a/examples/gcp.yaml +++ b/examples/gcp.yaml @@ -20,7 +20,7 @@ # "Allow only SSL connections" can stay on: Appwrite connects with TLS when # offered. # - Memorystore for Redis (not Redis Cluster), Standard tier for a replica -# with automatic failover, with in-transit encryption off (Appwrite 2.3.0 +# with automatic failover, with in-transit encryption off (Appwrite 2.5.0 # has no Redis TLS). AUTH can stay on: its AUTH string goes in the Secret. # - A Cloud Storage bucket, and an HMAC key (Cloud Storage > Settings > # Interoperability) for a service account with Storage Object Admin on the @@ -79,8 +79,10 @@ clickhouse: storage: device: s3 s3: - endpoint: https://storage.googleapis.com - region: auto + mode: external + external: + endpoint: https://storage.googleapis.com + region: auto bucket: appwrite-example-com accessKeyId: secretName: appwrite-external # s3-access-key: the HMAC key's access ID @@ -91,6 +93,12 @@ storage: accessModes: [ReadWriteMany] # Filestore lets the API and workers spread across nodes storageClass: standard-rwx +# The executor runs in a namespace of its own and can't read the S3 Secret +# above, so Functions and Sites stay off here: add +# functions-existing-secrets.yaml to turn them on. +compute: + enabled: false + smtp: host: smtp.sendgrid.net port: 587 diff --git a/examples/production.yaml b/examples/production.yaml index 366bd47..0785f55 100644 --- a/examples/production.yaml +++ b/examples/production.yaml @@ -81,8 +81,10 @@ clickhouse: storage: device: s3 s3: - endpoint: https://s3.example.com - region: us-east-1 + mode: external + external: + endpoint: https://s3.example.com + region: us-east-1 bucket: appwrite accessKeyId: secretName: appwrite-s3 @@ -93,6 +95,11 @@ storage: accessModes: [ReadWriteMany] # lets the API and workers spread across nodes storageClass: efs-sc # any class that supports ReadWriteMany +# Functions and Sites need their own credentials and a privileged executor on +# nodes of its own: add functions-existing-secrets.yaml to turn them on. +compute: + enabled: false + smtp: host: email-smtp.eu-west-1.amazonaws.com username: AKIAEXAMPLE diff --git a/hack/compute-plane.sh b/hack/compute-plane.sh deleted file mode 100755 index f2a3b1c..0000000 --- a/hack/compute-plane.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# Starts the S3 server Functions and Sites need (SeaweedFS) as a Docker -# container on the kind network, and writes chart values that point Appwrite -# at it and run builds and the executor in the cluster. -# -# Usage: hack/compute-plane.sh -# Requires: docker. -set -euo pipefail - -OUT="$1" -NETWORK=kind -FIXTURES="$(cd "$(dirname "$0")/fixtures/compute" && pwd)" -S3_KEY=appwrite -S3_SECRET=appwrite-s3-secret - -ip() { docker inspect "$1" -f "{{(index .NetworkSettings.Networks \"$NETWORK\").IPAddress}}"; } - -docker rm -f aw-s3 >/dev/null 2>&1 || true - -echo "==> S3" -# SeaweedFS sizes volumes at 30 GB and derives how many it may create from -# free disk, which leaves none on a CI runner: uploads then fail with "No -# writable volumes". Use small volumes and a fixed count. -docker run -d --name aw-s3 --network "$NETWORK" \ - -v "$FIXTURES/s3.json:/etc/seaweedfs/s3.json:ro" \ - chrislusf/seaweedfs:4.48 server -s3 -s3.config=/etc/seaweedfs/s3.json -dir=/data \ - -master.volumeSizeLimitMB=256 -volume.max=32 >/dev/null -for _ in $(seq 1 60); do - docker run --rm --network "$NETWORK" curlimages/curl:8.16.0 -fsS -o /dev/null \ - --aws-sigv4 "aws:amz:us-east-1:s3" --user "$S3_KEY:$S3_SECRET" -X PUT http://aw-s3:8333/appwrite 2>/dev/null && break - sleep 2 -done - -cat > "$OUT" < Wrote $OUT" diff --git a/hack/fixtures/compute/s3.json b/hack/fixtures/compute/s3.json deleted file mode 100644 index 1b67804..0000000 --- a/hack/fixtures/compute/s3.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "identities": [ - { - "name": "appwrite", - "credentials": [{"accessKey": "appwrite", "secretKey": "appwrite-s3-secret"}], - "actions": ["Admin", "Read", "Write", "List", "Tagging"] - } - ] -} diff --git a/hack/gen-env-defaults.php b/hack/gen-env-defaults.php index f8f5c74..b750c84 100644 --- a/hack/gen-env-defaults.php +++ b/hack/gen-env-defaults.php @@ -33,9 +33,30 @@ public static function getParam(string $key, mixed $default = []): mixed exit(2); } - $categories = require $argv[1]; $version = $argv[2]; + // variables.php can use Appwrite classes for defaults (2.4.0 and later: + // Appwrite\Network\TrustedProxies::DEFAULT). Load them from the same + // Appwrite version, as Composer's PSR-4 autoloader would. A class that + // needs more than its own file fails loudly. + spl_autoload_register(function (string $class) use ($version): void { + if (!str_starts_with($class, 'Appwrite\\')) { + return; + } + $path = 'src/' . str_replace('\\', '/', $class) . '.php'; + $source = @file_get_contents("https://raw.githubusercontent.com/appwrite/appwrite/{$version}/{$path}"); + if ($source === false) { + fwrite(STDERR, "variables.php needs {$class}, and {$path} couldn't be fetched for {$version}\n"); + exit(1); + } + $file = tempnam(sys_get_temp_dir(), 'appwrite-class-'); + file_put_contents($file, $source); + require $file; + unlink($file); + }); + + $categories = require $argv[1]; + // Secrets and connection strings that may embed credentials. The chart // passes these through Kubernetes Secrets instead. $secretPattern = '/(PASS|PASSWORD|SECRET|PRIVATE_KEY|OPENSSL_KEY|ACCESS_KEY|API_KEY)/'; diff --git a/hack/schema-postprocess.py b/hack/schema-postprocess.py index a318d03..22a4e82 100755 --- a/hack/schema-postprocess.py +++ b/hack/schema-postprocess.py @@ -169,6 +169,23 @@ def specific(schema): at(schema, "appwrite", key)["pattern"] = HOSTNAME +# Removed or renamed keys (dotted paths). The schema accepts them so a render +# gets as far as templates/_deprecations.tpl, which fails with what to do +# instead. Keep each for at least one minor release (CONTRIBUTING.md). +DEPRECATED = ( + "assistant", + "storage.s3.endpoint", + "storage.s3.region", +) + + +def deprecated(schema): + for path in DEPRECATED: + *parents, key = path.split(".") + node = at(schema, *parents) if parents else schema + node["properties"][key] = {"description": f"No longer supported: setting {path} fails with what to do instead."} + + def main(path): with open(path) as f: schema = json.load(f) @@ -187,6 +204,7 @@ def main(path): parent[keys[-1]] = dict(replacement, description=parent[keys[-1]].get("description", "")) constrain(schema) specific(schema) + deprecated(schema) with open(path, "w") as f: json.dump(schema, f, indent=2) f.write("\n") diff --git a/renovate.json b/renovate.json index f916682..7f2e7a7 100644 --- a/renovate.json +++ b/renovate.json @@ -223,7 +223,6 @@ "docker.io/appwrite/new", "docker.io/appwrite/geo", "docker.io/appwrite/browser", - "docker.io/appwrite/assistant", "docker.io/appwrite/embedding", "docker.io/appwrite/postgres", "ghcr.io/appwrite/autogravity", @@ -247,7 +246,8 @@ "matchPackageNames": [ "docker.io/redis", "docker.io/library/redis", - "docker.io/clickhouse/clickhouse-server" + "docker.io/clickhouse/clickhouse-server", + "docker.io/chrislusf/seaweedfs" ], "matchUpdateTypes": [ "major"