From e646a3fcb7deffbf3b772c4455e07a32d11fa56d Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:48:08 +0000 Subject: [PATCH 1/4] feat(hub): add provider-neutral artifact storage Route immutable byte operations through an artifacts domain with explicit filesystem or managed-identity Azure Blob selection. Validate configured storage without adding product routes. Co-authored-by: Ben Thomas <25218250+alliscode@users.noreply.github.com> Co-authored-by: Nishiki <1121087+nshki@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- hub/.env.example | 8 + hub/docs/README.md | 1 + hub/docs/architecture.md | 2 + hub/docs/artifact-storage.md | 60 ++ hub/package-lock.json | 591 +++++++++++++++++- hub/package.json | 2 + .../create-artifact-storage/index.test.ts | 119 ++++ .../create-artifact-storage/index.ts | 35 ++ .../create-artifact-storage/types.ts | 18 + .../index.test.ts | 69 ++ .../index.ts | 37 ++ .../index.test.ts | 52 ++ .../parse-artifact-storage-settings/index.ts | 36 ++ hub/server/index.ts | 4 + .../index.test.ts | 105 ++++ .../index.ts | 79 +++ .../types.ts | 18 + .../index.ts | 97 +++ .../types.ts | 12 + .../with-artifact-directory/index.ts | 33 + .../utils/parse-runtime-config/index.test.ts | 13 + .../utils/parse-runtime-config/index.ts | 6 + hub/server/workers/index.ts | 2 + 23 files changed, 1389 insertions(+), 10 deletions(-) create mode 100644 hub/docs/artifact-storage.md create mode 100644 hub/server/domains/artifacts/create-artifact-storage/index.test.ts create mode 100644 hub/server/domains/artifacts/create-artifact-storage/index.ts create mode 100644 hub/server/domains/artifacts/create-artifact-storage/types.ts create mode 100644 hub/server/domains/artifacts/create-configured-artifact-storage/index.test.ts create mode 100644 hub/server/domains/artifacts/create-configured-artifact-storage/index.ts create mode 100644 hub/server/domains/artifacts/parse-artifact-storage-settings/index.test.ts create mode 100644 hub/server/domains/artifacts/parse-artifact-storage-settings/index.ts create mode 100644 hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.test.ts create mode 100644 hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.ts create mode 100644 hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/types.ts create mode 100644 hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts create mode 100644 hub/server/integrations/filesystem/create-filesystem-artifact-storage/types.ts create mode 100644 hub/server/integrations/filesystem/with-artifact-directory/index.ts diff --git a/hub/.env.example b/hub/.env.example index 6d41d671..1599f71a 100644 --- a/hub/.env.example +++ b/hub/.env.example @@ -18,6 +18,14 @@ OTEL_TRACES_EXPORTER=none ON_DEMAND_JOB_CONCURRENCY=1 SCHEDULED_JOB_CONCURRENCY=1 +# Optional storage foundation. Choose one provider; see docs/artifact-storage.md. +# ARTIFACT_STORAGE_PROVIDER=filesystem +# ARTIFACT_STORAGE_DIRECTORY=/absolute/service-owned/artifacts +# For Azure, use provider azure and omit ARTIFACT_STORAGE_DIRECTORY: +# AZURE_STORAGE_ACCOUNT=youraccount +# AZURE_STORAGE_CONTAINER=artifacts +# AZURE_CLIENT_ID=your-user-assigned-managed-identity-uuid + # Local development only. The devcontainer supplies its own database hostname. DATABASE_URL=postgresql://replica:replica@127.0.0.1:5432/replica diff --git a/hub/docs/README.md b/hub/docs/README.md index f8c34b03..2bedca82 100644 --- a/hub/docs/README.md +++ b/hub/docs/README.md @@ -7,6 +7,7 @@ Hub is being rebuilt. The [Hub overview](../README.md) describes what the baseli - [Coding conventions](coding-conventions.md): frontend layers and interface rules. - [Development](development.md): local API and frontend setup. - [Deployment](deployment.md): production image, runtime roles, setup, and health. +- [Artifact storage](artifact-storage.md): provider selection and immutable byte storage. - [Database](database.md): Prisma schema conventions, migrations, and the shared client. - [Authentication proof](authentication.md): internal provider proof and one-use authorization attempts. - [Logging](logging.md): application events, automatic request records, and payload safety. diff --git a/hub/docs/architecture.md b/hub/docs/architecture.md index 22ea8f56..15cf0017 100644 --- a/hub/docs/architecture.md +++ b/hub/docs/architecture.md @@ -24,6 +24,8 @@ The [Pino integration](logging.md) writes structured application and HTTP logs t The [background-job foundation](background-jobs.md) uses BullMQ with PostgreSQL and separate on-demand and scheduled worker processes. Setup migrates BullMQ's own schema independently of Prisma. Product processors and schedules are not registered yet. +The [artifact storage domain](artifact-storage.md) selects filesystem or managed-identity Azure Blob storage behind one immutable-byte contract. Future product domains call this provider-neutral API, not SDK integrations directly. Startup validates explicitly configured storage, but no product route or processor consumes artifact bytes yet. + The [telemetry integration](telemetry.md) provides explicit server-side OpenTelemetry spans with optional operator-configured OTLP export. API and worker entrypoints own startup and flush telemetry after draining active work. Automatic instrumentation and product analytics are not implemented. The [WebSocket foundation](websockets.md) attaches to the API's HTTP server and broadcasts to local channel subscribers without persistence. The application rejects upgrades until access checks are supplied. Product channels, client recovery, and communication between processes are not implemented. diff --git a/hub/docs/artifact-storage.md b/hub/docs/artifact-storage.md new file mode 100644 index 00000000..e3902751 --- /dev/null +++ b/hub/docs/artifact-storage.md @@ -0,0 +1,60 @@ +# Artifact storage + +The artifact storage foundation saves immutable bytes behind a provider-neutral domain API. It does not import Builds, extract archives, host previews, authorize access, or add product routes or database models. + +## Composition + +Call `createArtifactStorage(settings)` from `server/domains/artifacts/create-artifact-storage/index.ts` during startup. The asynchronous factory validates settings and returns `read(key)`, `write(key, bytes)`, and `delete(key)`. `parseArtifactStorageSettings(input)` validates unknown configuration separately when startup needs a typed selection before composition. + +Choose exactly one settings shape: + +```ts +{ provider: "filesystem", directory: "/srv/replica/artifacts" } +``` + +```ts +{ + provider: "azure-blob", + accountUrl: "https://example.blob.core.windows.net", + container: "artifacts", + managedIdentityClientId: "ee5eeebb-74e0-4b79-8030-c325e50fec43", // Optional user-assigned identity. +} +``` + +Alternatively, startup can call `createConfiguredArtifactStorage(environment)` from `server/domains/artifacts/create-configured-artifact-storage/index.ts`, supplying a read-only environment record explicitly. Neither domain factory reads `process.env`. + +| Environment variable | Selection | +| ---------------------------- | ---------------------------------------------------------------------------------------------- | +| `ARTIFACT_STORAGE_PROVIDER` | Required: exactly `azure` or `filesystem`. | +| `ARTIFACT_STORAGE_DIRECTORY` | Required absolute directory for `filesystem`; forbidden for `azure`. | +| `AZURE_STORAGE_ACCOUNT` | Required public-cloud account name for `azure`; forbidden for `filesystem`. | +| `AZURE_STORAGE_CONTAINER` | Required container name for `azure`; forbidden for `filesystem`. | +| `AZURE_CLIENT_ID` | Optional explicit user-assigned managed identity UUID for `azure`; forbidden for `filesystem`. | + +The wrapper constructs the public-cloud HTTPS account URL and calls the typed factory. Missing or invalid providers, incomplete configuration, and mixed-provider settings fail with a safe error. An empty value still counts as a supplied setting when checking mixed providers. Unrelated environment variables are ignored. Startup may leave storage uncomposed when artifact storage is not configured; calling the wrapper without a provider always rejects. + +API, combined, and independent worker entry points validate and construct configured storage before startup. Any supplied provider, directory, account, or container setting opts in, so incomplete settings fail rather than being ignored. Setup does not need storage. Without storage settings, development keeps working unchanged. This startup check does not read or write artifacts, prove Azure access, or connect an unimplemented product flow. + +Unknown typed settings, relative directories, account credentials, URL query strings, and unsupported endpoints fail validation with a safe error. Azure selection never falls back to local storage. + +The Azure endpoint accepts public-cloud HTTPS account URLs only. The adapter uses `ManagedIdentityCredential`, with either the system-assigned identity or the explicitly selected user-assigned identity. It never uses account keys, connection strings, developer credentials, or `DefaultAzureCredential`. Operators provision the account and container and grant the runtime identity `Storage Blob Data Contributor` at the intended container scope. Construction does not contact Azure; missing identity, permissions, or container availability is reported when an operation runs. + +## Bytes and results + +Keys are case-sensitive opaque strings of 1–128 ASCII letters, digits, underscores, or hyphens, beginning with a letter or digit. They are not paths or URLs. The domain rejects traversal, separators, percent encoding, and other characters before calling a provider. + +Writes accept a `Uint8Array` of at most 32 MiB, including empty bytes, and never overwrite an existing key. Reads return a bounded `Uint8Array`. A successful operation returns `{ status: "ok" }`; a successful read also includes `bytes`. Failures return a typed `status`: `invalid-key`, `too-large`, `already-exists`, `not-found`, or `unavailable`. Missing reads and deletes are explicit `not-found` results, not empty bytes or silent success. Provider messages and credentials are not included in results. + +Immutability prevents overwriting bytes while a key exists. Deleting a key permits later reuse. Callers own authorization, key allocation, retention, and coordination between deletion and other work. + +## Local filesystem + +Operators must provision an absolute directory before startup. The adapter supports Linux with `/proc/self/fd`, including the Hub container runtime. Other platforms fail safely rather than weaken path protection. No directory component or artifact file may be a symbolic link. Directory descriptors pin each component during an operation so replacing an ancestor cannot redirect that operation through a symlink. + +Writes create a private staging file, flush the bytes, and publish it with an exclusive hard link. Readers see either the complete artifact or a missing key; concurrent writers cannot overwrite each other. Staging files are removed after operations. An abrupt process exit can leave a hidden staging file; operators may remove those files while the service is stopped. The adapter does not promise recovery across filesystem or host failure. + +Use a private service-owned directory and restrict local write access. Symlink protection does not make an attacker-writable storage directory safe: another local process could replace regular files or create hard links. Missing storage directories and other filesystem failures return `unavailable`, not a successful fallback. + +## Verification + +Colocated Vitest cases cover local byte round trips, concurrent immutable publication, explicit missing results, limits, traversal, symlink rejection, settings selection, managed-identity construction, and safe Azure error mapping. Azure cases replace the SDK boundary; they do not prove a deployed identity, role assignment, or real Blob access. Real Azure verification remains an operator check. diff --git a/hub/package-lock.json b/hub/package-lock.json index 8ffc9044..d5471d65 100644 --- a/hub/package-lock.json +++ b/hub/package-lock.json @@ -9,6 +9,8 @@ "version": "0.1.0", "hasInstallScript": true, "dependencies": { + "@azure/identity": "4.13.0", + "@azure/storage-blob": "12.33.0", "@opentelemetry/api": "1.10.0", "@opentelemetry/context-async-hooks": "3.0.0", "@opentelemetry/exporter-trace-otlp-proto": "0.300.0", @@ -126,6 +128,265 @@ "node": "20 || >=22" } }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-http-compat": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.5.0.tgz", + "integrity": "sha512-BoSmXPx2er1Ai+wKlDvj29jIQespCNBwEmKyZVHO2kEFsWbGjAjwMCGzug3DJM5/QYIV3vej0S1zcU5bq9fa8w==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.7.0.tgz", + "integrity": "sha512-7GEAoIsaoBr6KELNRb8nypowCqvk8dnCHFCYg4XD4lOQGY2GqjQg5IhkRjyBFRO18CGSMq05PaNqSOE9GQro3g==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.6.0.tgz", + "integrity": "sha512-e7lX/dk//F6Qf7BB6PTY4+p2yuOQtyOeHGyapYHNwqSp2OnYpwQt49A/Nin2XmKBQ69pwagR4k/lQBq8lbHQkA==", + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.5.9", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/identity": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.13.0.tgz", + "integrity": "sha512-uWC0fssc+hs1TGGVkkghiaFkkS7NkTxfnCH+Hdg+yTehTpMcehpok4PgUKKdyCH+9ldu6FhiHRv84Ntqj1vVcw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-rest-pipeline": "^1.17.0", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "@azure/msal-browser": "^4.2.0", + "@azure/msal-node": "^3.5.0", + "open": "^10.1.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/msal-browser": { + "version": "4.30.0", + "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-4.30.0.tgz", + "integrity": "sha512-HBBKfbZkMVzzF5bofvS1cXuNHFVc+gt4/HOnCmG/0hsHuZRJvJvDg/+7nTwIpoqvJc8BQp5o23rBUfisOLxR+w==", + "license": "MIT", + "dependencies": { + "@azure/msal-common": "15.17.0" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-common": { + "version": "15.17.0", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-15.17.0.tgz", + "integrity": "sha512-VQ5/gTLFADkwue+FohVuCqlzFPUq4xSrX8jeZe+iwZuY6moliNC8xt86qPVNYdtbQfELDf2Nu6LI+demFPHGgw==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-node": { + "version": "3.8.10", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-3.8.10.tgz", + "integrity": "sha512-0Hz7Kx4hs70KZWep/Rd7aw/qOLUF92wUOhn7ZsOuB5xNR/06NL1E2RAI9+UKH1FtvN8nD6mFjH7UKSjv6vOWvQ==", + "license": "MIT", + "dependencies": { + "@azure/msal-common": "15.17.0", + "jsonwebtoken": "^9.0.0", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.33.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.33.0.tgz", + "integrity": "sha512-2SX8oP8PyblUcAFZSg39c8Ls+tFjavM6sBeV+qpw33mRzRhI/5hrFJmJ/x0H9xx5l6ECPvgSP8uPxqTeVbHNIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.4.1", + "events": "^3.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/storage-common": { + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.5.0.tgz", + "integrity": "sha512-bttzuhQiCIwrkzjPDA+AtAR7dg19L/CC6ztcqJ5LfvWpXuys9mHp0UQ0udYnoUvv9SCT9KTR5kqFvFr0e6k0lQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.4.0", + "@azure/core-rest-pipeline": "^1.24.0", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -1945,6 +2206,18 @@ "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" } }, + "node_modules/@nodable/entities": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.1.0.tgz", + "integrity": "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@opentelemetry/api": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.10.0.tgz", @@ -4874,6 +5147,20 @@ "node": ">=16.20.0" } }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@visx/curve": { "version": "4.0.1-alpha.0", "resolved": "https://registry.npmjs.org/@visx/curve/-/curve-4.0.1-alpha.0.tgz", @@ -5194,6 +5481,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -5233,6 +5529,18 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -5404,6 +5712,12 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/bullmq": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-6.4.0.tgz", @@ -5456,7 +5770,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", - "dev": true, "license": "MIT", "dependencies": { "run-applescript": "^7.0.0" @@ -5944,7 +6257,6 @@ "version": "5.5.1", "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.1.tgz", "integrity": "sha512-m1pAzaJgZ/gssEqlOhJkPJp8Xly7QyW6xcrkUa2KKcDeDSEMP7X8xipU3snUcfisTQx0w1AGae+9UtJSfVnXGw==", - "dev": true, "license": "MIT", "dependencies": { "bundle-name": "^4.1.0", @@ -5961,7 +6273,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -5974,7 +6285,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -6082,6 +6392,15 @@ "node": ">= 0.4" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -6518,6 +6837,15 @@ "node": ">= 0.6" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/expect-type": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", @@ -6650,6 +6978,45 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.2", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.2.tgz", + "integrity": "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.1", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -7055,6 +7422,32 @@ "url": "https://opencollective.com/express" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -7145,7 +7538,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", - "dev": true, "license": "MIT", "bin": { "is-docker": "cli.js" @@ -7184,7 +7576,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", - "dev": true, "license": "MIT", "dependencies": { "is-docker": "^3.0.0" @@ -7218,11 +7609,22 @@ "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", "license": "MIT" }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/is-wsl": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", - "dev": true, "license": "MIT", "dependencies": { "is-inside-container": "^1.0.0" @@ -7352,6 +7754,61 @@ "dev": true, "license": "MIT" }, + "node_modules/jsonwebtoken": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", + "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", + "license": "MIT", + "dependencies": { + "jws": "^4.0.1", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jsonwebtoken/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/keyv": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", @@ -7702,6 +8159,48 @@ "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT" }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -8090,7 +8589,6 @@ "version": "10.2.0", "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", - "dev": true, "license": "MIT", "dependencies": { "default-browser": "^5.2.1", @@ -8256,6 +8754,21 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -9077,7 +9590,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -9086,6 +9598,26 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safe-regex2": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.2.tgz", @@ -9489,6 +10021,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.5.1.tgz", + "integrity": "sha512-UM4nTLWzQiDBseVxUGxN0wJzIk0UU89Qi7pKXz9qISo+lhviJikTuikBsK/oFRlpvYHvSdz0TVoLyQ+sEpYwxA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", @@ -9839,6 +10386,16 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/valibot": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/valibot/-/valibot-1.4.2.tgz", @@ -10167,7 +10724,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", - "dev": true, "license": "MIT", "dependencies": { "is-wsl": "^3.1.0" @@ -10189,6 +10745,21 @@ "node": ">=18" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", diff --git a/hub/package.json b/hub/package.json index d5b338c2..1a7ca715 100644 --- a/hub/package.json +++ b/hub/package.json @@ -48,6 +48,8 @@ "typecheck": "tsc --noEmit && tsc --project ../.github/extensions/replica-prototype/tsconfig.json" }, "dependencies": { + "@azure/identity": "4.13.0", + "@azure/storage-blob": "12.33.0", "@opentelemetry/api": "1.10.0", "@opentelemetry/context-async-hooks": "3.0.0", "@opentelemetry/exporter-trace-otlp-proto": "0.300.0", diff --git a/hub/server/domains/artifacts/create-artifact-storage/index.test.ts b/hub/server/domains/artifacts/create-artifact-storage/index.test.ts new file mode 100644 index 00000000..a6587122 --- /dev/null +++ b/hub/server/domains/artifacts/create-artifact-storage/index.test.ts @@ -0,0 +1,119 @@ +import { randomUUID } from "node:crypto" +import { mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises" +import { resolve } from "node:path" +import { describe, expect, it, onTestFinished } from "vitest" + +import { artifactMaxBytes, createArtifactStorage } from "." + +describe("createArtifactStorage()", () => { + it("publishes complete immutable bytes and explicitly reports missing reads and deletes", async () => { + const directory = resolve(`server/domains/artifacts/.storage-test-${randomUUID()}`) + await mkdir(directory) + onTestFinished(() => rm(directory, { recursive: true, force: true })) + const storage = await createArtifactStorage({ provider: "filesystem", directory }) + const first = Uint8Array.from([0, 255, 7]) + const second = Uint8Array.from([9, 8]) + + const results = await Promise.all([ + storage.write("opaque-key", first), + storage.write("opaque-key", second), + ]) + expect(results.map((result) => result.status).sort()).toEqual(["already-exists", "ok"]) + const result = await storage.read("opaque-key") + expect(result.status).toBe("ok") + if (result.status !== "ok") throw new Error("Expected saved bytes.") + expect(Array.from(result.bytes)).toEqual( + Array.from(results[0]?.status === "ok" ? first : second), + ) + expect(await storage.delete("opaque-key")).toEqual({ status: "ok" }) + expect(await storage.read("opaque-key")).toEqual({ status: "not-found" }) + expect(await storage.delete("opaque-key")).toEqual({ status: "not-found" }) + expect(await storage.write("empty", new Uint8Array())).toEqual({ status: "ok" }) + expect(await storage.read("empty")).toEqual({ status: "ok", bytes: new Uint8Array() }) + }) + + it("rejects traversal and encoded path keys before touching storage", async () => { + const directory = resolve(`server/domains/artifacts/.storage-test-${randomUUID()}`) + await mkdir(directory) + onTestFinished(() => rm(directory, { recursive: true, force: true })) + const storage = await createArtifactStorage({ provider: "filesystem", directory }) + + for (const key of [ + "", + "../outside", + "/absolute", + "a/b", + "a\\b", + "%2e%2e", + ".", + "a\0", + "x".repeat(129), + ]) { + expect(await storage.write(key, new Uint8Array())).toEqual({ status: "invalid-key" }) + expect(await storage.read(key)).toEqual({ status: "invalid-key" }) + expect(await storage.delete(key)).toEqual({ status: "invalid-key" }) + } + }) + + it("refuses symlink leaves and directories without reading or changing the target", async () => { + const directory = resolve(`server/domains/artifacts/.storage-test-${randomUUID()}`) + await mkdir(directory) + onTestFinished(() => rm(directory, { recursive: true, force: true })) + await mkdir(`${directory}/root`) + await writeFile(`${directory}/outside`, "private") + await symlink(`${directory}/outside`, `${directory}/root/key`) + const storage = await createArtifactStorage({ + provider: "filesystem", + directory: `${directory}/root`, + }) + + expect(await storage.read("key")).toEqual({ status: "unavailable" }) + expect(await storage.write("key", Uint8Array.of(1))).toEqual({ status: "already-exists" }) + expect(await storage.delete("key")).toEqual({ status: "unavailable" }) + expect(await readFile(`${directory}/outside`, "utf8")).toBe("private") + await symlink(`${directory}/root`, `${directory}/alias`) + await expect( + createArtifactStorage({ provider: "filesystem", directory: `${directory}/alias` }), + ).rejects.toThrow("Artifact directory is unavailable.") + await expect( + createArtifactStorage({ provider: "filesystem", directory: `${directory}/alias/nested` }), + ).rejects.toThrow("Artifact directory is unavailable.") + await rm(`${directory}/root`, { recursive: true }) + await mkdir(`${directory}/other`) + await writeFile(`${directory}/other/key`, "private") + await symlink(`${directory}/other`, `${directory}/root`) + expect(await storage.read("key")).toEqual({ status: "unavailable" }) + expect(await storage.write("key", Uint8Array.of(1))).toEqual({ status: "unavailable" }) + expect(await storage.delete("key")).toEqual({ status: "unavailable" }) + expect(await readFile(`${directory}/other/key`, "utf8")).toBe("private") + }) + + it("bounds write and read bytes while accepting the exact limit", async () => { + const directory = resolve(`server/domains/artifacts/.storage-test-${randomUUID()}`) + await mkdir(directory) + onTestFinished(() => rm(directory, { recursive: true, force: true })) + const storage = await createArtifactStorage({ provider: "filesystem", directory }) + + expect(await storage.write("limit", new Uint8Array(artifactMaxBytes))).toEqual({ status: "ok" }) + const result = await storage.read("limit") + expect(result.status).toBe("ok") + if (result.status === "ok") expect(result.bytes.length).toBe(artifactMaxBytes) + expect(await storage.write("large", new Uint8Array(artifactMaxBytes + 1))).toEqual({ + status: "too-large", + }) + await writeFile(`${directory}/large`, new Uint8Array(artifactMaxBytes + 1)) + expect(await storage.read("large")).toEqual({ status: "too-large" }) + }) + + it("fails safely when the configured filesystem directory disappears", async () => { + const directory = resolve(`server/domains/artifacts/.storage-test-${randomUUID()}`) + await mkdir(directory) + onTestFinished(() => rm(directory, { recursive: true, force: true })) + const storage = await createArtifactStorage({ provider: "filesystem", directory }) + await rm(directory, { recursive: true }) + + expect(await storage.read("key")).toEqual({ status: "unavailable" }) + expect(await storage.write("key", Uint8Array.of(1))).toEqual({ status: "unavailable" }) + expect(await storage.delete("key")).toEqual({ status: "unavailable" }) + }) +}) diff --git a/hub/server/domains/artifacts/create-artifact-storage/index.ts b/hub/server/domains/artifacts/create-artifact-storage/index.ts new file mode 100644 index 00000000..ec3d8c4e --- /dev/null +++ b/hub/server/domains/artifacts/create-artifact-storage/index.ts @@ -0,0 +1,35 @@ +import { createAzureBlobArtifactStorage } from "../../../integrations/azure-blob/create-azure-blob-artifact-storage/index.ts" +import { createFilesystemArtifactStorage } from "../../../integrations/filesystem/create-filesystem-artifact-storage/index.ts" +import { parseArtifactStorageSettings } from "../parse-artifact-storage-settings/index.ts" +import type { ArtifactStorage, ArtifactStorageSettings } from "./types.ts" + +export const artifactMaxBytes = 32 * 1024 * 1024 +const keyPattern = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/ + +export async function createArtifactStorage( + input: ArtifactStorageSettings, +): Promise { + const settings = parseArtifactStorageSettings(input) + const adapter = + settings.provider === "filesystem" + ? await createFilesystemArtifactStorage(settings.directory, artifactMaxBytes) + : createAzureBlobArtifactStorage(settings, artifactMaxBytes) + + return { + async read(key) { + if (typeof key !== "string" || !keyPattern.test(key)) return { status: "invalid-key" } + return adapter.read(key) + }, + async write(key, bytes) { + if (typeof key !== "string" || !keyPattern.test(key)) return { status: "invalid-key" } + if (!(bytes instanceof Uint8Array) || bytes.byteLength > artifactMaxBytes) { + return { status: "too-large" } + } + return adapter.write(key, Uint8Array.from(bytes)) + }, + async delete(key) { + if (typeof key !== "string" || !keyPattern.test(key)) return { status: "invalid-key" } + return adapter.delete(key) + }, + } +} diff --git a/hub/server/domains/artifacts/create-artifact-storage/types.ts b/hub/server/domains/artifacts/create-artifact-storage/types.ts new file mode 100644 index 00000000..7e6ab538 --- /dev/null +++ b/hub/server/domains/artifacts/create-artifact-storage/types.ts @@ -0,0 +1,18 @@ +export type ArtifactStorageSettings = + | { provider: "filesystem"; directory: string } + | { + provider: "azure-blob" + accountUrl: string + container: string + managedIdentityClientId?: string + } + +export type ArtifactFailure = { + status: "invalid-key" | "too-large" | "not-found" | "already-exists" | "unavailable" +} + +export interface ArtifactStorage { + read(key: string): Promise<{ status: "ok"; bytes: Uint8Array } | ArtifactFailure> + write(key: string, bytes: Uint8Array): Promise<{ status: "ok" } | ArtifactFailure> + delete(key: string): Promise<{ status: "ok" } | ArtifactFailure> +} diff --git a/hub/server/domains/artifacts/create-configured-artifact-storage/index.test.ts b/hub/server/domains/artifacts/create-configured-artifact-storage/index.test.ts new file mode 100644 index 00000000..0c8462bb --- /dev/null +++ b/hub/server/domains/artifacts/create-configured-artifact-storage/index.test.ts @@ -0,0 +1,69 @@ +import { randomUUID } from "node:crypto" +import { mkdir, readFile, rm } from "node:fs/promises" +import { resolve } from "node:path" +import { describe, expect, it, onTestFinished } from "vitest" + +import { createConfiguredArtifactStorage } from "." + +describe("createConfiguredArtifactStorage()", () => { + it("uses the explicitly supplied filesystem directory", async () => { + const directory = resolve(`server/domains/artifacts/.storage-test-${randomUUID()}`) + await mkdir(directory) + onTestFinished(() => rm(directory, { recursive: true, force: true })) + const storage = await createConfiguredArtifactStorage({ + ARTIFACT_STORAGE_PROVIDER: "filesystem", + ARTIFACT_STORAGE_DIRECTORY: directory, + UNRELATED_SETTING: "ignored", + }) + + expect(await storage.write("key", Uint8Array.of(0, 255))).toEqual({ status: "ok" }) + expect(await readFile(`${directory}/key`)).toEqual(Buffer.from([0, 255])) + }) + + it("rejects absent, invalid, incomplete, or mixed provider settings safely", async () => { + for (const environment of [ + {}, + { ARTIFACT_STORAGE_PROVIDER: "" }, + { ARTIFACT_STORAGE_PROVIDER: "unknown" }, + { ARTIFACT_STORAGE_PROVIDER: "filesystem" }, + { + ARTIFACT_STORAGE_PROVIDER: "filesystem", + ARTIFACT_STORAGE_DIRECTORY: "/srv/artifacts", + AZURE_STORAGE_ACCOUNT: "example", + }, + { + ARTIFACT_STORAGE_PROVIDER: "filesystem", + ARTIFACT_STORAGE_DIRECTORY: "/srv/artifacts", + AZURE_STORAGE_CONTAINER: "artifacts", + }, + { + ARTIFACT_STORAGE_PROVIDER: "filesystem", + ARTIFACT_STORAGE_DIRECTORY: "/srv/artifacts", + AZURE_CLIENT_ID: "", + }, + { + ARTIFACT_STORAGE_PROVIDER: "azure", + AZURE_STORAGE_CONTAINER: "artifacts", + }, + { + ARTIFACT_STORAGE_PROVIDER: "azure", + AZURE_STORAGE_ACCOUNT: "example", + }, + { + ARTIFACT_STORAGE_PROVIDER: "azure", + AZURE_STORAGE_ACCOUNT: "example?secret", + AZURE_STORAGE_CONTAINER: "artifacts", + }, + { + ARTIFACT_STORAGE_PROVIDER: "azure", + AZURE_STORAGE_ACCOUNT: "example", + AZURE_STORAGE_CONTAINER: "artifacts", + ARTIFACT_STORAGE_DIRECTORY: "", + }, + ]) { + await expect(createConfiguredArtifactStorage(environment)).rejects.toThrow( + "Artifact storage configuration is invalid.", + ) + } + }) +}) diff --git a/hub/server/domains/artifacts/create-configured-artifact-storage/index.ts b/hub/server/domains/artifacts/create-configured-artifact-storage/index.ts new file mode 100644 index 00000000..fbee4dfb --- /dev/null +++ b/hub/server/domains/artifacts/create-configured-artifact-storage/index.ts @@ -0,0 +1,37 @@ +import { createArtifactStorage } from "../create-artifact-storage/index.ts" +import type { ArtifactStorage } from "../create-artifact-storage/types.ts" + +export async function createConfiguredArtifactStorage( + environment: Readonly>, +): Promise { + if (environment.ARTIFACT_STORAGE_PROVIDER === "filesystem") { + if ( + environment.AZURE_STORAGE_ACCOUNT !== undefined || + environment.AZURE_STORAGE_CONTAINER !== undefined || + environment.AZURE_CLIENT_ID !== undefined + ) { + throw new Error("Artifact storage configuration is invalid.") + } + return createArtifactStorage({ + provider: "filesystem", + directory: environment.ARTIFACT_STORAGE_DIRECTORY ?? "", + }) + } + if (environment.ARTIFACT_STORAGE_PROVIDER === "azure") { + const account = environment.AZURE_STORAGE_ACCOUNT + if ( + environment.ARTIFACT_STORAGE_DIRECTORY !== undefined || + !account || + !/^[a-z0-9]{3,24}$/.test(account) + ) { + throw new Error("Artifact storage configuration is invalid.") + } + return createArtifactStorage({ + provider: "azure-blob", + accountUrl: `https://${account}.blob.core.windows.net`, + container: environment.AZURE_STORAGE_CONTAINER ?? "", + managedIdentityClientId: environment.AZURE_CLIENT_ID, + }) + } + throw new Error("Artifact storage configuration is invalid.") +} diff --git a/hub/server/domains/artifacts/parse-artifact-storage-settings/index.test.ts b/hub/server/domains/artifacts/parse-artifact-storage-settings/index.test.ts new file mode 100644 index 00000000..7099932e --- /dev/null +++ b/hub/server/domains/artifacts/parse-artifact-storage-settings/index.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest" + +import { parseArtifactStorageSettings } from "." + +describe("parseArtifactStorageSettings()", () => { + it("accepts explicit filesystem or managed-identity Azure settings", () => { + const filesystem = { provider: "filesystem", directory: "/srv/artifacts" } + const azure = { + provider: "azure-blob", + accountUrl: "https://example.blob.core.windows.net", + container: "artifacts", + managedIdentityClientId: "ee5eeebb-74e0-4b79-8030-c325e50fec43", + } + expect(parseArtifactStorageSettings(filesystem)).toEqual(filesystem) + expect(parseArtifactStorageSettings(azure)).toEqual(azure) + }) + + it("rejects missing, unknown, ambiguous, or credential-bearing settings without echoing input", () => { + for (const input of [ + {}, + { provider: "other" }, + { provider: "filesystem", directory: "relative" }, + { provider: "filesystem", directory: "/srv/../artifacts" }, + { provider: "filesystem", directory: "/srv/artifacts", container: "artifacts" }, + { + provider: "azure-blob", + accountUrl: "https://example.blob.core.windows.net?sig=secret", + container: "artifacts", + }, + { + provider: "azure-blob", + accountUrl: "http://example.blob.core.windows.net", + container: "artifacts", + }, + { + provider: "azure-blob", + accountUrl: "https://example.blob.core.windows.net", + container: "artifacts", + accountKey: "secret", + }, + { + provider: "azure-blob", + accountUrl: "https://example.blob.core.windows.net", + container: "invalid/container", + }, + ]) { + expect(() => parseArtifactStorageSettings(input)).toThrow( + "Artifact storage configuration is invalid.", + ) + } + }) +}) diff --git a/hub/server/domains/artifacts/parse-artifact-storage-settings/index.ts b/hub/server/domains/artifacts/parse-artifact-storage-settings/index.ts new file mode 100644 index 00000000..acb57101 --- /dev/null +++ b/hub/server/domains/artifacts/parse-artifact-storage-settings/index.ts @@ -0,0 +1,36 @@ +import { isAbsolute } from "node:path" +import { z } from "zod" + +import type { ArtifactStorageSettings } from "../create-artifact-storage/types.ts" + +const settingsSchema = z.discriminatedUnion("provider", [ + z + .object({ + provider: z.literal("filesystem"), + directory: z + .string() + .min(1) + .refine((value) => isAbsolute(value) && !value.includes("\0")), + }) + .strict(), + z + .object({ + provider: z.literal("azure-blob"), + accountUrl: z.string().regex(/^https:\/\/[a-z0-9]{3,24}\.blob\.core\.windows\.net\/?$/), + container: z.string().regex(/^(?!.*--)[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/), + managedIdentityClientId: z.uuid().optional(), + }) + .strict(), +]) + +export function parseArtifactStorageSettings(input: unknown): ArtifactStorageSettings { + const parsed = settingsSchema.safeParse(input) + if (!parsed.success) throw new Error("Artifact storage configuration is invalid.") + if ( + parsed.data.provider === "filesystem" && + parsed.data.directory.split("/").some((part) => part === "." || part === "..") + ) { + throw new Error("Artifact storage configuration is invalid.") + } + return parsed.data +} diff --git a/hub/server/index.ts b/hub/server/index.ts index 8f544603..63816c08 100644 --- a/hub/server/index.ts +++ b/hub/server/index.ts @@ -1,5 +1,6 @@ import "dotenv/config" +import { createConfiguredArtifactStorage } from "./domains/artifacts/create-configured-artifact-storage/index.ts" import { startTelemetry } from "./integrations/opentelemetry/start-telemetry/index.ts" import { logger } from "./integrations/pino/index.ts" import { runApi } from "./run-api/index.ts" @@ -11,6 +12,9 @@ import { runJobWorker } from "./workers/run-job-worker/index.ts" try { const configuration = parseRuntimeConfig(process.env) + if (configuration.mode !== "setup" && configuration.hasArtifactStorage) { + await createConfiguredArtifactStorage(process.env) + } const telemetry = startTelemetry(process.env, logger) try { if (configuration.mode === "setup") await setup() diff --git a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.test.ts b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.test.ts new file mode 100644 index 00000000..9cf487fe --- /dev/null +++ b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.test.ts @@ -0,0 +1,105 @@ +import { Readable } from "node:stream" +import { describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + credential: vi.fn(), + service: vi.fn(), + download: vi.fn(), + uploadData: vi.fn(), + delete: vi.fn(), +})) +vi.mock("@azure/identity", () => ({ + ManagedIdentityCredential: class { + constructor(...args: unknown[]) { + mocks.credential(...args) + } + }, +})) +vi.mock("@azure/storage-blob", () => ({ + BlobServiceClient: class { + constructor(...args: unknown[]) { + mocks.service(...args) + } + getContainerClient() { + return { getBlockBlobClient: () => mocks } + } + }, +})) + +import { createAzureBlobArtifactStorage } from "." + +describe("createAzureBlobArtifactStorage()", () => { + it("uses only managed identity and conditional writes with bounded binary reads", async () => { + const clientId = "ee5eeebb-74e0-4b79-8030-c325e50fec43" + const storage = createAzureBlobArtifactStorage( + { + accountUrl: "https://example.blob.core.windows.net", + container: "artifacts", + managedIdentityClientId: clientId, + }, + 3, + ) + const bytes = Uint8Array.from([0, 255, 1]) + mocks.download.mockResolvedValue({ + contentLength: 3, + readableStreamBody: Readable.from([bytes]), + }) + + expect(await storage.write("key", bytes)).toEqual({ status: "ok" }) + expect(mocks.uploadData).toHaveBeenCalledWith( + bytes, + expect.objectContaining({ conditions: { ifNoneMatch: "*" } }), + ) + expect(await storage.read("key")).toEqual({ status: "ok", bytes: Buffer.from(bytes) }) + expect(await storage.delete("key")).toEqual({ status: "ok" }) + expect(mocks.credential).toHaveBeenCalledExactlyOnceWith({ clientId }) + }) + + it("returns explicit missing and immutable-write conflict results", async () => { + const storage = createAzureBlobArtifactStorage( + { accountUrl: "https://example.blob.core.windows.net", container: "artifacts" }, + 3, + ) + mocks.download.mockRejectedValue({ details: { errorCode: "BlobNotFound" } }) + mocks.delete.mockRejectedValue({ details: { errorCode: "BlobNotFound" } }) + mocks.uploadData.mockRejectedValue({ details: { errorCode: "ConditionNotMet" } }) + + expect(await storage.read("key")).toEqual({ status: "not-found" }) + expect(await storage.delete("key")).toEqual({ status: "not-found" }) + expect(await storage.write("key", Uint8Array.of(1))).toEqual({ status: "already-exists" }) + expect(mocks.credential).toHaveBeenCalledExactlyOnceWith() + }) + + it("maps provider failures without leaking SDK errors or treating a missing container as missing bytes", async () => { + const storage = createAzureBlobArtifactStorage( + { accountUrl: "https://example.blob.core.windows.net", container: "artifacts" }, + 3, + ) + mocks.download.mockRejectedValue({ + details: { errorCode: "ContainerNotFound" }, + message: "secret", + }) + mocks.delete.mockRejectedValue(new Error("secret")) + mocks.uploadData.mockRejectedValue(new Error("secret")) + + expect(await storage.read("key")).toEqual({ status: "unavailable" }) + expect(await storage.delete("key")).toEqual({ status: "unavailable" }) + expect(await storage.write("key", Uint8Array.of(1))).toEqual({ status: "unavailable" }) + }) + + it("stops oversized reads and accepts empty objects", async () => { + const storage = createAzureBlobArtifactStorage( + { accountUrl: "https://example.blob.core.windows.net", container: "artifacts" }, + 3, + ) + const stream = Readable.from([Uint8Array.of(1, 2, 3, 4)]) + mocks.download.mockResolvedValueOnce({ readableStreamBody: stream }) + expect(await storage.read("key")).toEqual({ status: "too-large" }) + expect(stream.destroyed).toBe(true) + mocks.download.mockResolvedValueOnce({ + contentLength: 0, + readableStreamBody: Readable.from([]), + }) + expect(await storage.read("empty")).toEqual({ status: "ok", bytes: Buffer.alloc(0) }) + }) +}) diff --git a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.ts b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.ts new file mode 100644 index 00000000..2e22b9a5 --- /dev/null +++ b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/index.ts @@ -0,0 +1,79 @@ +import { ManagedIdentityCredential } from "@azure/identity" +import { BlobServiceClient } from "@azure/storage-blob" + +import type { AzureBlobArtifactSettings, AzureBlobArtifactStorage } from "./types.ts" + +export function createAzureBlobArtifactStorage( + settings: AzureBlobArtifactSettings, + maxBytes: number, +): AzureBlobArtifactStorage { + const credential = settings.managedIdentityClientId + ? new ManagedIdentityCredential({ clientId: settings.managedIdentityClientId }) + : new ManagedIdentityCredential() + const container = new BlobServiceClient(settings.accountUrl, credential).getContainerClient( + settings.container, + ) + + return { + async read(key) { + try { + const response = await container.getBlockBlobClient(key).download() + const stream = response.readableStreamBody + if (!stream) return { status: "unavailable" } + try { + if (response.contentLength !== undefined && response.contentLength > maxBytes) { + return { status: "too-large" } + } + const chunks: Buffer[] = [] + let length = 0 + for await (const chunk of stream) { + const bytes = Buffer.from(chunk) + length += bytes.length + if (length > maxBytes) return { status: "too-large" } + chunks.push(bytes) + } + return { status: "ok", bytes: Buffer.concat(chunks, length) } + } finally { + stream.destroy() + } + } catch (error) { + return { + status: + (error as { details?: { errorCode?: string } }).details?.errorCode === "BlobNotFound" + ? "not-found" + : "unavailable", + } + } + }, + async write(key, bytes) { + try { + await container.getBlockBlobClient(key).uploadData(bytes, { + conditions: { ifNoneMatch: "*" }, + blobHTTPHeaders: { blobContentType: "application/octet-stream" }, + }) + return { status: "ok" } + } catch (error) { + const code = (error as { details?: { errorCode?: string } }).details?.errorCode + return { + status: + code === "BlobAlreadyExists" || code === "ConditionNotMet" + ? "already-exists" + : "unavailable", + } + } + }, + async delete(key) { + try { + await container.getBlockBlobClient(key).delete() + return { status: "ok" } + } catch (error) { + return { + status: + (error as { details?: { errorCode?: string } }).details?.errorCode === "BlobNotFound" + ? "not-found" + : "unavailable", + } + } + }, + } +} diff --git a/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/types.ts b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/types.ts new file mode 100644 index 00000000..49b1dec7 --- /dev/null +++ b/hub/server/integrations/azure-blob/create-azure-blob-artifact-storage/types.ts @@ -0,0 +1,18 @@ +export interface AzureBlobArtifactSettings { + accountUrl: string + container: string + managedIdentityClientId?: string +} + +export interface AzureBlobArtifactStorage { + read( + key: string, + ): Promise< + { status: "ok"; bytes: Uint8Array } | { status: "not-found" | "unavailable" | "too-large" } + > + write( + key: string, + bytes: Uint8Array, + ): Promise<{ status: "ok" | "already-exists" | "unavailable" }> + delete(key: string): Promise<{ status: "ok" | "not-found" | "unavailable" }> +} diff --git a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts new file mode 100644 index 00000000..a23c63d9 --- /dev/null +++ b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts @@ -0,0 +1,97 @@ +import { randomUUID } from "node:crypto" +import { constants } from "node:fs" +import { link, lstat, open, unlink } from "node:fs/promises" + +import { withArtifactDirectory } from "../with-artifact-directory/index.ts" +import type { FilesystemArtifactStorage } from "./types.ts" + +export async function createFilesystemArtifactStorage( + directory: string, + maxBytes: number, +): Promise { + try { + await withArtifactDirectory(directory, () => Promise.resolve(undefined)) + } catch { + throw new Error("Artifact directory is unavailable.") + } + return { + async read(key) { + if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(key)) return { status: "unavailable" } + try { + return await withArtifactDirectory(directory, async (path) => { + const handle = await open( + `${path}/${key}`, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ) + try { + const stat = await handle.stat() + if (!stat.isFile()) return { status: "unavailable" } as const + if (stat.size > maxBytes) return { status: "too-large" } as const + const bytes = new Uint8Array(stat.size + 1) + let length = 0 + while (length < bytes.length) { + const { bytesRead } = await handle.read(bytes, length, bytes.length - length) + if (!bytesRead) break + length += bytesRead + } + if (length > stat.size) return { status: "unavailable" } as const + return { status: "ok", bytes: bytes.slice(0, length) } as const + } finally { + await handle.close() + } + }) + } catch (error) { + return { + status: (error as NodeJS.ErrnoException).code === "ENOENT" ? "not-found" : "unavailable", + } + } + }, + async write(key, bytes) { + if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(key) || bytes.length > maxBytes) { + return { status: "unavailable" } + } + try { + return await withArtifactDirectory(directory, async (path) => { + const staging = `${path}/.${randomUUID()}` + const handle = await open( + staging, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ) + try { + await handle.writeFile(bytes) + await handle.sync() + await handle.close() + // Publishing the completed file with link() atomically refuses existing keys. + await link(staging, `${path}/${key}`) + return { status: "ok" } as const + } finally { + await handle.close() + await unlink(staging) + } + }) + } catch (error) { + return { + status: + (error as NodeJS.ErrnoException).code === "EEXIST" ? "already-exists" : "unavailable", + } + } + }, + async delete(key) { + if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(key)) return { status: "unavailable" } + try { + return await withArtifactDirectory(directory, async (path) => { + if (!(await lstat(`${path}/${key}`)).isFile()) { + return { status: "unavailable" } as const + } + await unlink(`${path}/${key}`) + return { status: "ok" } as const + }) + } catch (error) { + return { + status: (error as NodeJS.ErrnoException).code === "ENOENT" ? "not-found" : "unavailable", + } + } + }, + } +} diff --git a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/types.ts b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/types.ts new file mode 100644 index 00000000..90c84f19 --- /dev/null +++ b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/types.ts @@ -0,0 +1,12 @@ +export interface FilesystemArtifactStorage { + read( + key: string, + ): Promise< + { status: "ok"; bytes: Uint8Array } | { status: "not-found" | "unavailable" | "too-large" } + > + write( + key: string, + bytes: Uint8Array, + ): Promise<{ status: "ok" | "already-exists" | "unavailable" }> + delete(key: string): Promise<{ status: "ok" | "not-found" | "unavailable" }> +} diff --git a/hub/server/integrations/filesystem/with-artifact-directory/index.ts b/hub/server/integrations/filesystem/with-artifact-directory/index.ts new file mode 100644 index 00000000..8a11fe86 --- /dev/null +++ b/hub/server/integrations/filesystem/with-artifact-directory/index.ts @@ -0,0 +1,33 @@ +import { constants } from "node:fs" +import { open } from "node:fs/promises" + +export async function withArtifactDirectory( + directory: string, + operation: (path: string) => Promise, +): Promise { + if ( + process.platform !== "linux" || + !directory.startsWith("/") || + directory.includes("\0") || + directory.split("/").some((part) => part === "." || part === "..") + ) { + throw new Error("Artifact directory is unavailable.") + } + let handle = await open("/", constants.O_RDONLY | constants.O_DIRECTORY) + try { + // Pinned directory descriptors prevent ancestor replacement from following a symlink. + for (const part of directory.split("/").filter(Boolean)) { + const next = await open( + `/proc/self/fd/${handle.fd}/${part}`, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ).catch(() => { + throw new Error("Artifact directory is unavailable.") + }) + await handle.close() + handle = next + } + return await operation(`/proc/self/fd/${handle.fd}`) + } finally { + await handle.close() + } +} diff --git a/hub/server/utils/parse-runtime-config/index.test.ts b/hub/server/utils/parse-runtime-config/index.test.ts index b72639dd..8282d458 100644 --- a/hub/server/utils/parse-runtime-config/index.test.ts +++ b/hub/server/utils/parse-runtime-config/index.test.ts @@ -8,6 +8,7 @@ describe("parseRuntimeConfig()", () => { mode: "api", host: "127.0.0.1", port: 3001, + hasArtifactStorage: false, }) expect( parseRuntimeConfig({ @@ -32,6 +33,18 @@ describe("parseRuntimeConfig()", () => { }) }) + it("detects explicit or incomplete storage configuration without choosing a provider", () => { + for (const env of [ + { ARTIFACT_STORAGE_PROVIDER: "azure" }, + { ARTIFACT_STORAGE_PROVIDER: "" }, + { ARTIFACT_STORAGE_DIRECTORY: "/srv/artifacts" }, + { AZURE_STORAGE_ACCOUNT: "example" }, + { AZURE_STORAGE_CONTAINER: "artifacts" }, + ]) { + expect(parseRuntimeConfig(env).hasArtifactStorage).toBe(true) + } + }) + it.each([ { RUNTIME_MODE: "private-config-marker" }, { PORT: "3001private-config-marker" }, diff --git a/hub/server/utils/parse-runtime-config/index.ts b/hub/server/utils/parse-runtime-config/index.ts index 812b0b55..5ac041fe 100644 --- a/hub/server/utils/parse-runtime-config/index.ts +++ b/hub/server/utils/parse-runtime-config/index.ts @@ -39,6 +39,12 @@ export function parseRuntimeConfig(env: NodeJS.ProcessEnv) { } return { mode, + hasArtifactStorage: [ + "ARTIFACT_STORAGE_PROVIDER", + "ARTIFACT_STORAGE_DIRECTORY", + "AZURE_STORAGE_ACCOUNT", + "AZURE_STORAGE_CONTAINER", + ].some((name) => env[name] !== undefined), port, host, shutdownTimeoutMs, diff --git a/hub/server/workers/index.ts b/hub/server/workers/index.ts index edfc7b1a..9fff515c 100644 --- a/hub/server/workers/index.ts +++ b/hub/server/workers/index.ts @@ -1,6 +1,7 @@ import "dotenv/config" import { z } from "zod" +import { createConfiguredArtifactStorage } from "../domains/artifacts/create-configured-artifact-storage/index.ts" import { startTelemetry } from "../integrations/opentelemetry/start-telemetry/index.ts" import { logger } from "../integrations/pino/index.ts" import { jobProcessors } from "./configuration.ts" @@ -10,6 +11,7 @@ import { parseRuntimeConfig } from "../utils/parse-runtime-config/index.ts" try { const name = z.enum(["on-demand", "scheduled"]).parse(process.argv[2]) const configuration = parseRuntimeConfig({ ...process.env, RUNTIME_MODE: name }) + if (configuration.hasArtifactStorage) await createConfiguredArtifactStorage(process.env) const databaseUrl = configuration.databaseUrl if (!databaseUrl) throw new Error("Workers require DATABASE_URL.") const concurrency = From 803ad3dbad1d2c670914da147d8105cd456f478d Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:09:38 +0000 Subject: [PATCH 2/4] docs(hub): clarify managed identity placeholder Co-authored-by: Ben Thomas <25218250+alliscode@users.noreply.github.com> Co-authored-by: Nishiki <1121087+nshki@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- hub/docs/artifact-storage.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hub/docs/artifact-storage.md b/hub/docs/artifact-storage.md index e3902751..ebd8ba74 100644 --- a/hub/docs/artifact-storage.md +++ b/hub/docs/artifact-storage.md @@ -17,10 +17,12 @@ Choose exactly one settings shape: provider: "azure-blob", accountUrl: "https://example.blob.core.windows.net", container: "artifacts", - managedIdentityClientId: "ee5eeebb-74e0-4b79-8030-c325e50fec43", // Optional user-assigned identity. + managedIdentityClientId: "", } ``` +Replace the placeholder with your user-assigned identity's actual client ID UUID, not its object ID or Azure resource ID. Omit `managedIdentityClientId` when using a system-assigned identity. + Alternatively, startup can call `createConfiguredArtifactStorage(environment)` from `server/domains/artifacts/create-configured-artifact-storage/index.ts`, supplying a read-only environment record explicitly. Neither domain factory reads `process.env`. | Environment variable | Selection | From 07d0f91c83d0d3f591dc1e16e35af26e8462fe7c Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:03:02 +0000 Subject: [PATCH 3/4] fix(hub): preserve committed artifact writes and precise results Co-authored-by: Ben Thomas <25218250+alliscode@users.noreply.github.com> Co-authored-by: Nishiki <1121087+nshki@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- hub/docs/artifact-storage.md | 4 +-- .../create-artifact-storage/index.ts | 5 ++- .../create-artifact-storage/types.ts | 18 ++++++++--- .../index.test.ts | 32 +++++++++++++++++++ .../index.ts | 13 ++++++-- 5 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.test.ts diff --git a/hub/docs/artifact-storage.md b/hub/docs/artifact-storage.md index ebd8ba74..9519515e 100644 --- a/hub/docs/artifact-storage.md +++ b/hub/docs/artifact-storage.md @@ -45,7 +45,7 @@ The Azure endpoint accepts public-cloud HTTPS account URLs only. The adapter use Keys are case-sensitive opaque strings of 1–128 ASCII letters, digits, underscores, or hyphens, beginning with a letter or digit. They are not paths or URLs. The domain rejects traversal, separators, percent encoding, and other characters before calling a provider. -Writes accept a `Uint8Array` of at most 32 MiB, including empty bytes, and never overwrite an existing key. Reads return a bounded `Uint8Array`. A successful operation returns `{ status: "ok" }`; a successful read also includes `bytes`. Failures return a typed `status`: `invalid-key`, `too-large`, `already-exists`, `not-found`, or `unavailable`. Missing reads and deletes are explicit `not-found` results, not empty bytes or silent success. Provider messages and credentials are not included in results. +Writes accept a `Uint8Array` of at most 32 MiB, including empty bytes, and never overwrite an existing key. Reads return a bounded `Uint8Array`. A successful operation returns `{ status: "ok" }`; a successful read also includes `bytes`. Each operation has its own result type: reads may fail with `invalid-key`, `too-large`, `not-found`, or `unavailable`; writes with `invalid-key`, `too-large`, `already-exists`, or `unavailable`; deletes with `invalid-key`, `not-found`, or `unavailable`. Missing reads and deletes are explicit `not-found` results, not empty bytes or silent success. Provider messages and credentials are not included in results. Immutability prevents overwriting bytes while a key exists. Deleting a key permits later reuse. Callers own authorization, key allocation, retention, and coordination between deletion and other work. @@ -53,7 +53,7 @@ Immutability prevents overwriting bytes while a key exists. Deleting a key permi Operators must provision an absolute directory before startup. The adapter supports Linux with `/proc/self/fd`, including the Hub container runtime. Other platforms fail safely rather than weaken path protection. No directory component or artifact file may be a symbolic link. Directory descriptors pin each component during an operation so replacing an ancestor cannot redirect that operation through a symlink. -Writes create a private staging file, flush the bytes, and publish it with an exclusive hard link. Readers see either the complete artifact or a missing key; concurrent writers cannot overwrite each other. Staging files are removed after operations. An abrupt process exit can leave a hidden staging file; operators may remove those files while the service is stopped. The adapter does not promise recovery across filesystem or host failure. +Writes create a private staging file, flush the bytes, and publish it with an exclusive hard link. Readers see either the complete artifact or a missing key; concurrent writers cannot overwrite each other. Publication commits the write: later staging cleanup failure produces a warning, not a failed write result. Cleanup also preserves an earlier publication failure. Cleanup failure or an abrupt process exit can leave a hidden staging file; operators may remove those files while the service is stopped. The adapter does not promise recovery across filesystem or host failure. Use a private service-owned directory and restrict local write access. Symlink protection does not make an attacker-writable storage directory safe: another local process could replace regular files or create hard links. Missing storage directories and other filesystem failures return `unavailable`, not a successful fallback. diff --git a/hub/server/domains/artifacts/create-artifact-storage/index.ts b/hub/server/domains/artifacts/create-artifact-storage/index.ts index ec3d8c4e..383abca7 100644 --- a/hub/server/domains/artifacts/create-artifact-storage/index.ts +++ b/hub/server/domains/artifacts/create-artifact-storage/index.ts @@ -1,5 +1,6 @@ import { createAzureBlobArtifactStorage } from "../../../integrations/azure-blob/create-azure-blob-artifact-storage/index.ts" import { createFilesystemArtifactStorage } from "../../../integrations/filesystem/create-filesystem-artifact-storage/index.ts" +import { log } from "../../../integrations/pino/log/index.ts" import { parseArtifactStorageSettings } from "../parse-artifact-storage-settings/index.ts" import type { ArtifactStorage, ArtifactStorageSettings } from "./types.ts" @@ -12,7 +13,9 @@ export async function createArtifactStorage( const settings = parseArtifactStorageSettings(input) const adapter = settings.provider === "filesystem" - ? await createFilesystemArtifactStorage(settings.directory, artifactMaxBytes) + ? await createFilesystemArtifactStorage(settings.directory, artifactMaxBytes, () => + log("warn", "Artifact staging cleanup failed"), + ) : createAzureBlobArtifactStorage(settings, artifactMaxBytes) return { diff --git a/hub/server/domains/artifacts/create-artifact-storage/types.ts b/hub/server/domains/artifacts/create-artifact-storage/types.ts index 7e6ab538..4918776b 100644 --- a/hub/server/domains/artifacts/create-artifact-storage/types.ts +++ b/hub/server/domains/artifacts/create-artifact-storage/types.ts @@ -7,12 +7,20 @@ export type ArtifactStorageSettings = managedIdentityClientId?: string } -export type ArtifactFailure = { - status: "invalid-key" | "too-large" | "not-found" | "already-exists" | "unavailable" +export type ArtifactReadResult = + | { status: "ok"; bytes: Uint8Array } + | { status: "invalid-key" | "too-large" | "not-found" | "unavailable" } + +export type ArtifactWriteResult = { + status: "ok" | "invalid-key" | "too-large" | "already-exists" | "unavailable" +} + +export type ArtifactDeleteResult = { + status: "ok" | "invalid-key" | "not-found" | "unavailable" } export interface ArtifactStorage { - read(key: string): Promise<{ status: "ok"; bytes: Uint8Array } | ArtifactFailure> - write(key: string, bytes: Uint8Array): Promise<{ status: "ok" } | ArtifactFailure> - delete(key: string): Promise<{ status: "ok" } | ArtifactFailure> + read(key: string): Promise + write(key: string, bytes: Uint8Array): Promise + delete(key: string): Promise } diff --git a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.test.ts b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.test.ts new file mode 100644 index 00000000..5f273cc6 --- /dev/null +++ b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.test.ts @@ -0,0 +1,32 @@ +import { randomUUID } from "node:crypto" +import { mkdir, readFile, rm } from "node:fs/promises" +import { resolve } from "node:path" +import { describe, expect, it, onTestFinished, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ unlink: vi.fn() })) +vi.mock("node:fs/promises", async (importOriginal) => ({ + ...(await importOriginal()), + unlink: mocks.unlink, +})) + +import { createFilesystemArtifactStorage } from "." + +describe("createFilesystemArtifactStorage()", () => { + it("preserves committed writes and publication failures when staging cleanup fails", async () => { + const directory = resolve(`server/integrations/filesystem/.storage-test-${randomUUID()}`) + await mkdir(directory) + onTestFinished(() => rm(directory, { recursive: true, force: true })) + const reportCleanupFailure = vi.fn() + const storage = await createFilesystemArtifactStorage(directory, 3, reportCleanupFailure) + mocks.unlink.mockRejectedValue(new Error("Private filesystem diagnostics")) + + expect(await storage.write("key", Uint8Array.of(1, 2))).toEqual({ status: "ok" }) + expect(await readFile(`${directory}/key`)).toEqual(Buffer.from([1, 2])) + expect(reportCleanupFailure).toHaveBeenCalledTimes(1) + + expect(await storage.write("key", Uint8Array.of(3))).toEqual({ status: "already-exists" }) + expect(await readFile(`${directory}/key`)).toEqual(Buffer.from([1, 2])) + expect(reportCleanupFailure).toHaveBeenCalledTimes(2) + expect(reportCleanupFailure.mock.calls).toEqual([[], []]) + }) +}) diff --git a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts index a23c63d9..57644ab7 100644 --- a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts +++ b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.ts @@ -8,6 +8,7 @@ import type { FilesystemArtifactStorage } from "./types.ts" export async function createFilesystemArtifactStorage( directory: string, maxBytes: number, + reportCleanupFailure: () => void, ): Promise { try { await withArtifactDirectory(directory, () => Promise.resolve(undefined)) @@ -66,8 +67,16 @@ export async function createFilesystemArtifactStorage( await link(staging, `${path}/${key}`) return { status: "ok" } as const } finally { - await handle.close() - await unlink(staging) + try { + await handle.close() + } catch { + reportCleanupFailure() + } + try { + await unlink(staging) + } catch { + reportCleanupFailure() + } } }) } catch (error) { From 4cba18324c260d9410e0249e4b9c576fe4f7e62c Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:22:33 +0000 Subject: [PATCH 4/4] test(hub): follow filesystem mock import conventions Co-authored-by: Ben Thomas <25218250+alliscode@users.noreply.github.com> Co-authored-by: Nishiki <1121087+nshki@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../create-filesystem-artifact-storage/index.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.test.ts b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.test.ts index 5f273cc6..d03af267 100644 --- a/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.test.ts +++ b/hub/server/integrations/filesystem/create-filesystem-artifact-storage/index.test.ts @@ -1,11 +1,12 @@ import { randomUUID } from "node:crypto" +import type * as fs from "node:fs/promises" import { mkdir, readFile, rm } from "node:fs/promises" import { resolve } from "node:path" import { describe, expect, it, onTestFinished, vi } from "vitest" const mocks = vi.hoisted(() => ({ unlink: vi.fn() })) vi.mock("node:fs/promises", async (importOriginal) => ({ - ...(await importOriginal()), + ...(await importOriginal()), unlink: mocks.unlink, }))