retention interval

project/server/make/dev.mk

@@ -53,6 +53,7 @@ repl: build
 repl-nest: build
 	@ \
 		SIGNING_VOUCHER=$(shell $(call secret,voucher,signing-key)) \
+		SCHEDULER_ENABLE='false' \
 		node --enable-source-maps $(SERVER_OUT_REPL_NEST)
 
 start-container:

project/server/src/config/index.ts

@@ -52,6 +52,8 @@ export const config = () =>
 				/** how long a voucher is valid for — in seconds */
 				ttl: env.integer(required("SNAPSHOT_VOUCHER_TTL", floor(60 * 60 * 2))),
 			},
+			/** subsequent days with no submissions after which all data pertaining to a submitting instance is deleted */
+			revokeAfter: env.integer(required("SNAPSHOT_REVOKE_AFTER", floor(60))),
 			defer: {
 				target: env.choice(Schema.Enums(SnapshotDeferTarget))(
 					required("SNAPSHOT_DEFER_TARGET", SnapshotDeferTarget.None),

project/server/src/layer/scheduler/scheduled/compaction/optimize.module.ts

@@ -0,0 +1,11 @@
+import { Module } from "@nestjs/common";
+
+import { ModuleDatabase } from "../../../database/database.module";
+import { ServiceSchedulerScheduledCompactionOptimize } from "./optimize.service";
+
+@Module({
+	imports: [ModuleDatabase],
+	providers: [ServiceSchedulerScheduledCompactionOptimize],
+	exports: [ServiceSchedulerScheduledCompactionOptimize],
+})
+export class ModuleSchedulerScheduledCompactionOptimize {}

project/server/src/layer/scheduler/scheduled/compaction/optimize.service.ts

@@ -0,0 +1,45 @@
+import { Inject, Injectable } from "@nestjs/common";
+
+import { DatabaseStaging } from "../../../database/database.module";
+import { ServiceSchedulerScheduledCompactionVacuum } from "./vacuum.service";
+
+import type { IDatabase } from "../../../../service/database";
+import type { SchedulerScheduled } from "../../../../service/scheduler/base";
+
+@Injectable()
+export class ServiceSchedulerScheduledCompactionOptimize
+	implements
+		SchedulerScheduled<typeof ServiceSchedulerScheduledCompactionOptimize>
+{
+	static readonly id = Symbol("ServiceSchedulerScheduledCompactionOptimize");
+
+	static readonly prerequisites = [
+		ServiceSchedulerScheduledCompactionVacuum.id,
+	];
+	static readonly schedule = {
+		minute: "0",
+		hour: "0",
+	} as const;
+
+	constructor(@Inject(DatabaseStaging) private db: IDatabase<"staging">) {}
+
+	async run(): Promise<void> {
+		await this.db.begin("w", async (t) => {
+			t.run({
+				/*
+				https://sqlite.org/lang_analyze.html#periodically_run_pragma_optimize_
+				"The PRAGMA optimize command will normally only consider running ANALYZE on tables that have been previously queried by the same database connection or that do not have entries in the sqlite_stat1 table.
+        However, if the 0x10000 bit is added to the argument, PRAGMA optimize will examine all tables to see if they can benefit from ANALYZE, not just those that have been recently queried."
+        */
+				query: "pragma optimize=0x10000;",
+				name: "Optimize",
+				parameters: [],
+				database: "staging",
+				connectionMode: "w",
+				integerMode: "number",
+				resultMode: "one",
+				rowMode: "object",
+			});
+		});
+	}
+}

project/server/src/layer/scheduler/scheduled/compaction/orphaned/attached.module.ts

@@ -0,0 +1,11 @@
+import { Module } from "@nestjs/common";
+
+import { ModuleDatabase } from "../../../../database/database.module";
+import { ServiceSchedulerScheduledCompactionOrphanedAttached } from "./attached.service";
+
+@Module({
+	imports: [ModuleDatabase],
+	providers: [ServiceSchedulerScheduledCompactionOrphanedAttached],
+	exports: [ServiceSchedulerScheduledCompactionOrphanedAttached],
+})
+export class ModuleSchedulerScheduledCompactionOrphanedAttached {}

project/server/src/layer/scheduler/scheduled/compaction/orphaned/attached.service.test.ts

@@ -0,0 +1,200 @@
+import { randomBytes } from "node:crypto";
+import { type TestContext, test } from "node:test";
+
+import { testDatabase } from "../../../../../service/database/utility";
+import { StubIntrospection } from "../../../../../service/introspect/stub";
+import { Snapshot } from "../../../../../service/snapshot";
+import { Voucher } from "../../../../../service/voucher";
+import { floor } from "../../../../../type/codec/integer";
+import { uuid } from "../../../../../type/codec/uuid";
+import { unroll } from "../../../../../utility/iterable";
+import { omit } from "../../../../../utility/omit";
+import { ServiceSchedulerScheduledCompactionOrphanedAttached } from "./attached.service";
+
+import type { IDatabase } from "../../../../../service/database";
+
+const buildSnapshot = (database: IDatabase<"staging">) =>
+	new Snapshot(
+		database,
+		new StubIntrospection(),
+		new Voucher(randomBytes(64).toString()),
+		{
+			voucher: {
+				expectedAfter: floor(60 * 60 * 23),
+				ttl: floor(60 * 60 * 2),
+			},
+		},
+	);
+
+const device1 = {
+	entry_type: null,
+	has_configuration_url: false,
+	hw_version: null,
+	manufacturer: "Signify Netherlands B.V.",
+	model: "Hue white lamp",
+	model_id: "LWB010",
+	sw_version: "1.116.3",
+	via_device: null,
+} as const;
+
+const device2 = {
+	entry_type: null,
+	has_configuration_url: false,
+	hw_version: null,
+	manufacturer: "Signify Netherlands B.V.",
+	model: "Hue green lamp",
+	model_id: "LWB011",
+	sw_version: "1.116.4",
+	via_device: null,
+} as const;
+
+const entity1 = {
+	assumed_state: false,
+	domain: "light",
+	entity_category: null,
+	has_entity_name: true,
+	original_device_class: null,
+	unit_of_measurement: null,
+} as const;
+
+test("orphaned attachable", async (t: TestContext) => {
+	await using database = await testDatabase("staging", true);
+
+	const snapshot = buildSnapshot(database);
+	const service = new ServiceSchedulerScheduledCompactionOrphanedAttached(
+		database,
+	);
+
+	const subjectA = uuid();
+	{
+		const created = await snapshot.create(snapshot.voucher.initial(subjectA));
+		t.assert.strictEqual(created.kind, "success");
+
+		await snapshot.attach.device(created.handle, "foo", device1, [entity1]);
+		await snapshot.attach.device(created.handle, "bar", device2, [entity1]);
+
+		await snapshot.finalize(
+			created.handle,
+			{
+				version: 1,
+				hash: Buffer.alloc(32, 0xaa),
+			},
+			"2025.3.1",
+		);
+	}
+
+	const subjectB = uuid();
+	{
+		const created = await snapshot.create(snapshot.voucher.initial(subjectB));
+		t.assert.strictEqual(created.kind, "success");
+
+		await snapshot.attach.device(created.handle, "foo", device1, []);
+
+		await snapshot.finalize(
+			created.handle,
+			{
+				version: 1,
+				hash: Buffer.alloc(32, 0xab),
+			},
+			"2025.3.1",
+		);
+	}
+
+	t.assert.deepStrictEqual(
+		(await unroll(snapshot.staging.devices({ integration: "foo" }))).map(
+			(item) => omit(item, "id"),
+		),
+		[
+			{
+				integration: "foo",
+				manufacturer: device1.manufacturer,
+				model: device1.model,
+				modelId: device1.model_id,
+			},
+		],
+	);
+	t.assert.deepStrictEqual(
+		(await unroll(snapshot.staging.devices({ integration: "bar" }))).map(
+			(item) => omit(item, "id"),
+		),
+		[
+			{
+				integration: "bar",
+				manufacturer: device2.manufacturer,
+				model: device2.model,
+				modelId: device2.model_id,
+			},
+		],
+	);
+	t.assert.deepStrictEqual(
+		(await unroll(snapshot.staging.entities({ domain: "light" }))).map((item) =>
+			omit(item, "id"),
+		),
+		[
+			{
+				domain: entity1.domain,
+				assumedState: entity1.assumed_state,
+				hasName: entity1.has_entity_name,
+				category: entity1.entity_category ?? undefined,
+				originalDeviceClass: entity1.original_device_class ?? undefined,
+				unitOfMeasurement: entity1.unit_of_measurement ?? undefined,
+			},
+		],
+	);
+
+	let attributionA;
+	{
+		const all = await unroll(
+			snapshot.staging.attribution.submissions({ subject: subjectA }),
+		);
+		t.assert.deepEqual(all.length, 1);
+		attributionA = all[0];
+	}
+
+	await snapshot.delete(attributionA.submissionId);
+
+	await service.run();
+
+	t.assert.deepStrictEqual(
+		(await unroll(snapshot.staging.devices({ integration: "foo" }))).map(
+			(item) => omit(item, "id"),
+		),
+		[
+			{
+				integration: "foo",
+				manufacturer: device1.manufacturer,
+				model: device1.model,
+				modelId: device1.model_id,
+			},
+		],
+	);
+	t.assert.deepStrictEqual(
+		await unroll(snapshot.staging.devices({ integration: "bar" })),
+		[],
+	);
+	t.assert.deepStrictEqual(
+		await unroll(snapshot.staging.entities({ domain: "light" })),
+		[],
+	);
+
+	let attributionB;
+	{
+		const all = await unroll(
+			snapshot.staging.attribution.submissions({ subject: subjectB }),
+		);
+		t.assert.deepEqual(all.length, 1);
+		attributionB = all[0];
+	}
+	await snapshot.delete(attributionB.submissionId);
+
+	await service.run();
+
+	t.assert.deepStrictEqual(
+		await unroll(snapshot.staging.devices({ integration: "foo" })),
+		[],
+	);
+	t.assert.deepStrictEqual(
+		await unroll(snapshot.staging.devices({ integration: "bar" })),
+		[],
+	);
+});

project/server/src/layer/scheduler/scheduled/compaction/orphaned/attached.service.ts

@@ -0,0 +1,107 @@
+import { Inject, Injectable } from "@nestjs/common";
+
+import { logger as parentLogger } from "../../../../../logger";
+import {
+	deleteOrphanedDevice,
+	deleteOrphanedDevicePermutation,
+	deleteOrphanedDevicePermutationLink,
+	deleteOrphanedEntity,
+	deleteOrphanedEntitySet,
+	deleteOrphanedEntitySetContent,
+} from "../../../../../service/database/query/staging/snapshot-delete";
+import { formatNs } from "../../../../../utility/format";
+import { timed } from "../../../../../utility/timed";
+import { DatabaseStaging } from "../../../../database/database.module";
+import { ServiceSchedulerScheduledCompactionOrphanedSubmission } from "./submission.service";
+
+import type { IDatabase } from "../../../../../service/database";
+import type { SchedulerScheduled } from "../../../../../service/scheduler/base";
+
+const logger = parentLogger.child({
+	label: "scheduler-scheduled-compaction-orphaned-attached",
+});
+
+@Injectable()
+/** required in addition to {@link ServiceSchedulerScheduledCompactionOrphanedSubmission}
+ * the former deletes submissions and accompanying attributions, but not actual devices / device permutations / entities that have become orphaned */
+export class ServiceSchedulerScheduledCompactionOrphanedAttached
+	implements
+		SchedulerScheduled<
+			typeof ServiceSchedulerScheduledCompactionOrphanedAttached
+		>
+{
+	static readonly id = Symbol(
+		"ServiceSchedulerScheduledCompactionOrphanedAttached",
+	);
+
+	static readonly prerequisites = [
+		ServiceSchedulerScheduledCompactionOrphanedSubmission.id,
+	];
+
+	constructor(@Inject(DatabaseStaging) private db: IDatabase<"staging">) {}
+
+	async run(): Promise<void> {
+		await this.db.begin("w", async (t) => {
+			// run before device permutation deletion to prevent expensive cascading deletes
+			{
+				const took = await timed(() =>
+					t.run(deleteOrphanedEntitySetContent.bind.anonymous([])),
+				);
+				logger.info(`pruned set entity content in ${formatNs(took)}s`, {
+					took,
+					table: "snapshot_submission_set_content_entity_device_permutation",
+				});
+			}
+
+			{
+				const took = await timed(() =>
+					t.run(deleteOrphanedEntitySet.bind.anonymous([])),
+				);
+				logger.info(`pruned set entity descriptors in ${formatNs(took)}s`, {
+					took,
+					table: "snapshot_submission_set_entity_device_permutation",
+				});
+			}
+
+			{
+				const took = await timed(() =>
+					t.run(deleteOrphanedEntity.bind.anonymous([])),
+				);
+				logger.info(`pruned entities in ${formatNs(took)}s`, {
+					took,
+					table: "snapshot_submission_entity",
+				});
+			}
+
+			{
+				const took = await timed(() =>
+					t.run(deleteOrphanedDevicePermutationLink.bind.anonymous([])),
+				);
+				logger.info(`pruned device permutation links in ${formatNs(took)}s`, {
+					took,
+					table: "snapshot_submission_device_permutation_link",
+				});
+			}
+
+			{
+				const took = await timed(() =>
+					t.run(deleteOrphanedDevicePermutation.bind.anonymous([])),
+				);
+				logger.info(`pruned device permutations in ${formatNs(took)}s`, {
+					took,
+					table: "snapshot_submission_device_permutation",
+				});
+			}
+
+			{
+				const took = await timed(() =>
+					t.run(deleteOrphanedDevice.bind.anonymous([])),
+				);
+				logger.info(`pruned devices in ${formatNs(took)}s`, {
+					took,
+					table: "snapshot_submission_device",
+				});
+			}
+		});
+	}
+}