feat: full layer deletion flow (MAPCO-7285)

.gitignore

@@ -113,3 +113,12 @@ dist
 #helm
 Chart.lock
 
+.claude
+.claude-flow
+.swarm
+.mcp.json
+CLAUDE.md
+/ai-docs
+# ruflo / claude-flow vector memory store
+ruvector.db
+/docs

config/custom-environment-variables.json

@@ -67,7 +67,8 @@
     "accessKeyId": "S3_ACCESS_KEY_ID",
     "secretAccessKey": "S3_SECRET_ACCESS_KEY",
     "endpointUrl": "S3_ENDPOINT_URL",
-    "bucket": "S3_ARTIFACTS_BUCKET",
+    "artifactsBucket": "S3_ARTIFACTS_BUCKET",
+    "tilesBucket": "S3_TILES_BUCKET",
     "sslEnabled": {
       "__name": "S3_SSL_ENABLED",
       "__format": "boolean"
@@ -124,7 +125,8 @@
       "tasks": {
         "createTasks": "POLLING_CREATE_TASKS_TASK",
         "finalize": "POLLING_FINALIZE_TASK",
-        "init": "POLLING_INIT_TASK"
+        "init": "POLLING_INIT_TASK",
+        "delete": "POLLING_DELETE_TASK"
       }
     },
     "ingestion": {
@@ -137,6 +139,9 @@
         },
         "swapUpdate": {
           "type": "INGESTION_SWAP_UPDATE_JOB_TYPE"
+        },
+        "deleteLayer": {
+          "type": "INGESTION_DELETE_LAYER_JOB_TYPE"
         }
       },
       "jobs": {

config/default.json

@@ -48,7 +48,8 @@
     "accessKeyId": "minioadmin",
     "secretAccessKey": "minioadmin",
     "endpointUrl": "http://localhost:9000",
-    "bucket": "",
+    "artifactsBucket": "",
+    "tilesBucket": "",
     "sslEnabled": false
   },
   "disableHttpClientLogs": true,
@@ -91,7 +92,8 @@
       "tasks": {
         "createTasks": "create-tasks",
         "init": "init",
-        "finalize": "finalize"
+        "finalize": "finalize",
+        "delete": "delete"
       }
     },
     "ingestion": {
@@ -104,6 +106,9 @@
         },
         "swapUpdate": {
           "type": "Ingestion_Swap_Update"
+        },
+        "deleteLayer": {
+          "type": "Delete_Layer"
         }
       },
       "jobs": {

helm/templates/configmap.yaml

@@ -33,6 +33,9 @@ data:
   S3_ARTIFACTS_BUCKET: {{ $storage.s3.artifactsBucket | quote }}
   S3_SSL_ENABLED: {{ $storage.s3.sslEnabled | quote }}
   {{- end }}
+  {{- if eq (upper $storage.tilesStorageProvider) "S3" }}
+  S3_TILES_BUCKET: {{ $storage.s3.tilesBucket | quote }}
+  {{- end }}
   npm_config_cache: /tmp/
 
   HTTP_RETRY_ATTEMPTS: {{ .Values.env.httpRetry.attempts | quote }}
@@ -49,9 +52,11 @@ data:
   POLLING_CREATE_TASKS_TASK: {{ $jobDefinitions.tasks.createTasks.type | quote }}
   POLLING_INIT_TASK: {{ $jobDefinitions.tasks.init.type | quote }}
   POLLING_FINALIZE_TASK: {{ $jobDefinitions.tasks.finalize.type | quote }}
+  POLLING_DELETE_TASK: {{ $jobDefinitions.tasks.delete.type | quote }}
   INGESTION_NEW_JOB_TYPE: {{ $jobDefinitions.jobs.new.type | quote }}
   INGESTION_UPDATE_JOB_TYPE: {{ $jobDefinitions.jobs.update.type | quote }}
   INGESTION_SWAP_UPDATE_JOB_TYPE: {{ $jobDefinitions.jobs.swapUpdate.type | quote }}
+  INGESTION_DELETE_LAYER_JOB_TYPE: {{ $jobDefinitions.jobs.deleteLayer.type | quote }}
   INGESTION_SEED_JOB_TYPE : {{ $jobDefinitions.jobs.seed.type | quote }}
   EXPORT_JOB_TYPE: {{ $jobDefinitions.jobs.export.type | quote }}
   EXPORT_CLEANUP_EXPIRATION_DAYS: {{ $jobDefinitions.jobs.export.cleanupExpirationDays | quote }}

helm/values.yaml

@@ -87,6 +87,7 @@ storage:
   s3:
     endpointUrl: ""
     artifactsBucket: ""
+    tilesBucket: ""
     sslEnabled: false
     secretName: ""
   fs:
@@ -137,6 +138,8 @@ jobDefinitions:
       type: ""
     swapUpdate:
       type: ""
+    deleteLayer:
+      type: ""
     seed:
       type: ""
     export:
@@ -150,6 +153,8 @@ jobDefinitions:
       type: ""
     finalize:
       type: ""
+    delete:
+      type: ""
     merge:
       type: ""
       tileBatchSize: 10000

package.json

@@ -48,7 +48,7 @@
     "@map-colonies/mc-priority-queue": "^9.1.2",
     "@map-colonies/mc-utils": "^6.0.1",
     "@map-colonies/prometheus": "^1.0.0",
-    "@map-colonies/raster-shared": "8.3.0-alpha.0",
+    "@map-colonies/raster-shared": "^8.3.0-alpha.1",
     "@map-colonies/read-pkg": "^1.0.0",
     "@map-colonies/schemas": "^1.18.0",
     "@map-colonies/shapefile-reader": "^1.0.1",

src/common/errors.ts

@@ -51,6 +51,14 @@ export class UpdateLayerError extends Error {
   }
 }
 
+export class DeleteLayerError extends Error {
+  public constructor(targetClient: string, layerIdentifier: string, err: Error) {
+    super(`Failed to delete layer ${layerIdentifier} from ${targetClient} client: ${err.message}`);
+    this.name = DeleteLayerError.name;
+    this.stack = err.stack;
+  }
+}
+
 export class LayerNotFoundError extends Error {
   public constructor(id: string) {
     super(`Record with id ${id} not found`);

src/common/interfaces.ts

@@ -67,6 +67,7 @@ export interface IngestionPollingJobsConfig {
   new: JobConfig | undefined;
   update: JobConfig | undefined;
   swapUpdate: JobConfig | undefined;
+  deleteLayer: JobConfig | undefined;
 }
 
 export interface ExportPollingJobsConfig {
@@ -90,6 +91,7 @@ export interface PollingTasks {
   createTasks: string;
   init: string;
   finalize: string;
+  delete: string;
 }
 
 export interface PollingConfig {
@@ -137,9 +139,17 @@ export interface JobManagementConfig {
 //#endregion config
 
 //#region job/task interfaces
-export interface IJobHandler<TInitJob = unknown, TInitTask = unknown, TFinalizeJob = unknown, TFinalizeTask = unknown> {
-  handleJobInit: (job: TInitJob, task: TInitTask) => Promise<void>;
-  handleJobFinalize: (job: TFinalizeJob, task: TFinalizeTask) => Promise<void>;
+export interface IJobHandler<
+  TInitJob = unknown,
+  TInitTask = unknown,
+  TFinalizeJob = unknown,
+  TFinalizeTask = unknown,
+  TDeleteJob = unknown,
+  TDeleteTask = unknown,
+> {
+  handleJobInit?: (job: TInitJob, task: TInitTask) => Promise<void>;
+  handleJobFinalize?: (job: TFinalizeJob, task: TFinalizeTask) => Promise<void>;
+  handleJobDelete?: (job: TDeleteJob, task: TDeleteTask) => Promise<void>;
 }
 
 export interface JobAndTaskResponse {
@@ -327,7 +337,8 @@ export interface GetMapproxyCacheRequest {
 
 export interface GetMapproxyCacheResponse {
   cacheName: string;
-  cache: { type: LayerCacheType };
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  cache: { type: LayerCacheType; directory?: string; directory_layout?: string; bucket_name?: string };
 }
 //#endregion mapproxyApi
 
@@ -412,7 +423,8 @@ export interface IS3Config {
   accessKeyId: string;
   secretAccessKey: string;
   endpointUrl: string;
-  bucket: string;
+  artifactsBucket: string;
+  tilesBucket: string;
   objectKey: string;
   sslEnabled: boolean;
 }

src/containerConfig.ts

@@ -18,6 +18,7 @@ import { ExportJobHandler } from './job/models/export/exportJobHandler';
 import { NewJobHandler } from './job/models/ingestion/newJobHandler';
 import { SwapJobHandler } from './job/models/ingestion/swapJobHandler';
 import { UpdateJobHandler } from './job/models/ingestion/updateJobHandler';
+import { DeleteLayerHandler } from './job/models/deletion/deleteLayerHandler';
 import { JOB_HANDLER_FACTORY_SYMBOL, jobHandlerFactory } from './job/models/jobHandlerFactory';
 import { getPollingJobs, parseInstanceType, validateAndGetHandlersTokens } from './utils/configUtil';
 import { productReaderFactory } from './utils/storage/productReader';
@@ -71,6 +72,8 @@ const registerInstanceHandlers = (instanceType: InstanceType, handlersTokens: Re
         { token: handlersTokens['Ingestion_Update']!, provider: { useClass: UpdateJobHandler } },
 
         { token: handlersTokens['Ingestion_Swap_Update']!, provider: { useClass: SwapJobHandler } },
+
+        { token: handlersTokens['Delete_Layer']!, provider: { useClass: DeleteLayerHandler } },
       ];
     case 'export':
       return [{ token: handlersTokens['Export']!, provider: { useClass: ExportJobHandler } }];

src/httpClients/catalogClient.ts

@@ -2,14 +2,15 @@ import type { Logger } from '@map-colonies/js-logger';
 import { LayerNameFormats, PolygonPartsEntityName, type LayerName } from '@map-colonies/raster-shared';
 import { HttpClient, type IHttpRetryConfig } from '@map-colonies/mc-utils';
 import { type IRasterCatalogUpsertRequestBody, LayerMetadata, Link, PycswLayerCatalogRecord } from '@map-colonies/mc-model-types';
+import { NotFoundError } from '@map-colonies/error-types';
 import { RecordType } from '@map-colonies/types';
 import { context, SpanStatusCode, trace, type Tracer } from '@opentelemetry/api';
 import { inject, injectable } from 'tsyringe';
 import type { IConfig, CatalogUpdateRequestBody, FindLayerResponse, FindLayerBody } from '../common/interfaces';
 import { IngestionNewFinalizeJob, IngestionSwapUpdateFinalizeJob, IngestionUpdateFinalizeJob } from '../utils/zod/schemas/job.schema';
 import { SERVICES } from '../common/constants';
 import { internalIdSchema } from '../utils/zod/schemas/jobParameters.schema';
-import { LayerNotFoundError, PublishLayerError, UpdateLayerError } from '../common/errors';
+import { DeleteLayerError, LayerNotFoundError, PublishLayerError, UpdateLayerError } from '../common/errors';
 import { LinkBuilder, type ILinkBuilderData } from '../utils/linkBuilder';
 import { PolygonPartsMangerClient } from './polygonPartsMangerClient';
 
@@ -86,6 +87,32 @@ export class CatalogClient extends HttpClient {
     });
   }
 
+  public async deleteRecord(catalogId: string): Promise<void> {
+    await context.with(trace.setSpan(context.active(), this.tracer.startSpan(`${CatalogClient.name}.${this.deleteRecord.name}`)), async () => {
+      const activeSpan = trace.getActiveSpan();
+      activeSpan?.setAttribute('catalogId', catalogId);
+
+      try {
+        const url = `/records/${catalogId}`;
+        await this.delete(url);
+        activeSpan?.setStatus({ code: SpanStatusCode.OK, message: 'Catalog record deleted successfully' });
+      } catch (err) {
+        if (err instanceof NotFoundError) {
+          this.logger.warn({ msg: 'catalog record not found, skipping', catalogId });
+          activeSpan?.setStatus({ code: SpanStatusCode.OK, message: 'catalog record not found, skipping' });
+          return;
+        }
+        if (err instanceof Error) {
+          activeSpan?.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
+          activeSpan?.recordException(err);
+          throw new DeleteLayerError(this.targetService, catalogId, err);
+        }
+      } finally {
+        activeSpan?.end();
+      }
+    });
+  }
+
   public async findLayer(id: string): Promise<FindLayerResponse> {
     // eslint-disable-next-line @typescript-eslint/return-await
     return await context.with(trace.setSpan(context.active(), this.tracer.startSpan(`${CatalogClient.name}.${this.findLayer.name}`)), async () => {

src/httpClients/geoserverClient.ts

@@ -3,9 +3,10 @@ import type { LayerNameFormats } from '@map-colonies/raster-shared';
 import { HttpClient, type IHttpRetryConfig } from '@map-colonies/mc-utils';
 import { context, SpanStatusCode, trace, type Tracer } from '@opentelemetry/api';
 import { inject, injectable } from 'tsyringe';
+import { NotFoundError } from '@map-colonies/error-types';
 import type { IConfig, InsertGeoserverRequest } from '../common/interfaces';
 import { SERVICES } from '../common/constants';
-import { PublishLayerError } from '../common/errors';
+import { DeleteLayerError, PublishLayerError } from '../common/errors';
 
 @injectable()
 export class GeoserverClient extends HttpClient {
@@ -52,4 +53,31 @@ export class GeoserverClient extends HttpClient {
       }
     });
   }
+
+  public async unpublishLayer(layerName: string): Promise<void> {
+    await context.with(trace.setSpan(context.active(), this.tracer.startSpan(`${GeoserverClient.name}.${this.unpublishLayer.name}`)), async () => {
+      const activeSpan = trace.getActiveSpan();
+      activeSpan?.setAttribute('layerName', layerName);
+
+      try {
+        const url = `/featureTypes/${this.workspace}/${this.dataStore}/${layerName}`;
+        await this.delete(url, { isRecursive: true });
+        activeSpan?.setStatus({ code: SpanStatusCode.OK, message: 'Layer unpublished successfully from geoserver' });
+      } catch (err) {
+        if (err instanceof NotFoundError) {
+          // already gone — unpublish is idempotent, a 404 on (re)run is success (§6)
+          this.logger.warn({ msg: 'Layer feature type not found in geoserver, skipping', layerName });
+          activeSpan?.setStatus({ code: SpanStatusCode.OK, message: 'Layer feature type not found in geoserver, skipping' });
+          return;
+        }
+        if (err instanceof Error) {
+          activeSpan?.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
+          activeSpan?.recordException(err);
+          throw new DeleteLayerError(this.targetService, layerName, err);
+        }
+      } finally {
+        activeSpan?.end();
+      }
+    });
+  }
 }