From 4ba52dffb37017a506eba38e678a62ee67a5151f Mon Sep 17 00:00:00 2001 From: Asif Date: Wed, 15 Jul 2026 15:05:10 +0300 Subject: [PATCH 1/5] fix: planet-dumper --- schemas/vector/planetDumper/v1.configs.json | 19 ++ schemas/vector/planetDumper/v1.schema.json | 307 ++++++++++++++++++++ 2 files changed, 326 insertions(+) create mode 100644 schemas/vector/planetDumper/v1.configs.json create mode 100644 schemas/vector/planetDumper/v1.schema.json diff --git a/schemas/vector/planetDumper/v1.configs.json b/schemas/vector/planetDumper/v1.configs.json new file mode 100644 index 00000000..c7d03977 --- /dev/null +++ b/schemas/vector/planetDumper/v1.configs.json @@ -0,0 +1,19 @@ +[ + { + "name": "vector-planet-dumper", + "value": { + "telemetry": { + "shared": { "serviceName": "planet-dumper" }, + "tracing": { "isEnabled": false }, + "logger": { "level": "info", "prettyPrint": false } + }, + "httpClient": { "timeout": 1000 }, + "postgres": { "enableSslAuth": false }, + "pgDump": { "verbose": false }, + "ngDump": { "verbose": false, "maxConcurrency": 4 }, + "osmium": { "verbose": false, "progress": false }, + "s3": { "upload": { "concurrency": 4, "partSize": 5 } }, + "arstotzka": { "enabled": false } + } + } +] diff --git a/schemas/vector/planetDumper/v1.schema.json b/schemas/vector/planetDumper/v1.schema.json new file mode 100644 index 00000000..10784f37 --- /dev/null +++ b/schemas/vector/planetDumper/v1.schema.json @@ -0,0 +1,307 @@ +{ + "$id": "https://mapcolonies.com/vector/planetDumper/v1", + "type": "object", + "title": "vectorPlanetDumperV1", + "description": "Vector's planet-dumper schema", + "required": ["telemetry", "httpClient", "postgres", "pgDump", "ngDump", "osmium", "s3", "arstotzka"], + "properties": { + "telemetry": { + "description": "Telemetry configuration", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "examples": [ + { + "shared": { "serviceName": "planet-dumper" }, + "tracing": { "isEnabled": false }, + "logger": { "level": "info", "prettyPrint": false } + } + ], + "properties": { + "shared": { + "$ref": "https://mapcolonies.com/common/telemetry/base/v1" + }, + "tracing": { + "$ref": "https://mapcolonies.com/common/telemetry/tracing/v1" + }, + "logger": { + "$ref": "https://mapcolonies.com/common/telemetry/logger/v2" + }, + "metrics": { + "description": "Metrics configuration" + } + }, + "required": ["logger", "shared", "tracing"] + }, + "httpClient": { + "description": "Configuration for the internal http client used to talk to the dump-server and to fetch remote replication state files", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "required": ["timeout"], + "properties": { + "timeout": { + "type": "integer", + "description": "The http client timeout duration in milliseconds", + "default": 1000, + "exclusiveMinimum": 0, + "x-env-value": "HTTP_CLIENT_TIMEOUT" + } + } + }, + "postgres": { + "description": "Configuration for connecting to the source OSM postgres database. Note: host/port/user/password/database themselves are intentionally not part of this schema - pg_dump and psql are invoked as subprocesses and read those natively from the standard PGHOST/PGPORT/PGUSER/PGPASSWORD/PGDATABASE environment variables", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "required": ["enableSslAuth"], + "properties": { + "enableSslAuth": { + "type": "boolean", + "description": "Whether to authenticate to postgres using an SSL client certificate", + "default": false, + "x-env-value": "POSTGRES_ENABLE_SSL_AUTH" + }, + "sslPaths": { + "type": "object", + "description": "Paths to the SSL certificate files, required when enableSslAuth is true", + "default": {}, + "unevaluatedProperties": false, + "properties": { + "ca": { + "type": "string", + "description": "Path to the root CA certificate file", + "x-env-value": "POSTGRES_CA_PATH" + }, + "cert": { + "type": "string", + "description": "Path to the client certificate file", + "x-env-value": "POSTGRES_CERT_PATH" + }, + "key": { + "type": "string", + "description": "Path to the client certificate key file", + "x-env-value": "POSTGRES_KEY_PATH" + } + } + } + }, + "if": { + "properties": { + "enableSslAuth": { + "const": true + } + } + }, + "then": { + "required": ["sslPaths"], + "properties": { + "sslPaths": { + "type": "object", + "required": ["ca", "cert", "key"] + } + } + } + }, + "pgDump": { + "description": "pg_dump command configuration", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "required": ["verbose"], + "properties": { + "verbose": { + "type": "boolean", + "description": "Whether to run pg_dump in verbose mode", + "default": false, + "x-env-value": "PG_DUMP_VERBOSE" + } + } + }, + "ngDump": { + "description": "planet-dump-ng command configuration", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "required": ["verbose"], + "properties": { + "verbose": { + "type": "boolean", + "description": "Whether to run planet-dump-ng in verbose mode", + "default": false, + "x-env-value": "NG_DUMP_VERBOSE" + }, + "maxConcurrency": { + "type": "integer", + "description": "Maximum number of disk writing threads planet-dump-ng will run for each table", + "exclusiveMinimum": 0, + "x-env-value": "NG_DUMP_MAX_CONCURRENCY" + } + } + }, + "osmium": { + "description": "osmium fileinfo command configuration", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "required": ["verbose", "progress"], + "properties": { + "verbose": { + "type": "boolean", + "description": "Whether to run osmium in verbose mode", + "default": false, + "x-env-value": "OSMIUM_VERBOSE" + }, + "progress": { + "type": "boolean", + "description": "Whether to show a progress bar while osmium runs", + "default": false, + "x-env-value": "OSMIUM_PROGRESS" + } + } + }, + "s3": { + "description": "S3 upload tuning. Bucket/endpoint/acl are provided per-invocation as CLI arguments rather than static config, since they can differ per run", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "required": ["upload"], + "properties": { + "upload": { + "type": "object", + "description": "Multipart upload tuning for the created dump file", + "default": {}, + "unevaluatedProperties": false, + "required": ["concurrency", "partSize"], + "properties": { + "concurrency": { + "type": "integer", + "description": "Number of concurrent parts to upload at once", + "default": 4, + "exclusiveMinimum": 0, + "x-env-value": "S3_UPLOAD_CONCURRENCY" + }, + "partSize": { + "type": "integer", + "description": "The size in MB of each uploaded part", + "default": 5, + "exclusiveMinimum": 0, + "x-env-value": "S3_UPLOAD_PART_SIZE" + } + } + } + } + }, + "arstotzka": { + "description": "Optional distributed locking and action tracking integration, active around the pg_dump and ng_dump pipeline stages", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "required": ["enabled"], + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether the arstotzka integration is enabled", + "default": false, + "x-env-value": "ARSTOTZKA_ENABLED" + }, + "services": { + "type": "object", + "description": "The arstotzka service ids registered for each pipeline stage this app tracks", + "default": {}, + "unevaluatedProperties": false, + "properties": { + "planetDumperPg": { + "type": "string", + "description": "The registered arstotzka service id for the pg_dump stage", + "x-env-value": "ARSTOTZKA_SERVICE_PG" + }, + "planetDumperNg": { + "type": "string", + "description": "The registered arstotzka service id for the ng_dump stage", + "x-env-value": "ARSTOTZKA_SERVICE_NG" + } + } + }, + "mediator": { + "type": "object", + "description": "Configuration for the arstotzka mediator client", + "default": {}, + "unevaluatedProperties": false, + "properties": { + "timeout": { + "type": "integer", + "description": "The mediator http client timeout duration in milliseconds", + "exclusiveMinimum": 0, + "x-env-value": "MEDIATOR_TIMEOUT" + }, + "enableRetryStrategy": { + "type": "boolean", + "description": "Whether to enable the mediator client's retry strategy", + "default": false, + "x-env-value": "MEDIATOR_ENABLE_RETRY_STRATEGY" + }, + "retryStrategy": { + "type": "object", + "description": "The mediator client's retry strategy, used when enableRetryStrategy is true", + "default": {}, + "unevaluatedProperties": false, + "properties": { + "retries": { + "type": "integer", + "description": "Number of retry attempts", + "minimum": 0, + "x-env-value": "MEDIATOR_RETRY_STRATEGY_RETRIES" + }, + "shouldResetTimeout": { + "type": "boolean", + "description": "Whether to reset the client timeout between retries", + "x-env-value": "MEDIATOR_RETRY_STRATEGY_SHOULD_RESET_TIMEOUT" + }, + "isExponential": { + "type": "boolean", + "description": "Whether to back off exponentially between retries", + "x-env-value": "MEDIATOR_RETRY_STRATEGY_IS_EXPONENTIAL" + }, + "delay": { + "type": "integer", + "description": "The base delay in milliseconds between retries", + "minimum": 0, + "x-env-value": "MEDIATOR_RETRY_STRATEGY_DELAY" + } + } + }, + "actiony": { + "type": "object", + "description": "The actiony remote's connection options", + "default": {}, + "unevaluatedProperties": false, + "properties": { + "url": { + "type": "string", + "description": "The actiony remote's base url", + "format": "uri", + "x-env-value": "MEDIATOR_ACTIONY_URL" + } + } + }, + "locky": { + "type": "object", + "description": "The locky remote's connection options", + "default": {}, + "unevaluatedProperties": false, + "properties": { + "url": { + "type": "string", + "description": "The locky remote's base url", + "format": "uri", + "x-env-value": "MEDIATOR_LOCKY_URL" + } + } + } + } + } + } + } + } +} From 76952ddca82798896295ddd1612e36956f38792d Mon Sep 17 00:00:00 2001 From: Asif Date: Thu, 16 Jul 2026 09:00:45 +0300 Subject: [PATCH 2/5] fix: planet-dumper --- schemas/vector/planetDumper/v1.schema.json | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/schemas/vector/planetDumper/v1.schema.json b/schemas/vector/planetDumper/v1.schema.json index 10784f37..a175a22d 100644 --- a/schemas/vector/planetDumper/v1.schema.json +++ b/schemas/vector/planetDumper/v1.schema.json @@ -274,8 +274,8 @@ "actiony": { "type": "object", "description": "The actiony remote's connection options", - "default": {}, "unevaluatedProperties": false, + "required": ["url"], "properties": { "url": { "type": "string", @@ -288,8 +288,8 @@ "locky": { "type": "object", "description": "The locky remote's connection options", - "default": {}, "unevaluatedProperties": false, + "required": ["url"], "properties": { "url": { "type": "string", @@ -301,6 +301,22 @@ } } } + }, + "if": { + "properties": { + "enabled": { + "const": true + } + } + }, + "then": { + "required": ["services"], + "properties": { + "services": { + "type": "object", + "required": ["planetDumperPg", "planetDumperNg"] + } + } } } } From 47a5c7fa306964a72c3cf8658cb7ac64843e8e37 Mon Sep 17 00:00:00 2001 From: Asif Date: Thu, 16 Jul 2026 17:18:50 +0300 Subject: [PATCH 3/5] fix: planet-dumper-schema --- schemas/vector/planetDumper/v1.schema.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/schemas/vector/planetDumper/v1.schema.json b/schemas/vector/planetDumper/v1.schema.json index a175a22d..d028029b 100644 --- a/schemas/vector/planetDumper/v1.schema.json +++ b/schemas/vector/planetDumper/v1.schema.json @@ -310,11 +310,22 @@ } }, "then": { - "required": ["services"], + "required": ["services", "mediator"], "properties": { "services": { "type": "object", "required": ["planetDumperPg", "planetDumperNg"] + }, + "mediator": { + "type": "object", + "required": ["actiony", "locky"] + } + } + }, + "else": { + "properties": { + "enabled": { + "const": false } } } From 5ff9a1b3649b76ea95422d2f3209dfb70c1c6e78 Mon Sep 17 00:00:00 2001 From: Asif Date: Mon, 20 Jul 2026 16:25:07 +0300 Subject: [PATCH 4/5] fix: update schema --- schemas/vector/planetDumper/v1.schema.json | 137 ++++++++++++++++++++- 1 file changed, 135 insertions(+), 2 deletions(-) diff --git a/schemas/vector/planetDumper/v1.schema.json b/schemas/vector/planetDumper/v1.schema.json index d028029b..884e6d00 100644 --- a/schemas/vector/planetDumper/v1.schema.json +++ b/schemas/vector/planetDumper/v1.schema.json @@ -3,7 +3,7 @@ "type": "object", "title": "vectorPlanetDumperV1", "description": "Vector's planet-dumper schema", - "required": ["telemetry", "httpClient", "postgres", "pgDump", "ngDump", "osmium", "s3", "arstotzka"], + "required": ["telemetry", "httpClient", "postgres", "pgDump", "ngDump", "osmium", "s3", "arstotzka", "cli"], "properties": { "telemetry": { "description": "Telemetry configuration", @@ -161,12 +161,30 @@ } }, "s3": { - "description": "S3 upload tuning. Bucket/endpoint/acl are provided per-invocation as CLI arguments rather than static config, since they can differ per run", + "description": "S3 destination and upload tuning for the created dump file. endpoint/bucketName are required when running the create command directly, or when scheduling with target 'create' - see the root-level conditional", "type": "object", "default": {}, "unevaluatedProperties": false, "required": ["upload"], "properties": { + "endpoint": { + "type": "string", + "description": "The s3 endpoint the resulting dump will be uploaded to", + "format": "uri", + "x-env-value": "S3_ENDPOINT" + }, + "bucketName": { + "type": "string", + "description": "The bucket the resulting dump will be uploaded to", + "x-env-value": "S3_BUCKET_NAME" + }, + "acl": { + "type": "string", + "description": "The canned acl policy for uploaded objects", + "enum": ["authenticated-read", "private", "public-read", "public-read-write"], + "default": "private", + "x-env-value": "S3_ACL" + }, "upload": { "type": "object", "description": "Multipart upload tuning for the created dump file", @@ -329,6 +347,121 @@ } } } + }, + "cli": { + "description": "Per-run pipeline parameters for the pg_dump/create/schedule commands, previously supplied as CLI flags", + "type": "object", + "default": {}, + "unevaluatedProperties": false, + "required": ["outputFormat", "stateSource", "cleanupMode"], + "properties": { + "outputFormat": { + "type": "string", + "description": "The resulting output name format, example: prefix_{state}_{timestamp}_suffix.pbf", + "x-env-value": "OUTPUT_FORMAT" + }, + "stateSource": { + "type": "string", + "description": "Determines state sequence number to source - a replication state url or a specific state number", + "default": "1", + "x-env-value": "STATE_SOURCE" + }, + "cleanupMode": { + "type": "string", + "description": "The command execution cleanup mode", + "enum": ["none", "pre-clean-others", "post-clean-others", "post-clean-workdir", "post-clean-all"], + "default": "none", + "x-env-value": "CLEANUP_MODE" + }, + "resume": { + "type": "boolean", + "description": "Resume already existing dump state", + "default": false, + "x-env-value": "RESUME" + }, + "info": { + "type": "boolean", + "description": "Collect info on the resulted dump", + "default": false, + "x-env-value": "INFO" + }, + "dumpServer": { + "type": "object", + "description": "Configuration for registering the created dump's metadata with a dump-server", + "default": {}, + "unevaluatedProperties": false, + "properties": { + "endpoint": { + "type": "string", + "description": "The endpoint of the dump-server", + "format": "uri", + "pattern": "^https?://", + "x-env-value": "DUMP_SERVER_ENDPOINT" + }, + "headers": { + "type": "array", + "description": "The headers to attach to the dump-server request, each formatted as key=value", + "default": [], + "items": { + "type": "string", + "pattern": "^[^=]+=[^=]+$" + }, + "x-env-value": "DUMP_SERVER_HEADERS", + "x-env-format": "json" + } + } + }, + "schedule": { + "type": "object", + "description": "Configuration for the schedule command's cron loop", + "default": {}, + "unevaluatedProperties": false, + "properties": { + "target": { + "type": "string", + "description": "Which pipeline to run on each scheduled tick", + "enum": ["create", "pg_dump"], + "x-env-value": "TARGET" + }, + "cronExpression": { + "type": "string", + "description": "A cron expression controlling the schedule", + "x-env-value": "CRON_EXPRESSION" + }, + "runOnInit": { + "type": "boolean", + "description": "Immediately run the pipeline once at startup, before waiting for the first scheduled tick", + "default": false, + "x-env-value": "RUN_ON_INIT" + } + } + } + } + } + }, + "if": { + "required": ["cli"], + "properties": { + "cli": { + "required": ["schedule"], + "properties": { + "schedule": { + "required": ["target"], + "properties": { + "target": { + "const": "create" + } + } + } + } + } + } + }, + "then": { + "properties": { + "s3": { + "required": ["endpoint", "bucketName"] + } } } } From 3485af939ac3d3beb3c726571cda4ce4ad58ae36 Mon Sep 17 00:00:00 2001 From: Asif Date: Mon, 20 Jul 2026 16:28:53 +0300 Subject: [PATCH 5/5] fix: update schema --- schemas/vector/planetDumper/v1.configs.json | 3 ++- schemas/vector/planetDumper/v1.schema.json | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/schemas/vector/planetDumper/v1.configs.json b/schemas/vector/planetDumper/v1.configs.json index c7d03977..97083470 100644 --- a/schemas/vector/planetDumper/v1.configs.json +++ b/schemas/vector/planetDumper/v1.configs.json @@ -13,7 +13,8 @@ "ngDump": { "verbose": false, "maxConcurrency": 4 }, "osmium": { "verbose": false, "progress": false }, "s3": { "upload": { "concurrency": 4, "partSize": 5 } }, - "arstotzka": { "enabled": false } + "arstotzka": { "enabled": false }, + "cli": { "outputFormat": "dump_{state}_{timestamp}.pbf", "stateSource": "1", "cleanupMode": "none" } } } ] diff --git a/schemas/vector/planetDumper/v1.schema.json b/schemas/vector/planetDumper/v1.schema.json index 884e6d00..769d1c5d 100644 --- a/schemas/vector/planetDumper/v1.schema.json +++ b/schemas/vector/planetDumper/v1.schema.json @@ -443,9 +443,11 @@ "required": ["cli"], "properties": { "cli": { + "type": "object", "required": ["schedule"], "properties": { "schedule": { + "type": "object", "required": ["target"], "properties": { "target": { @@ -460,6 +462,7 @@ "then": { "properties": { "s3": { + "type": "object", "required": ["endpoint", "bucketName"] } }