Remediate ClickPipes and organization OpenAPI drift

crates/clickhouse-cloud-api/README.md

@@ -52,6 +52,8 @@ Postgres slow-query aggregate durations (`*DurationUs`) and execution `durationU
 
 Kinesis source format enums now include `Protobuf`. Set `ClickPipePostKinesisSource.protobuf_schema` to the base64-encoded `.proto` source or serialized `FileDescriptorSet` for that format; omit it for other formats. Organization Prometheus discovery has graduated from beta and is no longer listed in `BETA_OPERATIONS`.
 
+`Organization.capabilities.snapshots` reports snapshot eligibility as an optional boolean. Kinesis create requests accept `ClickPipeKinesisSchemaRegistry` for AWS Glue; supply `type`, `glue_region`, and `glue_registry_name`, and optionally `glue_role_arn` to assume a different role from the Kinesis source. Kinesis responses use `ClickPipeKinesisSchemaRegistryResponse`, whose fields tolerate absence and null; convert it with `TryFrom` before writing it back. Kafka create requests accept `tombstone_mode: Some(Delete)` to delete matching destination rows for tombstone records. This requires exactly-once delivery and is set only at creation. Struct-literal callers of `Organization`, `ClickPipeKinesisSource`, `ClickPipeKafkaSource`, `ClickPipePostKinesisSource`, and `ClickPipePostKafkaSource` need to supply the new optional fields as `None` or use `..Default::default()`.
+
 The beta Query API endpoint management methods are `query_api_endpoint_create`, `query_api_endpoint_get`, `query_api_endpoint_list`, `query_api_endpoint_update`, and `query_api_endpoint_delete`. Create and update take `PublicQueryApiEndpointRequest`; list accepts an optional cursor and limit (1–100) and returns `items` with `pagination.next_cursor`. User-owned endpoints can be listed and read, but cannot be updated or deleted through this API.
 
 ### ClickHouse settings models

crates/clickhouse-cloud-api/clickhouse_cloud_openapi.json

@@ -5,7 +5,7 @@
     "version": "1.0",
     "contact": {
       "name": "ClickHouse Support",
-      "url": "https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-1152576",
+      "url": "https://clickhouse.com/docs/en/cloud/manage/openapi?referrer=openapi-1156400",
       "email": "support@clickhouse.com"
     }
   },
@@ -8887,7 +8887,7 @@
       },
       "patch": {
         "summary": "Update ClickPipe",
-        "description": "Update the specified ClickPipe. Source fields not present in the per-source update schemas are immutable after creation. For Kafka sources, values submitted for immutable fields (type, format, brokers, topics, consumerGroup, offset, schemaRegistry, exactlyOnce) are not applied, except schema registry credentials, which are rejected.",
+        "description": "Update the specified ClickPipe. Source fields not present in the per-source update schemas are immutable after creation. For Kafka sources, values submitted for immutable fields (type, format, brokers, topics, consumerGroup, offset, schemaRegistry, exactlyOnce, tombstoneMode) are not applied, except schema registry credentials, which are rejected.",
         "operationId": "clickPipeUpdate",
         "parameters": [
           {
@@ -23024,6 +23024,14 @@
           }
         }
       },
+      "OrganizationCapabilities": {
+        "properties": {
+          "snapshots": {
+            "description": "Whether the organization is eligible to use service snapshots: true only when the organization has the snapshots feature enabled, is on a PPv2 tier, and has the backups entitlement — the same conditions enforced when a snapshotConfiguration is saved. Check this before configuring snapshots on a service. Snapshots apply to primary services only, so a secondary/replica service is rejected regardless of organization eligibility.",
+            "type": "boolean"
+          }
+        }
+      },
       "Organization": {
         "properties": {
           "id": {
@@ -23057,6 +23065,9 @@
           "enableCoreDumps": {
             "description": "Whether crash reports (core dumps) collection is enabled for services in the organization. When disabled at the organization level, individual services cannot enable crash reports.",
             "type": "boolean"
+          },
+          "capabilities": {
+            "$ref": "#/components/schemas/OrganizationCapabilities"
           }
         }
       },
@@ -23914,8 +23925,9 @@
             "type": "string"
           },
           "topics": {
-            "description": "Topics of the Kafka source.",
-            "type": "string"
+            "description": "One or more Kafka topics as a comma-separated string. All topics must have the same schema and are ingested into the same destination table by a single ClickPipe.",
+            "type": "string",
+            "example": "topic1,topic2"
           },
           "consumerGroup": {
             "description": "Consumer group of the Kafka source. If not provided \"clickpipes-<<ID>>\" will be used.",
@@ -23986,6 +23998,17 @@
               "boolean",
               "null"
             ]
+          },
+          "tombstoneMode": {
+            "description": "How Kafka tombstone records are handled. Set to \"delete\" to delete the matching destination row. Requires exactly-once delivery and can only be set at pipe creation.",
+            "type": [
+              "string",
+              "null"
+            ],
+            "enum": [
+              "delete"
+            ],
+            "example": "delete"
           }
         }
       },
@@ -24020,8 +24043,9 @@
             "type": "string"
           },
           "topics": {
-            "description": "Topics of the Kafka source.",
-            "type": "string"
+            "description": "One or more Kafka topics as a comma-separated string. All topics must have the same schema and are ingested into the same destination table by a single ClickPipe.",
+            "type": "string",
+            "example": "topic1,topic2"
           },
           "consumerGroup": {
             "description": "Consumer group of the Kafka source. If not provided \"clickpipes-<<ID>>\" will be used.",
@@ -24093,6 +24117,17 @@
               "null"
             ]
           },
+          "tombstoneMode": {
+            "description": "How Kafka tombstone records are handled. Set to \"delete\" to delete the matching destination row. Requires exactly-once delivery and can only be set at pipe creation.",
+            "type": [
+              "string",
+              "null"
+            ],
+            "enum": [
+              "delete"
+            ],
+            "example": "delete"
+          },
           "credentials": {
             "description": "Credentials for Kafka source. Choose one that is supported by the authentication method.",
             "oneOf": [
@@ -24179,6 +24214,40 @@
           }
         }
       },
+      "ClickPipeKinesisSchemaRegistry": {
+        "properties": {
+          "type": {
+            "description": "Type of the schema registry. Kinesis ClickPipes support the AWS Glue Schema Registry, which authenticates with IAM instead of credentials.",
+            "type": "string",
+            "enum": [
+              "glue"
+            ]
+          },
+          "glueRegion": {
+            "description": "AWS region of the Glue Schema Registry.",
+            "type": "string",
+            "example": "us-east-1"
+          },
+          "glueRegistryName": {
+            "description": "Name of the Glue Schema Registry.",
+            "type": "string",
+            "example": "my-registry"
+          },
+          "glueRoleArn": {
+            "description": "IAM role to assume for Glue Schema Registry access. Defaults to the IAM identity of the Kinesis source.",
+            "type": [
+              "string",
+              "null"
+            ],
+            "example": "arn:aws:iam::123456789012:role/MyGlueRegistryRole"
+          }
+        },
+        "required": [
+          "type",
+          "glueRegion",
+          "glueRegistryName"
+        ]
+      },
       "ClickPipeKinesisSource": {
         "properties": {
           "format": {
@@ -24240,6 +24309,16 @@
               "null"
             ],
             "example": "arn:aws:iam::123456789012:role/MyRole"
+          },
+          "schemaRegistry": {
+            "oneOf": [
+              {
+                "$ref": "#/components/schemas/ClickPipeKinesisSchemaRegistry"
+              },
+              {
+                "type": "null"
+              }
+            ]
           }
         }
       },
@@ -24305,6 +24384,16 @@
             ],
             "example": "arn:aws:iam::123456789012:role/MyRole"
           },
+          "schemaRegistry": {
+            "oneOf": [
+              {
+                "$ref": "#/components/schemas/ClickPipeKinesisSchemaRegistry"
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
           "accessKey": {
             "oneOf": [
               {
@@ -24316,7 +24405,7 @@
             ]
           },
           "protobufSchema": {
-            "description": "Base64-encoded .proto source or serialized FileDescriptorSet. Required with Protobuf format and not supported with other formats.",
+            "description": "Base64-encoded .proto source or serialized FileDescriptorSet. Required with Protobuf format unless a schema registry is configured, and not supported with other formats.",
             "type": "string",
             "example": "c3ludGF4ID0gInByb3RvMyI7IG1lc3NhZ2UgRXZlbnQge30=",
             "maxLength": 1048576,
@@ -33796,7 +33885,7 @@
           },
           "groups": {
             "type": "array",
-            "description": "A list of groups to which the user belongs. Role may be derived from group display or value.",
+            "description": "A list of groups to which the user belongs. Read-only; ignored on write. Membership is managed via the Groups endpoints.",
             "items": {
               "$ref": "#/components/schemas/ScimUserGroup"
             }
@@ -33810,7 +33899,7 @@
           },
           "roles": {
             "type": "array",
-            "description": "A list of roles for the user.",
+            "description": "A list of roles for the user. Read-only; ignored on write. Membership is managed via the Groups endpoints.",
             "items": {
               "$ref": "#/components/schemas/ScimUserRole"
             }
@@ -34018,7 +34107,7 @@
           },
           "groups": {
             "type": "array",
-            "description": "A list of groups to which the user belongs. Role may be derived from group display or value.",
+            "description": "A list of groups to which the user belongs. Read-only; ignored on write. Membership is managed via the Groups endpoints.",
             "items": {
               "$ref": "#/components/schemas/ScimUserGroup"
             }
@@ -34032,7 +34121,7 @@
           },
           "roles": {
             "type": "array",
-            "description": "A list of roles for the user.",
+            "description": "A list of roles for the user. Read-only; ignored on write. Membership is managed via the Groups endpoints.",
             "items": {
               "$ref": "#/components/schemas/ScimUserRole"
             }
@@ -38902,4 +38991,4 @@
       ]
     }
   ]
-}
+}

crates/clickhouse-cloud-api/src/convert.rs

@@ -19,6 +19,7 @@
 
 use std::fmt;
 
+mod clickpipes;
 mod clickstack;
 mod postgres;
 mod service;

crates/clickhouse-cloud-api/src/convert/clickpipes.rs

@@ -0,0 +1,29 @@
+use super::MissingRequiredFields;
+use crate::models::{ClickPipeKinesisSchemaRegistry, ClickPipeKinesisSchemaRegistryResponse};
+
+impl TryFrom<ClickPipeKinesisSchemaRegistryResponse> for ClickPipeKinesisSchemaRegistry {
+    type Error = MissingRequiredFields;
+
+    fn try_from(value: ClickPipeKinesisSchemaRegistryResponse) -> Result<Self, Self::Error> {
+        let mut missing = Vec::new();
+        if value.r#type.is_none() {
+            missing.push("type");
+        }
+        if value.glue_region.is_none() {
+            missing.push("glueRegion");
+        }
+        if value.glue_registry_name.is_none() {
+            missing.push("glueRegistryName");
+        }
+        if !missing.is_empty() {
+            return Err(MissingRequiredFields::new(missing));
+        }
+
+        Ok(Self {
+            r#type: value.r#type.expect("checked above"),
+            glue_region: value.glue_region.expect("checked above"),
+            glue_registry_name: value.glue_registry_name.expect("checked above"),
+            glue_role_arn: value.glue_role_arn,
+        })
+    }
+}

crates/clickhouse-cloud-api/src/models.rs

@@ -185,7 +185,8 @@ pub use organization_private_endpoints::{
 };
 pub use organizations::{
     ActiveBalance, ActiveBalances, CreditBalance, CreditBalanceType, CreditBalances, Organization,
-    OrganizationPatchRequest, PrometheusDiscoveryLabels, PrometheusDiscoveryTargetGroup,
+    OrganizationCapabilities, OrganizationPatchRequest, PrometheusDiscoveryLabels,
+    PrometheusDiscoveryTargetGroup,
 };
 pub use postgres::{
     BasePostgresService, PgBouncerConfig, PgBouncerConfigResponse, PgConfig,