diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs index 1887885ec9..1dc63f1c44 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/FullTextSearchSql.cs @@ -3,10 +3,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql; using Microsoft.EntityFrameworkCore.Migrations.Operations; /// -/// Full text search DDL for the failed messages table. EF Core cannot model a GIN index over an -/// expression, so it is applied by the AddFullTextSearch migration. The statements live here, and -/// not in the migration itself, so that regenerating the migrations with the dotnet-ef CLI only -/// costs a one line migration body. +/// Full text search DDL for the failed messages table. EF Core cannot model a GIN index over an expression, so migrations apply it as SQL: AddFullTextSearch created the index, and WidenMessageIdAndCapFullTextBody replaced it with one that indexes only the start of the body. The statements live here, and not in the migrations themselves, so that regenerating the migrations with the dotnet-ef CLI only costs a one line migration body. /// static class FullTextSearchSql { @@ -17,6 +14,9 @@ static class FullTextSearchSql // stopword removal do more harm than good. public const string Configuration = "simple"; + // to_tsvector fails once the vector it builds passes 1 MB, and the index builds one on every insert, so a body with enough distinct words would fail its whole ingestion batch. Only the start of the body is indexed, which keeps the vector well under that limit. + public const int IndexedBodyLength = 262_144; + // Written the way PostgreSqlFullTextSearchDialect makes EF Core render it, down to the casing // and the redundant looking parentheses: PostgreSQL only uses an expression index when the // query expression parses to the same tree, and a mismatch downgrades search to a sequential @@ -26,12 +26,18 @@ static class FullTextSearchSql // "ServiceControl.MessageFailures.MyMessage" would not otherwise match a search for // "MyMessage". It mirrors the SearchableMessageType that MessageTypeEnricher produces for // RavenDB, and is not the duplicate of the headers it looks like. - public const string IndexedExpression = + public static readonly string IndexedExpression = + $"""to_tsvector('{Configuration}', headers_json || ' ' || substring(COALESCE(body_text, ''), 1, {IndexedBodyLength}) || ' ' || replace(replace(COALESCE(message_type, ''), '.', ' '), '+', ' '))"""; + + // What AddFullTextSearch indexed. WidenMessageIdAndCapFullTextBody replaces it on the way up and restores it on the way down. + const string UncappedIndexedExpression = $"""to_tsvector('{Configuration}', headers_json || ' ' || COALESCE(body_text, '') || ' ' || replace(replace(COALESCE(message_type, ''), '.', ' '), '+', ' '))"""; - public static readonly string Up = CreateIndexSql(null); + public static readonly string CreateUncappedIndex = CreateIndexSql(null, UncappedIndexedExpression); + + public static readonly string CreateIndex = CreateIndexSql(null, IndexedExpression); - public static readonly string Down = DropIndexSql(null); + public static readonly string DropIndex = DropIndexSql(null); /// /// Re-renders the statement the migration carries, this time with the configured schema in it. @@ -42,15 +48,16 @@ static class FullTextSearchSql public static MigrationOperation Rewrite(SqlOperation operation, string schema) => operation.Sql switch { - var sql when sql == Up => WithSql(operation, CreateIndexSql(schema)), - var sql when sql == Down => WithSql(operation, DropIndexSql(schema)), + var sql when sql == CreateUncappedIndex => WithSql(operation, CreateIndexSql(schema, UncappedIndexedExpression)), + var sql when sql == CreateIndex => WithSql(operation, CreateIndexSql(schema, IndexedExpression)), + var sql when sql == DropIndex => WithSql(operation, DropIndexSql(schema)), _ => operation }; - public static bool IsHandled(string sql) => sql == Up || sql == Down; + public static bool IsHandled(string sql) => sql == CreateUncappedIndex || sql == CreateIndex || sql == DropIndex; - static string CreateIndexSql(string? schema) => - $"CREATE INDEX {IndexName} ON {Qualify(schema, TableName)} USING GIN ({IndexedExpression})"; + static string CreateIndexSql(string? schema, string indexedExpression) => + $"CREATE INDEX {IndexName} ON {Qualify(schema, TableName)} USING GIN ({indexedExpression})"; // An index belongs to its table's schema, so it is the index that gets qualified here. static string DropIndexSql(string? schema) => $"DROP INDEX IF EXISTS {Qualify(schema, IndexName)}"; diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260902040422_AddFullTextSearch.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260902040422_AddFullTextSearch.cs index dfcd89e387..e2515b3234 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260902040422_AddFullTextSearch.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260902040422_AddFullTextSearch.cs @@ -8,9 +8,9 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations public partial class AddFullTextSearch : Migration { /// - protected override void Up(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.Up); + protected override void Up(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.CreateUncappedIndex); /// - protected override void Down(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.Down); + protected override void Down(MigrationBuilder migrationBuilder) => migrationBuilder.Sql(FullTextSearchSql.DropIndex); } } diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.Designer.cs new file mode 100644 index 0000000000..ace61e5c47 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.Designer.cs @@ -0,0 +1,923 @@ +// +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20260928064706_WidenMessageIdAndCapFullTextBody")] + partial class WidenMessageIdAndCapFullTextBody + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("request_id"); + + b.Property("ArchiveType") + .HasColumnType("integer") + .HasColumnName("archive_type"); + + b.Property("OperationType") + .HasColumnType("integer") + .HasColumnName("operation_type"); + + b.Property("CurrentBatch") + .HasColumnType("integer") + .HasColumnName("current_batch"); + + b.Property("GroupName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("group_name"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_name"); + + b.Property("NumberOfBatches") + .HasColumnType("integer") + .HasColumnName("number_of_batches"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property("Started") + .HasColumnType("timestamp with time zone") + .HasColumnName("started"); + + b.Property("TotalNumberOfMessages") + .HasColumnType("integer") + .HasColumnName("total_number_of_messages"); + + b.HasKey("RequestId", "ArchiveType", "OperationType") + .HasName("pk_archive_operations"); + + b.HasIndex("Started") + .HasDatabaseName("ix_archive_operations_started"); + + b.ToTable("archive_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Category") + .IsRequired() + .HasColumnType("text") + .HasColumnName("category"); + + b.Property("CustomCheckId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("custom_check_id"); + + b.Property("FailureReason") + .HasColumnType("text") + .HasColumnName("failure_reason"); + + b.Property("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_host"); + + b.Property("OriginatingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("originating_endpoint_host_id"); + + b.Property("OriginatingEndpointName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_name"); + + b.Property("ReportedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("reported_at"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_custom_checks"); + + b.HasIndex("ReportedAt") + .HasDatabaseName("ix_custom_checks_reported_at"); + + b.HasIndex("Status", "ReportedAt") + .HasDatabaseName("ix_custom_checks_status_reported_at"); + + b.ToTable("custom_checks", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.Property("TrackInstances") + .HasColumnType("boolean") + .HasColumnName("track_instances"); + + b.HasKey("Name") + .HasName("pk_endpoint_settings"); + + b.ToTable("endpoint_settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("event_type"); + + b.Property("RaisedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("raised_at"); + + b.PrimitiveCollection>("RelatedTo") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("related_to"); + + b.Property("Severity") + .HasColumnType("integer") + .HasColumnName("severity"); + + b.HasKey("Id") + .HasName("pk_event_log_items"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_event_log_items_raised_at_id"); + + b.ToTable("event_log_items", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DispatchContextJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("dispatch_context_json"); + + b.Property("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("dispatch_context_type_name"); + + b.HasKey("Id") + .HasName("pk_external_integration_dispatch_requests"); + + b.ToTable("external_integration_dispatch_requests", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("MessageId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_error_imports"); + + b.HasIndex("FailedAt") + .HasDatabaseName("ix_failed_error_imports_failed_at"); + + b.ToTable("failed_error_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("edit_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_edits"); + + b.HasIndex("EditId") + .HasDatabaseName("ix_failed_message_edits_edit_id"); + + b.ToTable("failed_message_edits", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("failing_endpoint_address"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasColumnType("text") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("FailingEndpointAddress") + .HasDatabaseName("ix_failed_messages_failing_endpoint_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastTimeOfFailure") + .HasDatabaseName("ix_failed_messages_status_last_time_of_failure"); + + b.HasIndex("Status", "LastModified", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_last_modified_unique_message_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified", "UniqueMessageId"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "MessageType", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_message_type_unique_message_id"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.HasIndex("Type", "GroupId") + .HasDatabaseName("ix_failed_message_groups_type_group_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "FailedMessageUniqueId", "Title" }); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.Property("StageAttempts") + .HasColumnType("integer") + .HasColumnName("stage_attempts"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.HasIndex("RetryBatchId") + .HasDatabaseName("ix_failed_message_retries_retry_batch_id"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Comment") + .IsRequired() + .HasColumnType("text") + .HasColumnName("comment"); + + b.HasKey("GroupId") + .HasName("pk_group_comments"); + + b.ToTable("group_comments", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("Id") + .HasName("pk_historic_retry_operations"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending() + .HasDatabaseName("ix_historic_retry_operations_completion_time_id"); + + b.ToTable("historic_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.PrimitiveCollection>("EndpointIndicators") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("endpoint_indicators"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("name"); + + b.Property("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("normalized_sanitized_name"); + + b.Property("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sanitized_name"); + + b.Property("Scope") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("scope"); + + b.Property("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("user_indicator"); + + b.HasKey("NormalizedName", "ThroughputSource") + .HasName("pk_licensing_endpoints"); + + b.HasIndex("NormalizedSanitizedName") + .HasDatabaseName("ix_licensing_endpoints_normalized_sanitized_name"); + + b.ToTable("licensing_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.Property("DateUtc") + .HasColumnType("date") + .HasColumnName("date_utc"); + + b.Property("MessageCount") + .HasColumnType("bigint") + .HasColumnName("message_count"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc") + .HasName("pk_licensing_endpoint_throughput"); + + b.HasIndex("DateUtc") + .HasDatabaseName("ix_licensing_endpoint_throughput_date_utc"); + + b.ToTable("licensing_endpoint_throughput", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("from_physical_address"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("to_physical_address"); + + b.HasKey("FromPhysicalAddress") + .HasName("pk_message_redirects"); + + b.ToTable("message_redirects", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Classifier") + .HasColumnType("text") + .HasColumnName("classifier"); + + b.Property("Context") + .HasColumnType("text") + .HasColumnName("context"); + + b.Property("InitialBatchSize") + .HasColumnType("integer") + .HasColumnName("initial_batch_size"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property("InitiatedByName") + .HasColumnType("text") + .HasColumnName("initiated_by_name"); + + b.Property("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("request_id"); + + b.Property("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_session_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("StagingId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("staging_id"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_retry_batches"); + + b.HasIndex("Status", "RetrySessionId") + .HasDatabaseName("ix_retry_batches_status_retry_session_id"); + + b.ToTable("retry_batches", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property("Id") + .HasColumnType("integer") + .HasColumnName("id"); + + b.Property("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.HasKey("Id") + .HasName("pk_retry_batch_now_forwarding"); + + b.ToTable("retry_batch_now_forwarding", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property("Key") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("key"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Key") + .HasName("pk_settings"); + + b.ToTable("settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_type"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("transport_address"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("endpoint"); + + b.HasKey("MessageType", "TransportAddress") + .HasName("pk_subscriptions"); + + b.ToTable("subscriptions", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("Classifier") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("classifier"); + + b.Property("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("RequestId", "RetryType") + .HasName("pk_unacknowledged_retry_operations"); + + b.ToTable("unacknowledged_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_licensing_endpoint_throughput_licensing_endpoints_normalize"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.cs new file mode 100644 index 0000000000..1f1f47fc20 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20260928064706_WidenMessageIdAndCapFullTextBody.cs @@ -0,0 +1,62 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class WidenMessageIdAndCapFullTextBody : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "message_id", + table: "failed_messages", + type: "text", + nullable: true, + oldClrType: typeof(string), + oldType: "character varying(450)", + oldMaxLength: 450, + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "message_id", + table: "failed_error_imports", + type: "text", + nullable: false, + oldClrType: typeof(string), + oldType: "character varying(450)", + oldMaxLength: 450); + + migrationBuilder.Sql(FullTextSearchSql.DropIndex); + migrationBuilder.Sql(FullTextSearchSql.CreateIndex); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(FullTextSearchSql.DropIndex); + migrationBuilder.Sql(FullTextSearchSql.CreateUncappedIndex); + + migrationBuilder.AlterColumn( + name: "message_id", + table: "failed_messages", + type: "character varying(450)", + maxLength: 450, + nullable: true, + oldClrType: typeof(string), + oldType: "text", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "message_id", + table: "failed_error_imports", + type: "character varying(450)", + maxLength: 450, + nullable: false, + oldClrType: typeof(string), + oldType: "text"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index 06b1a9b639..3403a3d831 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -264,8 +264,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("MessageId") .IsRequired() - .HasMaxLength(450) - .HasColumnType("character varying(450)") + .HasColumnType("text") .HasColumnName("message_id"); b.HasKey("UniqueMessageId") @@ -366,8 +365,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnName("last_time_of_failure"); b.Property("MessageId") - .HasMaxLength(450) - .HasColumnType("character varying(450)") + .HasColumnType("text") .HasColumnName("message_id"); b.Property("MessageType") diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs index d2c01fb908..8461a8a12c 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs @@ -15,7 +15,7 @@ public IQueryable Search(IQueryable so source.Where(message => EF.Functions.ToTsVector(FullTextSearchSql.Configuration, message.HeadersJson + " " + - (message.BodyText ?? "") + " " + + (message.BodyText ?? "").Substring(0, FullTextSearchSql.IndexedBodyLength) + " " + (message.MessageType ?? "").Replace(".", " ").Replace("+", " ")) .Matches(EF.Functions.WebSearchToTsQuery(FullTextSearchSql.Configuration, ToOrQuery(searchTerms)))); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.Designer.cs new file mode 100644 index 0000000000..704d62a08e --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.Designer.cs @@ -0,0 +1,741 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20260928064709_WidenMessageId")] + partial class WidenMessageId + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ArchiveType") + .HasColumnType("int"); + + b.Property("OperationType") + .HasColumnType("int"); + + b.Property("CurrentBatch") + .HasColumnType("int"); + + b.Property("GroupName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("NumberOfBatches") + .HasColumnType("int"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Started") + .HasColumnType("datetime2"); + + b.Property("TotalNumberOfMessages") + .HasColumnType("int"); + + b.HasKey("RequestId", "ArchiveType", "OperationType"); + + b.HasIndex("Started"); + + b.ToTable("ArchiveOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Category") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("CustomCheckId") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailureReason") + .HasColumnType("nvarchar(max)"); + + b.Property("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OriginatingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("OriginatingEndpointName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReportedAt") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ReportedAt"); + + b.HasIndex("Status", "ReportedAt"); + + b.ToTable("CustomChecks"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("TrackInstances") + .HasColumnType("bit"); + + b.HasKey("Name"); + + b.ToTable("EndpointSettings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Severity") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("DispatchContextJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("ExternalIntegrationDispatchRequests"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt"); + + b.ToTable("FailedErrorImports"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("EditId"); + + b.ToTable("FailedMessageEdits"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("FailingEndpointAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "LastTimeOfFailure"); + + b.HasIndex("Status", "MessageType", "UniqueMessageId"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.HasIndex("Type", "GroupId"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "Title" }); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.Property("StageAttempts") + .HasColumnType("int"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("RetryBatchId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Comment") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("GroupId"); + + b.ToTable("GroupComments"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CompletionTime") + .HasColumnType("datetime2"); + + b.Property("Failed") + .HasColumnType("bit"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending(); + + b.ToTable("HistoricRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("ThroughputSource") + .HasColumnType("int"); + + b.PrimitiveCollection("EndpointIndicators") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Scope") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("NormalizedName", "ThroughputSource"); + + b.HasIndex("NormalizedSanitizedName"); + + b.ToTable("LicensingEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("ThroughputSource") + .HasColumnType("int"); + + b.Property("DateUtc") + .HasColumnType("date"); + + b.Property("MessageCount") + .HasColumnType("bigint"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc"); + + b.HasIndex("DateUtc"); + + b.ToTable("LicensingEndpointThroughput"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("FromPhysicalAddress"); + + b.ToTable("MessageRedirects"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Classifier") + .HasColumnType("nvarchar(max)"); + + b.Property("Context") + .HasColumnType("nvarchar(max)"); + + b.Property("InitialBatchSize") + .HasColumnType("int"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("InitiatedByName") + .HasColumnType("nvarchar(max)"); + + b.Property("Last") + .HasColumnType("datetime2"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("StagingId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Status", "RetrySessionId"); + + b.ToTable("RetryBatches"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property("Id") + .HasColumnType("int"); + + b.Property("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.ToTable("RetryBatchNowForwarding"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property("Key") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Value") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("MessageType", "TransportAddress"); + + b.ToTable("Subscriptions"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("Classifier") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("CompletionTime") + .HasColumnType("datetime2"); + + b.Property("Failed") + .HasColumnType("bit"); + + b.Property("Last") + .HasColumnType("datetime2"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("RequestId", "RetryType"); + + b.ToTable("UnacknowledgedRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.cs new file mode 100644 index 0000000000..3f5dc955aa --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20260928064709_WidenMessageId.cs @@ -0,0 +1,56 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class WidenMessageId : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "MessageId", + table: "FailedMessages", + type: "nvarchar(max)", + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(450)", + oldMaxLength: 450, + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "MessageId", + table: "FailedErrorImports", + type: "nvarchar(max)", + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(450)", + oldMaxLength: 450); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "MessageId", + table: "FailedMessages", + type: "nvarchar(450)", + maxLength: 450, + nullable: true, + oldClrType: typeof(string), + oldType: "nvarchar(max)", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "MessageId", + table: "FailedErrorImports", + type: "nvarchar(450)", + maxLength: 450, + nullable: false, + oldClrType: typeof(string), + oldType: "nvarchar(max)"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index b9e41a3681..72f4d45511 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -215,8 +215,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("MessageId") .IsRequired() - .HasMaxLength(450) - .HasColumnType("nvarchar(450)"); + .HasColumnType("nvarchar(max)"); b.HasKey("UniqueMessageId"); @@ -295,8 +294,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("datetime2"); b.Property("MessageId") - .HasMaxLength(450) - .HasColumnType("nvarchar(450)"); + .HasColumnType("nvarchar(max)"); b.Property("MessageType") .HasMaxLength(450) diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs index 90ce6a48e8..91e578f75f 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerServiceControlDbContext.cs @@ -1,6 +1,5 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; -using System; using EntityConfigurations; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; @@ -10,14 +9,6 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; public class SqlServerServiceControlDbContext(DbContextOptions options) : ServiceControlDbContext(options) { - protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) - { - base.ConfigureConventions(configurationBuilder); - - configurationBuilder.Properties().HaveConversion(); - configurationBuilder.Properties().HaveConversion(); - } - protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/UtcDateTimeConverters.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/UtcDateTimeConverters.cs deleted file mode 100644 index e464acc5b6..0000000000 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/UtcDateTimeConverters.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace ServiceControl.Persistence.EFCore.SqlServer; - -using System; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; - -// datetime2 stores no offset, so values come back as DateTimeKind.Unspecified and are then serialized -// by the API without the UTC marker. Everything persisted here is UTC. -sealed class UtcDateTimeConverter() : ValueConverter( - value => value.Kind == DateTimeKind.Local ? value.ToUniversalTime() : value, - value => DateTime.SpecifyKind(value, DateTimeKind.Utc)); - -sealed class NullableUtcDateTimeConverter() : ValueConverter( - value => value.HasValue && value.Value.Kind == DateTimeKind.Local ? value.Value.ToUniversalTime() : value, - value => value.HasValue ? DateTime.SpecifyKind(value.Value, DateTimeKind.Utc) : value); diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs index 3717c6a7f3..d9a40b9b14 100644 --- a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs @@ -38,6 +38,14 @@ public abstract class ServiceControlDbContext(DbContextOptions options) : DbCont protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.EnableDetailedErrors(); + protected override void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) + { + base.ConfigureConventions(configurationBuilder); + + configurationBuilder.Properties().HaveConversion(); + configurationBuilder.Properties().HaveConversion(); + } + protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs new file mode 100644 index 0000000000..8f0cc514dd --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/UtcDateTimeConverters.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.EFCore.DbContexts; + +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +// SQL Server's datetime2 has no offset, so values read back as Unspecified. Npgsql refuses to write a Local or Unspecified value to timestamptz. EF applies these converters to query parameters as well, so a date range from the API needs no conversion of its own. +sealed class UtcDateTimeConverter() : ValueConverter( + value => ToUtc(value), + value => DateTime.SpecifyKind(value, DateTimeKind.Utc)) +{ + public static DateTime ToUtc(DateTime value) => + value.Kind == DateTimeKind.Local ? value.ToUniversalTime() : DateTime.SpecifyKind(value, DateTimeKind.Utc); +} + +sealed class NullableUtcDateTimeConverter() : ValueConverter( + value => value.HasValue ? UtcDateTimeConverter.ToUtc(value.Value) : value, + value => value.HasValue ? DateTime.SpecifyKind(value.Value, DateTimeKind.Utc) : value); diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs index bf40c72369..f86bedace8 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/ColumnLengths.cs @@ -1,5 +1,9 @@ namespace ServiceControl.Persistence.EFCore.EntityConfigurations; +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; +using System.Text; + static class ColumnLengths { // Indexed and short-by-nature values get a length so that SQL Server can index them, @@ -23,4 +27,24 @@ static class ColumnLengths // int, and the throughput key adds a date on top, so ShortTextLength would exceed SQL Server's // 900 byte index key limit. public const int LicensingEndpointNameLength = 300; + + // Cutting an indexed value short could make two different values look up as equal, so a longer value keeps as much of its start as fits, followed by ~ and the SHA-256 of the whole value. The result is never longer than ShortTextLength, so fitting it again returns it unchanged. + [return: NotNullIfNotNull(nameof(value))] + public static string? FitToIndex(string? value) + { + if (value is null || value.Length <= ShortTextLength) + { + return value; + } + + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + var prefixLength = ShortTextLength - hash.Length - 1; + + if (char.IsHighSurrogate(value[prefixLength - 1])) + { + prefixLength--; + } + + return $"{value[..prefixLength]}~{hash}"; + } } diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedErrorImportConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedErrorImportConfiguration.cs index 44c46981cc..e1d806fe0f 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedErrorImportConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedErrorImportConfiguration.cs @@ -12,7 +12,7 @@ public void Configure(EntityTypeBuilder builder) builder.Property(e => e.UniqueMessageId).ValueGeneratedNever(); builder.Property(e => e.FailedAt).IsRequired(); - builder.Property(e => e.MessageId).IsRequired().HasMaxLength(ColumnLengths.ShortTextLength); + builder.Property(e => e.MessageId).IsRequired(); builder.Property(e => e.HeadersJson).IsRequired(); builder.Property(e => e.Body).IsRequired(); builder.Property(e => e.BodyStoredExternally).IsRequired(); diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs index 1bff26175c..33ea408820 100644 --- a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/FailedMessageConfiguration.cs @@ -19,9 +19,11 @@ public void Configure(EntityTypeBuilder builder) builder.Property(e => e.LastTimeOfFailure).IsRequired(); builder.Property(e => e.LastAttemptedAt).IsRequired(); - builder.Property(e => e.MessageId).HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.MessageType).HasMaxLength(ColumnLengths.ShortTextLength); - builder.Property(e => e.ConversationId).HasMaxLength(ColumnLengths.ShortTextLength); + // The converter also runs on every value a query compares with the column, so a lookup by the full id finds the stored form. + builder.Property(e => e.ConversationId) + .HasMaxLength(ColumnLengths.ShortTextLength) + .HasConversion(value => ColumnLengths.FitToIndex(value), value => value); builder.Property(e => e.SendingEndpointName).HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.SendingEndpointHost).HasMaxLength(ColumnLengths.ShortTextLength); builder.Property(e => e.ReceivingEndpointName).HasMaxLength(ColumnLengths.ShortTextLength); diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs index 43a0e7c927..2b9f37cab5 100644 --- a/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs +++ b/src/ServiceControl.Persistence.EFCore/Implementation/UnitOfWork/EFRecoverabilityIngestionUnitOfWork.cs @@ -41,7 +41,8 @@ public Task RecordFailedProcessingAttempt(MessageContext context, MessageId = processingAttempt.MessageId, MessageType = TruncateTypeName(GetMetadata(processingAttempt, "MessageType")), TimeSent = GetMetadata(processingAttempt, "TimeSent"), - ConversationId = GetMetadata(processingAttempt, "ConversationId"), + // The ingestion upsert is hand-written SQL, which skips the value converter on the column. + ConversationId = ColumnLengths.FitToIndex(GetMetadata(processingAttempt, "ConversationId")), SendingEndpointName = sendingEndpoint?.Name, SendingEndpointHostId = sendingEndpoint?.HostId, SendingEndpointHost = sendingEndpoint?.Host, diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs index c8c66d83c2..fc3276dea5 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MessageHeaders.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -7,10 +8,13 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; static class MessageHeaders { public static string Write(Dictionary headers) => - JsonSerializer.Serialize(headers, HeadersJsonContext.Default.DictionaryStringString); + JsonSerializer.Serialize(headers, context.DictionaryStringString); public static Dictionary Read(string headersJson) => - JsonSerializer.Deserialize(headersJson, HeadersJsonContext.Default.DictionaryStringString) ?? []; + JsonSerializer.Deserialize(headersJson, context.DictionaryStringString) ?? []; + + // The relaxed encoder is only unsafe for JSON embedded in HTML, and this JSON never reaches HTML. Full text search needs apostrophes, plus signs and non-ASCII letters written as they are, or it cannot find the words next to them. + static readonly HeadersJsonContext context = new(new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); } // Source generated serialization, which keeps the reflection-based serializer off the ingestion hot path. diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs index cbfeafd87b..29686a18dc 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs @@ -6,9 +6,7 @@ namespace ServiceControl.Persistence.Tests; using NUnit.Framework; /// -/// PostgreSQL only uses the GIN index of the AddFullTextSearch migration when the query expression -/// parses to the same tree as the indexed one. A mismatch is silent: search keeps working, on a -/// sequential scan of every failed message. These tests need no database. +/// PostgreSQL only uses the full text GIN index when the query expression parses to the same tree as the indexed one. A mismatch is silent: search keeps working, on a sequential scan of every failed message. These tests need no database. /// class FullTextSearchIndexTests { diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchLimitTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchLimitTests.cs new file mode 100644 index 0000000000..e52c54b805 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchLimitTests.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.Infrastructure; + +class FullTextSearchLimitTests : ErrorIngestionTestBase +{ + const int TokenCount = 150_000; + + [SetUp] + public void StoreLargeBodiesInline() => EFSettings.BodyStorage.MaxBodySizeToStore = 2 * 1024 * 1024; + + [Test] + public async Task Ingests_and_searches_a_body_with_more_distinct_words_than_a_tsvector_can_hold() + { + var body = string.Join(' ', Enumerable.Range(0, TokenCount).Select(token => $"w{token:D7}")); + var failure = new IngestedFailure { ContentType = "text/plain", Body = Encoding.UTF8.GetBytes(body) }; + + Assert.That(failure.Body, Has.Length.GreaterThan(1024 * 1024), "The body has to be larger than the 1 MB a tsvector can hold"); + + await Ingest(failure); + + var result = await MessagesViewStore.GetAllMessagesForSearch("w0000001", new PagingInfo(), new SortInfo()); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { failure.UniqueMessageIdString })); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/BodyReadTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/BodyReadTests.cs index 7a2996c262..527217eff0 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/BodyReadTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/BodyReadTests.cs @@ -79,6 +79,18 @@ public async Task Fetches_by_message_id() Assert.That(result.State, Is.EqualTo(MessageBodyState.Available)); } + [Test] + public async Task Fetches_by_an_over_length_message_id() + { + var failure = new IngestedFailure { MessageId = new string('m', 600), Body = Encoding.UTF8.GetBytes("1") }; + await Ingest(failure); + + var result = await Fetch(failure.MessageId); + + Assert.That(result, Is.Not.Null); + Assert.That(result.State, Is.EqualTo(MessageBodyState.Available)); + } + [Test] public async Task Reports_an_empty_body() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs index b926be9e25..243d01980b 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/ErrorIngestionTests.cs @@ -81,6 +81,18 @@ public async Task An_over_length_message_type_is_capped_in_the_column_but_kept_i } } + [Test] + public async Task An_over_length_message_id_is_stored_in_full() + { + var failure = new IngestedFailure { MessageId = new string('m', 600) }; + + await Ingest(failure); + + var row = await GetFailedMessage(failure.UniqueMessageId); + + Assert.That(row.MessageId, Is.EqualTo(failure.MessageId)); + } + [Test] public async Task Later_attempt_replaces_the_stored_attempt() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/FailedErrorImportTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/FailedErrorImportTests.cs index 259aa61e0e..995a2e83c3 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/FailedErrorImportTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/FailedErrorImportTests.cs @@ -36,6 +36,18 @@ public async Task Stores_and_replays_a_failed_import() Assert.That(await FailedImportStore.QueryContainsFailedImports(), Is.False); } + [Test] + public async Task Stores_and_replays_a_failed_import_with_an_over_length_message_id() + { + var nativeId = new string('n', 600); + + await StoreImport(WellFormedHeaders(), Encoding.UTF8.GetBytes("1"), nativeId: nativeId); + + var replayed = await Replay(); + + Assert.That(replayed.Select(message => message.Id), Is.EqualTo(new[] { nativeId })); + } + [Test] public async Task Round_trips_a_binary_body_with_nul_bytes() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs index cf680e2418..8656464871 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs @@ -231,6 +231,75 @@ public async Task Keeps_system_messages_in_a_conversation() Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { system.UniqueMessageIdString })); } + [Test] + public async Task Filters_by_an_over_length_conversation() + { + var inConversation = new IngestedFailure { ConversationId = new string('c', 600) }; + + await Ingest(inConversation, new IngestedFailure()); + + var result = await MessagesViewStore.GetAllMessagesByConversation(inConversation.ConversationId, new PagingInfo(), new SortInfo(), false); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inConversation.UniqueMessageIdString })); + } + + [Test] + public async Task Filters_by_the_reported_id_of_an_over_length_conversation() + { + var inConversation = new IngestedFailure { ConversationId = new string('c', 600) }; + + await Ingest(inConversation, new IngestedFailure()); + + var all = await MessagesViewStore.GetAllMessages(new PagingInfo(), new SortInfo(), true); + var reportedId = all.Results.Single(view => view.Id == inConversation.UniqueMessageIdString).ConversationId; + + var result = await MessagesViewStore.GetAllMessagesByConversation(reportedId, new PagingInfo(), new SortInfo(), false); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inConversation.UniqueMessageIdString })); + } + + [Test] + public async Task Reports_an_over_length_conversation_id_that_needs_no_url_encoding() + { + await Ingest(new IngestedFailure { ConversationId = new string('c', 600) }); + + var view = await SingleMessage(); + + Assert.That(Uri.EscapeDataString(view.ConversationId), Is.EqualTo(view.ConversationId), "ServicePulse puts the conversation id into a URL path without encoding it"); + } + + [Test] + public async Task Keeps_over_length_conversations_that_share_a_prefix_apart() + { + var sharedPrefix = new string('c', 450); + var first = new IngestedFailure { ConversationId = sharedPrefix + new string('1', 150) }; + var second = new IngestedFailure { ConversationId = sharedPrefix + new string('2', 150) }; + + await Ingest(first, second); + + var firstConversation = await MessagesViewStore.GetAllMessagesByConversation(first.ConversationId, new PagingInfo(), new SortInfo(), false); + var secondConversation = await MessagesViewStore.GetAllMessagesByConversation(second.ConversationId, new PagingInfo(), new SortInfo(), false); + + using (Assert.EnterMultipleScope()) + { + Assert.That(firstConversation.Results.Select(view => view.Id), Is.EqualTo(new[] { first.UniqueMessageIdString })); + Assert.That(secondConversation.Results.Select(view => view.Id), Is.EqualTo(new[] { second.UniqueMessageIdString })); + } + } + + [Test] + public async Task Keeps_a_surrogate_pair_whole_when_shortening_a_conversation_id() + { + var conversationId = new string('c', 384) + char.ConvertFromUtf32(0x1F600) + new string('c', 214); + var failure = new IngestedFailure { ConversationId = conversationId }; + + await Ingest(failure); + + var view = await SingleMessage(); + + Assert.That(view.ConversationId, Does.StartWith(new string('c', 384) + "~"), "The pair straddles the end of the kept prefix, so it has to be left out whole"); + } + [Test] public async Task Searches_the_headers() { @@ -241,6 +310,26 @@ public async Task Searches_the_headers() await AssertSearchFinds("zarquon", matching); } + [Test] + public async Task Searches_a_non_ascii_word_in_the_headers() + { + var matching = new IngestedFailure { ExceptionMessage = "Bestellprüfung fehlgeschlagen" }; + + await Ingest(matching, new IngestedFailure()); + + await AssertSearchFinds("Bestellprüfung", matching); + } + + [Test] + public async Task Searches_a_single_quoted_word_in_the_headers() + { + var matching = new IngestedFailure { ExceptionMessage = "The given key 'CustomerId' was not present in the dictionary." }; + + await Ingest(matching, new IngestedFailure()); + + await AssertSearchFinds("CustomerId", matching); + } + [Test] public async Task Searches_the_body() { diff --git a/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs new file mode 100644 index 0000000000..fe441f981b --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/TimeRangeFilterTests.cs @@ -0,0 +1,130 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.Infrastructure; + +class TimeRangeFilterTests : ErrorIngestionTestBase +{ + [Test] + public async Task Filters_by_a_time_sent_range_with_an_offset() + { + var inside = await IngestThreeSentThreeMinutesApart(); + + var result = await MessagesViewStore.GetAllMessages(new PagingInfo(), new SortInfo(), true, + new DateTimeRange("2026-07-22T12:00:00+02:00", "2026-07-22T12:03:00+02:00")); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); + } + + [Test] + public async Task Filters_by_a_time_sent_range_without_a_zone_as_utc() + { + var inside = await IngestThreeSentThreeMinutesApart(); + + var result = await MessagesViewStore.GetAllMessages(new PagingInfo(), new SortInfo(), true, + new DateTimeRange("2026-07-22T10:00:00", "2026-07-22T10:03:00")); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); + } + + [Test] + public async Task Filters_by_a_modified_range_with_an_offset() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + + var result = await FailedMessageQueryStore.GetFailedMessages(null, $"{WithOffset(from)}...{WithOffset(to)}", null, new PagingInfo(), new SortInfo()); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); + } + + [Test] + public async Task Filters_by_a_modified_range_without_a_zone_as_utc() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + + var result = await FailedMessageQueryStore.GetFailedMessages(null, $"{WithoutZone(from)}...{WithoutZone(to)}", null, new PagingInfo(), new SortInfo()); + + Assert.That(result.Results.Select(view => view.Id), Is.EqualTo(new[] { inside })); + } + + [Test] + public async Task Unarchives_by_a_modified_range_with_an_offset() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + await SetEveryStatus(FailedMessageStatus.Archived); + + var unarchived = await FailedMessageLifecycleStore.UnArchiveMessagesByRange(ParsedWithOffset(from), ParsedWithOffset(to)); + + Assert.That(unarchived, Is.EqualTo(new[] { inside })); + } + + [Test] + public async Task Finds_pending_retries_by_a_modified_range_with_an_offset() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + await SetEveryStatus(FailedMessageStatus.RetryIssued); + + var pending = await FailedMessageRetryStore.GetRetryPendingMessages(ParsedWithOffset(from), ParsedWithOffset(to), "error"); + + Assert.That(pending, Is.EqualTo(new[] { inside }).IgnoreCase); + } + + [Test] + public async Task Processes_pending_retries_by_a_modified_range_with_an_offset() + { + var (inside, from, to) = await IngestThreeModifiedTwoMinutesApart(); + await SetEveryStatus(FailedMessageStatus.RetryIssued); + + var processed = new List(); + await FailedMessageRetryStore.ProcessPendingRetries(ParsedWithOffset(from), ParsedWithOffset(to), null, (id, _) => + { + processed.Add(id); + return Task.CompletedTask; + }); + + Assert.That(processed, Is.EqualTo(new[] { inside }).IgnoreCase); + } + + async Task IngestThreeSentThreeMinutesApart() + { + var inside = new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 10, 1, 0, DateTimeKind.Utc) }; + + await Ingest( + new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 9, 58, 0, DateTimeKind.Utc) }, + inside, + new IngestedFailure { TimeSent = new DateTime(2026, 7, 22, 10, 4, 0, DateTimeKind.Utc) }); + + return inside.UniqueMessageIdString; + } + + async Task<(string Inside, DateTime From, DateTime To)> IngestThreeModifiedTwoMinutesApart() + { + var start = Now; + var inside = new IngestedFailure(); + + await Ingest(new IngestedFailure()); + AdvanceClock(TimeSpan.FromMinutes(2)); + await Ingest(inside); + AdvanceClock(TimeSpan.FromMinutes(2)); + await Ingest(new IngestedFailure()); + + return (inside.UniqueMessageIdString, start.AddMinutes(1), start.AddMinutes(3)); + } + + Task SetEveryStatus(FailedMessageStatus status) => + Query(dbContext => dbContext.FailedMessages.ExecuteUpdateAsync(setters => setters.SetProperty(message => message.Status, status))); + + static string WithOffset(DateTime utc) => + new DateTimeOffset(utc).ToOffset(TimeSpan.FromHours(2)).ToString("yyyy-MM-ddTHH:mm:ss.fffffffzzz", CultureInfo.InvariantCulture); + + static string WithoutZone(DateTime utc) => utc.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture); + + static DateTime ParsedWithOffset(DateTime utc) => DateTime.Parse(WithOffset(utc), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); +}