From 089af9e84e9ccfda5c3be980f551ea6665bc83d0 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 11:00:17 +0200 Subject: [PATCH 01/16] Group the examples by transport and pin the registration rule The HTTP examples move under examples/Http/, making room for a Tcp/ series alongside them. Declared namespaces are unchanged, since ExampleRunner matches ClickHouse.Driver.Examples exactly whatever folder a file sits in. AGENTS.md carries the checklist for adding an example, because RunAllExamples in Program.cs is hand-maintained: an example left out of it still compiles, still lists, and still runs under --filter, so nothing reports that CI never runs it. DateTimeHandling, AsyncInsert and QBitSimilaritySearch were in that state; they are registered now, and the three that need a cluster, Cloud credentials or a JWT are listed as deliberate omissions. QBitSimilaritySearch read L2DistanceTransposed with GetFloat, which throws because the server returns Float64. The README's filter section described matching on the file name; the runner matches the class name. Co-Authored-By: Claude Opus 5 (1M context) --- examples/AGENTS.md | 54 +++++++++ examples/CLAUDE.md | 1 + .../Advanced/Advanced_001_QueryIdUsage.cs | 0 .../Advanced/Advanced_002_SessionIdUsage.cs | 0 .../Advanced_003_LongRunningQueries.cs | 0 .../Advanced/Advanced_004_CustomSettings.cs | 0 .../Advanced/Advanced_005_QueryStatistics.cs | 0 .../{ => Http}/Advanced/Advanced_006_Roles.cs | 0 .../Advanced/Advanced_007_CustomHeaders.cs | 0 .../Advanced_008_QueryCancellation.cs | 0 .../Advanced/Advanced_009_ReadOnlyUsers.cs | 0 .../Advanced_010_RetriesAndDeduplication.cs | 0 .../Advanced/Advanced_011_Compression.cs | 0 .../Advanced_012_ParameterTypeResolver.cs | 0 .../Advanced_013_ParameterFormatter.cs | 0 .../Advanced_014_ReadValueConverter.cs | 0 .../AspNet/AspNet_001_HealthChecks.cs | 0 .../Core/Auth_001_JwtAuthentication.cs | 0 .../{ => Http}/Core/Core_001_BasicUsage.cs | 0 .../Core_002_ConnectionStringConfiguration.cs | 0 .../Core/Core_003_DependencyInjection.cs | 0 .../Core/Core_004_HttpClientConfiguration.cs | 0 .../DataTypes/DataTypes_001_SimpleTypes.cs | 0 .../DataTypes_002_DateTimeHandling.cs | 0 .../DataTypes/DataTypes_003_ComplexTypes.cs | 0 .../DataTypes/DataTypes_004_StringHandling.cs | 0 .../DataTypes/DataTypes_005_JsonType.cs | 0 .../DataTypes/DataTypes_006_Geometry.cs | 0 .../Vector_001_QBitSimilaritySearch.cs | 4 +- .../Insert/Insert_001_SimpleDataInsert.cs | 0 .../Insert/Insert_002_BulkInsert.cs | 0 .../Insert/Insert_003_AsyncInsert.cs | 0 .../Insert/Insert_004_RawStreamInsert.cs | 0 .../Insert/Insert_005_InsertFromSelect.cs | 0 .../Insert/Insert_006_EphemeralColumns.cs | 0 ...nsert_007_UpsertsWithReplacingMergeTree.cs | 0 .../Insert/Insert_008_SchemaOptimization.cs | 0 .../Insert/Insert_009_PocoInsert.cs | 0 examples/{ => Http}/ORM/ORM_001_Dapper.cs | 0 examples/{ => Http}/ORM/ORM_002_Linq2Db.cs | 0 .../Select/Select_001_BasicSelect.cs | 0 .../Select/Select_002_SelectMetadata.cs | 0 .../Select_003_SelectWithParameterBinding.cs | 0 .../Select/Select_004_ExportToFile.cs | 0 .../Select/Select_005_CompressedRawExport.cs | 0 .../Select/Select_006_PocoSelect.cs | 0 .../Select/Select_007_ResponseCompression.cs | 0 .../Tables_001_CreateTableSingleNode.cs | 0 .../Tables/Tables_002_CreateTableCluster.cs | 0 .../Tables/Tables_003_CreateTableCloud.cs | 0 .../Testing/Testing_001_Testcontainers.cs | 0 ...roubleshooting_001_LoggingConfiguration.cs | 0 .../Troubleshooting_002_NetworkTracing.cs | 0 ...roubleshooting_003_OpenTelemetryTracing.cs | 0 examples/Program.cs | 12 ++ examples/README.md | 112 +++++++++--------- 56 files changed, 126 insertions(+), 57 deletions(-) create mode 100644 examples/AGENTS.md create mode 100644 examples/CLAUDE.md rename examples/{ => Http}/Advanced/Advanced_001_QueryIdUsage.cs (100%) rename examples/{ => Http}/Advanced/Advanced_002_SessionIdUsage.cs (100%) rename examples/{ => Http}/Advanced/Advanced_003_LongRunningQueries.cs (100%) rename examples/{ => Http}/Advanced/Advanced_004_CustomSettings.cs (100%) rename examples/{ => Http}/Advanced/Advanced_005_QueryStatistics.cs (100%) rename examples/{ => Http}/Advanced/Advanced_006_Roles.cs (100%) rename examples/{ => Http}/Advanced/Advanced_007_CustomHeaders.cs (100%) rename examples/{ => Http}/Advanced/Advanced_008_QueryCancellation.cs (100%) rename examples/{ => Http}/Advanced/Advanced_009_ReadOnlyUsers.cs (100%) rename examples/{ => Http}/Advanced/Advanced_010_RetriesAndDeduplication.cs (100%) rename examples/{ => Http}/Advanced/Advanced_011_Compression.cs (100%) rename examples/{ => Http}/Advanced/Advanced_012_ParameterTypeResolver.cs (100%) rename examples/{ => Http}/Advanced/Advanced_013_ParameterFormatter.cs (100%) rename examples/{ => Http}/Advanced/Advanced_014_ReadValueConverter.cs (100%) rename examples/{ => Http}/AspNet/AspNet_001_HealthChecks.cs (100%) rename examples/{ => Http}/Core/Auth_001_JwtAuthentication.cs (100%) rename examples/{ => Http}/Core/Core_001_BasicUsage.cs (100%) rename examples/{ => Http}/Core/Core_002_ConnectionStringConfiguration.cs (100%) rename examples/{ => Http}/Core/Core_003_DependencyInjection.cs (100%) rename examples/{ => Http}/Core/Core_004_HttpClientConfiguration.cs (100%) rename examples/{ => Http}/DataTypes/DataTypes_001_SimpleTypes.cs (100%) rename examples/{ => Http}/DataTypes/DataTypes_002_DateTimeHandling.cs (100%) rename examples/{ => Http}/DataTypes/DataTypes_003_ComplexTypes.cs (100%) rename examples/{ => Http}/DataTypes/DataTypes_004_StringHandling.cs (100%) rename examples/{ => Http}/DataTypes/DataTypes_005_JsonType.cs (100%) rename examples/{ => Http}/DataTypes/DataTypes_006_Geometry.cs (100%) rename examples/{ => Http}/DataTypes/Vector_001_QBitSimilaritySearch.cs (97%) rename examples/{ => Http}/Insert/Insert_001_SimpleDataInsert.cs (100%) rename examples/{ => Http}/Insert/Insert_002_BulkInsert.cs (100%) rename examples/{ => Http}/Insert/Insert_003_AsyncInsert.cs (100%) rename examples/{ => Http}/Insert/Insert_004_RawStreamInsert.cs (100%) rename examples/{ => Http}/Insert/Insert_005_InsertFromSelect.cs (100%) rename examples/{ => Http}/Insert/Insert_006_EphemeralColumns.cs (100%) rename examples/{ => Http}/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs (100%) rename examples/{ => Http}/Insert/Insert_008_SchemaOptimization.cs (100%) rename examples/{ => Http}/Insert/Insert_009_PocoInsert.cs (100%) rename examples/{ => Http}/ORM/ORM_001_Dapper.cs (100%) rename examples/{ => Http}/ORM/ORM_002_Linq2Db.cs (100%) rename examples/{ => Http}/Select/Select_001_BasicSelect.cs (100%) rename examples/{ => Http}/Select/Select_002_SelectMetadata.cs (100%) rename examples/{ => Http}/Select/Select_003_SelectWithParameterBinding.cs (100%) rename examples/{ => Http}/Select/Select_004_ExportToFile.cs (100%) rename examples/{ => Http}/Select/Select_005_CompressedRawExport.cs (100%) rename examples/{ => Http}/Select/Select_006_PocoSelect.cs (100%) rename examples/{ => Http}/Select/Select_007_ResponseCompression.cs (100%) rename examples/{ => Http}/Tables/Tables_001_CreateTableSingleNode.cs (100%) rename examples/{ => Http}/Tables/Tables_002_CreateTableCluster.cs (100%) rename examples/{ => Http}/Tables/Tables_003_CreateTableCloud.cs (100%) rename examples/{ => Http}/Testing/Testing_001_Testcontainers.cs (100%) rename examples/{ => Http}/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs (100%) rename examples/{ => Http}/Troubleshooting/Troubleshooting_002_NetworkTracing.cs (100%) rename examples/{ => Http}/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs (100%) diff --git a/examples/AGENTS.md b/examples/AGENTS.md new file mode 100644 index 000000000..3ca3c2313 --- /dev/null +++ b/examples/AGENTS.md @@ -0,0 +1,54 @@ +# Examples — contributor guide + +Each example is a self-contained, runnable demonstration of one topic. `dotnet run` in this directory +runs them all against a live server, and CI does the same on every pull request that touches +`examples/` or the driver, so an example that throws fails the build. + +Examples are grouped by transport: `Http/` uses `ClickHouseClient` / `ClickHouseConnection` over +HTTP, `Tcp/` uses `ClickHouseTcpClient` over the native protocol. See `Tcp/README.md` for what is +specific to the native client. + +## Adding an example + +Five steps. Skip any one of them and the example does not run. + +1. **Put the file** in the transport and category it belongs to: + `Http//_0NN_.cs` or `Tcp//Tcp_0NN_.cs`. Take the next + free number in that category. + +2. **Declare `namespace ClickHouse.Driver.Examples;`** — not a namespace derived from the folder. + `ExampleRunner` matches that namespace exactly and will not find a class in any other one. + +3. **Make it a `public static class` with a `public static Task Run()`.** Discovery requires both. + The class name is what `--list` prints and what `--filter` matches, so name it after the topic + (`BasicUsage`), not after the file (`Core_001_BasicUsage`). + +4. **Add it to `RunAllExamples` in `Program.cs`**, under its category banner, in file-number order. + That list is hand-maintained so the run order and the banners stay meaningful. An example missing + from it still compiles, still appears in `--list`, and still runs under `--filter` — it just never + runs in CI, so nothing tells you when it breaks. + +5. **Add it to the index in `README.md`**, in the matching section, with a one-line description. + +Then run it (`dotnet run -- --filter `) and read the output. An example whose output does not +teach the topic is not finished. + +## Examples deliberately left out of `RunAllExamples` + +Three need infrastructure the CI server does not have, so they are registered nowhere and run only +by explicit filter: + +- `Tables_002_CreateTableCluster` — needs a ClickHouse cluster. +- `Tables_003_CreateTableCloud` — needs ClickHouse Cloud credentials. +- `Auth_001_JwtAuthentication` — needs a JWT. + +If you add an example of that kind, leave it out of `RunAllExamples` and list it here. Otherwise the +omission is indistinguishable from having forgotten step 4. + +## Style + +- Console output is the teaching surface. Print what you did and what came back, not just "OK". +- Show one thing well rather than covering an API exhaustively. A reader who wants the full surface + reads the docs. +- Drop any table you create. +- Comments explain why the server or the driver behaves as it does, not what the line does. diff --git a/examples/CLAUDE.md b/examples/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/examples/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/examples/Advanced/Advanced_001_QueryIdUsage.cs b/examples/Http/Advanced/Advanced_001_QueryIdUsage.cs similarity index 100% rename from examples/Advanced/Advanced_001_QueryIdUsage.cs rename to examples/Http/Advanced/Advanced_001_QueryIdUsage.cs diff --git a/examples/Advanced/Advanced_002_SessionIdUsage.cs b/examples/Http/Advanced/Advanced_002_SessionIdUsage.cs similarity index 100% rename from examples/Advanced/Advanced_002_SessionIdUsage.cs rename to examples/Http/Advanced/Advanced_002_SessionIdUsage.cs diff --git a/examples/Advanced/Advanced_003_LongRunningQueries.cs b/examples/Http/Advanced/Advanced_003_LongRunningQueries.cs similarity index 100% rename from examples/Advanced/Advanced_003_LongRunningQueries.cs rename to examples/Http/Advanced/Advanced_003_LongRunningQueries.cs diff --git a/examples/Advanced/Advanced_004_CustomSettings.cs b/examples/Http/Advanced/Advanced_004_CustomSettings.cs similarity index 100% rename from examples/Advanced/Advanced_004_CustomSettings.cs rename to examples/Http/Advanced/Advanced_004_CustomSettings.cs diff --git a/examples/Advanced/Advanced_005_QueryStatistics.cs b/examples/Http/Advanced/Advanced_005_QueryStatistics.cs similarity index 100% rename from examples/Advanced/Advanced_005_QueryStatistics.cs rename to examples/Http/Advanced/Advanced_005_QueryStatistics.cs diff --git a/examples/Advanced/Advanced_006_Roles.cs b/examples/Http/Advanced/Advanced_006_Roles.cs similarity index 100% rename from examples/Advanced/Advanced_006_Roles.cs rename to examples/Http/Advanced/Advanced_006_Roles.cs diff --git a/examples/Advanced/Advanced_007_CustomHeaders.cs b/examples/Http/Advanced/Advanced_007_CustomHeaders.cs similarity index 100% rename from examples/Advanced/Advanced_007_CustomHeaders.cs rename to examples/Http/Advanced/Advanced_007_CustomHeaders.cs diff --git a/examples/Advanced/Advanced_008_QueryCancellation.cs b/examples/Http/Advanced/Advanced_008_QueryCancellation.cs similarity index 100% rename from examples/Advanced/Advanced_008_QueryCancellation.cs rename to examples/Http/Advanced/Advanced_008_QueryCancellation.cs diff --git a/examples/Advanced/Advanced_009_ReadOnlyUsers.cs b/examples/Http/Advanced/Advanced_009_ReadOnlyUsers.cs similarity index 100% rename from examples/Advanced/Advanced_009_ReadOnlyUsers.cs rename to examples/Http/Advanced/Advanced_009_ReadOnlyUsers.cs diff --git a/examples/Advanced/Advanced_010_RetriesAndDeduplication.cs b/examples/Http/Advanced/Advanced_010_RetriesAndDeduplication.cs similarity index 100% rename from examples/Advanced/Advanced_010_RetriesAndDeduplication.cs rename to examples/Http/Advanced/Advanced_010_RetriesAndDeduplication.cs diff --git a/examples/Advanced/Advanced_011_Compression.cs b/examples/Http/Advanced/Advanced_011_Compression.cs similarity index 100% rename from examples/Advanced/Advanced_011_Compression.cs rename to examples/Http/Advanced/Advanced_011_Compression.cs diff --git a/examples/Advanced/Advanced_012_ParameterTypeResolver.cs b/examples/Http/Advanced/Advanced_012_ParameterTypeResolver.cs similarity index 100% rename from examples/Advanced/Advanced_012_ParameterTypeResolver.cs rename to examples/Http/Advanced/Advanced_012_ParameterTypeResolver.cs diff --git a/examples/Advanced/Advanced_013_ParameterFormatter.cs b/examples/Http/Advanced/Advanced_013_ParameterFormatter.cs similarity index 100% rename from examples/Advanced/Advanced_013_ParameterFormatter.cs rename to examples/Http/Advanced/Advanced_013_ParameterFormatter.cs diff --git a/examples/Advanced/Advanced_014_ReadValueConverter.cs b/examples/Http/Advanced/Advanced_014_ReadValueConverter.cs similarity index 100% rename from examples/Advanced/Advanced_014_ReadValueConverter.cs rename to examples/Http/Advanced/Advanced_014_ReadValueConverter.cs diff --git a/examples/AspNet/AspNet_001_HealthChecks.cs b/examples/Http/AspNet/AspNet_001_HealthChecks.cs similarity index 100% rename from examples/AspNet/AspNet_001_HealthChecks.cs rename to examples/Http/AspNet/AspNet_001_HealthChecks.cs diff --git a/examples/Core/Auth_001_JwtAuthentication.cs b/examples/Http/Core/Auth_001_JwtAuthentication.cs similarity index 100% rename from examples/Core/Auth_001_JwtAuthentication.cs rename to examples/Http/Core/Auth_001_JwtAuthentication.cs diff --git a/examples/Core/Core_001_BasicUsage.cs b/examples/Http/Core/Core_001_BasicUsage.cs similarity index 100% rename from examples/Core/Core_001_BasicUsage.cs rename to examples/Http/Core/Core_001_BasicUsage.cs diff --git a/examples/Core/Core_002_ConnectionStringConfiguration.cs b/examples/Http/Core/Core_002_ConnectionStringConfiguration.cs similarity index 100% rename from examples/Core/Core_002_ConnectionStringConfiguration.cs rename to examples/Http/Core/Core_002_ConnectionStringConfiguration.cs diff --git a/examples/Core/Core_003_DependencyInjection.cs b/examples/Http/Core/Core_003_DependencyInjection.cs similarity index 100% rename from examples/Core/Core_003_DependencyInjection.cs rename to examples/Http/Core/Core_003_DependencyInjection.cs diff --git a/examples/Core/Core_004_HttpClientConfiguration.cs b/examples/Http/Core/Core_004_HttpClientConfiguration.cs similarity index 100% rename from examples/Core/Core_004_HttpClientConfiguration.cs rename to examples/Http/Core/Core_004_HttpClientConfiguration.cs diff --git a/examples/DataTypes/DataTypes_001_SimpleTypes.cs b/examples/Http/DataTypes/DataTypes_001_SimpleTypes.cs similarity index 100% rename from examples/DataTypes/DataTypes_001_SimpleTypes.cs rename to examples/Http/DataTypes/DataTypes_001_SimpleTypes.cs diff --git a/examples/DataTypes/DataTypes_002_DateTimeHandling.cs b/examples/Http/DataTypes/DataTypes_002_DateTimeHandling.cs similarity index 100% rename from examples/DataTypes/DataTypes_002_DateTimeHandling.cs rename to examples/Http/DataTypes/DataTypes_002_DateTimeHandling.cs diff --git a/examples/DataTypes/DataTypes_003_ComplexTypes.cs b/examples/Http/DataTypes/DataTypes_003_ComplexTypes.cs similarity index 100% rename from examples/DataTypes/DataTypes_003_ComplexTypes.cs rename to examples/Http/DataTypes/DataTypes_003_ComplexTypes.cs diff --git a/examples/DataTypes/DataTypes_004_StringHandling.cs b/examples/Http/DataTypes/DataTypes_004_StringHandling.cs similarity index 100% rename from examples/DataTypes/DataTypes_004_StringHandling.cs rename to examples/Http/DataTypes/DataTypes_004_StringHandling.cs diff --git a/examples/DataTypes/DataTypes_005_JsonType.cs b/examples/Http/DataTypes/DataTypes_005_JsonType.cs similarity index 100% rename from examples/DataTypes/DataTypes_005_JsonType.cs rename to examples/Http/DataTypes/DataTypes_005_JsonType.cs diff --git a/examples/DataTypes/DataTypes_006_Geometry.cs b/examples/Http/DataTypes/DataTypes_006_Geometry.cs similarity index 100% rename from examples/DataTypes/DataTypes_006_Geometry.cs rename to examples/Http/DataTypes/DataTypes_006_Geometry.cs diff --git a/examples/DataTypes/Vector_001_QBitSimilaritySearch.cs b/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs similarity index 97% rename from examples/DataTypes/Vector_001_QBitSimilaritySearch.cs rename to examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs index 589ad4ad3..fe7073562 100644 --- a/examples/DataTypes/Vector_001_QBitSimilaritySearch.cs +++ b/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs @@ -63,7 +63,7 @@ ORDER BY distance while (reader.Read()) { var word = reader.GetString(0); - var distance = reader.GetFloat(1); + var distance = reader.GetDouble(1); Console.WriteLine($"{word,-12}\t{distance:F6}"); } } @@ -86,7 +86,7 @@ ORDER BY distance while (reader.Read()) { var word = reader.GetString(0); - var distance = reader.GetFloat(1); + var distance = reader.GetDouble(1); Console.WriteLine($"{word,-12}\t{distance:F6}"); } } diff --git a/examples/Insert/Insert_001_SimpleDataInsert.cs b/examples/Http/Insert/Insert_001_SimpleDataInsert.cs similarity index 100% rename from examples/Insert/Insert_001_SimpleDataInsert.cs rename to examples/Http/Insert/Insert_001_SimpleDataInsert.cs diff --git a/examples/Insert/Insert_002_BulkInsert.cs b/examples/Http/Insert/Insert_002_BulkInsert.cs similarity index 100% rename from examples/Insert/Insert_002_BulkInsert.cs rename to examples/Http/Insert/Insert_002_BulkInsert.cs diff --git a/examples/Insert/Insert_003_AsyncInsert.cs b/examples/Http/Insert/Insert_003_AsyncInsert.cs similarity index 100% rename from examples/Insert/Insert_003_AsyncInsert.cs rename to examples/Http/Insert/Insert_003_AsyncInsert.cs diff --git a/examples/Insert/Insert_004_RawStreamInsert.cs b/examples/Http/Insert/Insert_004_RawStreamInsert.cs similarity index 100% rename from examples/Insert/Insert_004_RawStreamInsert.cs rename to examples/Http/Insert/Insert_004_RawStreamInsert.cs diff --git a/examples/Insert/Insert_005_InsertFromSelect.cs b/examples/Http/Insert/Insert_005_InsertFromSelect.cs similarity index 100% rename from examples/Insert/Insert_005_InsertFromSelect.cs rename to examples/Http/Insert/Insert_005_InsertFromSelect.cs diff --git a/examples/Insert/Insert_006_EphemeralColumns.cs b/examples/Http/Insert/Insert_006_EphemeralColumns.cs similarity index 100% rename from examples/Insert/Insert_006_EphemeralColumns.cs rename to examples/Http/Insert/Insert_006_EphemeralColumns.cs diff --git a/examples/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs b/examples/Http/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs similarity index 100% rename from examples/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs rename to examples/Http/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs diff --git a/examples/Insert/Insert_008_SchemaOptimization.cs b/examples/Http/Insert/Insert_008_SchemaOptimization.cs similarity index 100% rename from examples/Insert/Insert_008_SchemaOptimization.cs rename to examples/Http/Insert/Insert_008_SchemaOptimization.cs diff --git a/examples/Insert/Insert_009_PocoInsert.cs b/examples/Http/Insert/Insert_009_PocoInsert.cs similarity index 100% rename from examples/Insert/Insert_009_PocoInsert.cs rename to examples/Http/Insert/Insert_009_PocoInsert.cs diff --git a/examples/ORM/ORM_001_Dapper.cs b/examples/Http/ORM/ORM_001_Dapper.cs similarity index 100% rename from examples/ORM/ORM_001_Dapper.cs rename to examples/Http/ORM/ORM_001_Dapper.cs diff --git a/examples/ORM/ORM_002_Linq2Db.cs b/examples/Http/ORM/ORM_002_Linq2Db.cs similarity index 100% rename from examples/ORM/ORM_002_Linq2Db.cs rename to examples/Http/ORM/ORM_002_Linq2Db.cs diff --git a/examples/Select/Select_001_BasicSelect.cs b/examples/Http/Select/Select_001_BasicSelect.cs similarity index 100% rename from examples/Select/Select_001_BasicSelect.cs rename to examples/Http/Select/Select_001_BasicSelect.cs diff --git a/examples/Select/Select_002_SelectMetadata.cs b/examples/Http/Select/Select_002_SelectMetadata.cs similarity index 100% rename from examples/Select/Select_002_SelectMetadata.cs rename to examples/Http/Select/Select_002_SelectMetadata.cs diff --git a/examples/Select/Select_003_SelectWithParameterBinding.cs b/examples/Http/Select/Select_003_SelectWithParameterBinding.cs similarity index 100% rename from examples/Select/Select_003_SelectWithParameterBinding.cs rename to examples/Http/Select/Select_003_SelectWithParameterBinding.cs diff --git a/examples/Select/Select_004_ExportToFile.cs b/examples/Http/Select/Select_004_ExportToFile.cs similarity index 100% rename from examples/Select/Select_004_ExportToFile.cs rename to examples/Http/Select/Select_004_ExportToFile.cs diff --git a/examples/Select/Select_005_CompressedRawExport.cs b/examples/Http/Select/Select_005_CompressedRawExport.cs similarity index 100% rename from examples/Select/Select_005_CompressedRawExport.cs rename to examples/Http/Select/Select_005_CompressedRawExport.cs diff --git a/examples/Select/Select_006_PocoSelect.cs b/examples/Http/Select/Select_006_PocoSelect.cs similarity index 100% rename from examples/Select/Select_006_PocoSelect.cs rename to examples/Http/Select/Select_006_PocoSelect.cs diff --git a/examples/Select/Select_007_ResponseCompression.cs b/examples/Http/Select/Select_007_ResponseCompression.cs similarity index 100% rename from examples/Select/Select_007_ResponseCompression.cs rename to examples/Http/Select/Select_007_ResponseCompression.cs diff --git a/examples/Tables/Tables_001_CreateTableSingleNode.cs b/examples/Http/Tables/Tables_001_CreateTableSingleNode.cs similarity index 100% rename from examples/Tables/Tables_001_CreateTableSingleNode.cs rename to examples/Http/Tables/Tables_001_CreateTableSingleNode.cs diff --git a/examples/Tables/Tables_002_CreateTableCluster.cs b/examples/Http/Tables/Tables_002_CreateTableCluster.cs similarity index 100% rename from examples/Tables/Tables_002_CreateTableCluster.cs rename to examples/Http/Tables/Tables_002_CreateTableCluster.cs diff --git a/examples/Tables/Tables_003_CreateTableCloud.cs b/examples/Http/Tables/Tables_003_CreateTableCloud.cs similarity index 100% rename from examples/Tables/Tables_003_CreateTableCloud.cs rename to examples/Http/Tables/Tables_003_CreateTableCloud.cs diff --git a/examples/Testing/Testing_001_Testcontainers.cs b/examples/Http/Testing/Testing_001_Testcontainers.cs similarity index 100% rename from examples/Testing/Testing_001_Testcontainers.cs rename to examples/Http/Testing/Testing_001_Testcontainers.cs diff --git a/examples/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs b/examples/Http/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs similarity index 100% rename from examples/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs rename to examples/Http/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs diff --git a/examples/Troubleshooting/Troubleshooting_002_NetworkTracing.cs b/examples/Http/Troubleshooting/Troubleshooting_002_NetworkTracing.cs similarity index 100% rename from examples/Troubleshooting/Troubleshooting_002_NetworkTracing.cs rename to examples/Http/Troubleshooting/Troubleshooting_002_NetworkTracing.cs diff --git a/examples/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs b/examples/Http/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs similarity index 100% rename from examples/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs rename to examples/Http/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs diff --git a/examples/Program.cs b/examples/Program.cs index e41b65690..9bf3bced0 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -109,6 +109,10 @@ private static async Task RunAllExamples(bool isInteractive) await BulkInsert.Run(); WaitForUser(isInteractive); + Console.WriteLine($"\n\nRunning: {nameof(AsyncInsert)}"); + await AsyncInsert.Run(); + WaitForUser(isInteractive); + Console.WriteLine($"\n\nRunning: {nameof(RawStreamInsert)}"); await RawStreamInsert.Run(); WaitForUser(isInteractive); @@ -175,6 +179,10 @@ private static async Task RunAllExamples(bool isInteractive) await SimpleTypes.Run(); WaitForUser(isInteractive); + Console.WriteLine($"\n\nRunning: {nameof(DateTimeHandling)}"); + await DateTimeHandling.Run(); + WaitForUser(isInteractive); + Console.WriteLine($"\n\nRunning: {nameof(ComplexTypes)}"); await ComplexTypes.Run(); WaitForUser(isInteractive); @@ -191,6 +199,10 @@ private static async Task RunAllExamples(bool isInteractive) await GeometryTypes.Run(); WaitForUser(isInteractive); + Console.WriteLine($"\n\nRunning: {nameof(QBitSimilaritySearch)}"); + await QBitSimilaritySearch.Run(); + WaitForUser(isInteractive); + // ORM Integration Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ORM INTEGRATION"); diff --git a/examples/README.md b/examples/README.md index b4d75f31a..69b4a054e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,94 +6,96 @@ This directory contains examples demonstrating various features and usage patter We aim to cover various scenarios of driver usage with these examples. You should be able to run any of these examples by following the instructions in the [How to run](#how-to-run) section below. -If something is missing, or you found a mistake in one of these examples, please open an issue or a pull request. +If something is missing, or you found a mistake in one of these examples, please open an issue or a pull request. [AGENTS.md](AGENTS.md) has the checklist for adding one. + +Examples are grouped by transport. Everything under [Http/](Http) uses `ClickHouseClient` or `ClickHouseConnection` over HTTP. ## Examples ### Core Usage & Configuration -- [Core_001_BasicUsage.cs](Core/Core_001_BasicUsage.cs) - Creating a client, tables, and performing basic insert/select operations (using ClickHouseClientSettings) -- [Core_002_ConnectionStringConfiguration.cs](Core/Core_002_ConnectionStringConfiguration.cs) - Various connection string formats and configuration options -- [Core_003_DependencyInjection.cs](Core/Core_003_DependencyInjection.cs) - Using ClickHouse with dependency injection and config binding -- [Core_004_HttpClientConfiguration.cs](Core/Core_004_HttpClientConfiguration.cs) - Providing custom HttpClient or IHttpClientFactory for SSL/TLS, proxy, timeouts, and more control over connection settings +- [Core_001_BasicUsage.cs](Http/Core/Core_001_BasicUsage.cs) - Creating a client, tables, and performing basic insert/select operations (using ClickHouseClientSettings) +- [Core_002_ConnectionStringConfiguration.cs](Http/Core/Core_002_ConnectionStringConfiguration.cs) - Various connection string formats and configuration options +- [Core_003_DependencyInjection.cs](Http/Core/Core_003_DependencyInjection.cs) - Using ClickHouse with dependency injection and config binding +- [Core_004_HttpClientConfiguration.cs](Http/Core/Core_004_HttpClientConfiguration.cs) - Providing custom HttpClient or IHttpClientFactory for SSL/TLS, proxy, timeouts, and more control over connection settings ### ASP.NET Integration -- [AspNet_001_HealthChecks.cs](AspNet/AspNet_001_HealthChecks.cs) - Implementing ASP.NET health checks for ClickHouse +- [AspNet_001_HealthChecks.cs](Http/AspNet/AspNet_001_HealthChecks.cs) - Implementing ASP.NET health checks for ClickHouse ### Authentication -- [Auth_001_JwtAuthentication.cs](Core/Auth_001_JwtAuthentication.cs) - Using JWT/Bearer token authentication with ClickHouse. +- [Auth_001_JwtAuthentication.cs](Http/Core/Auth_001_JwtAuthentication.cs) - Using JWT/Bearer token authentication with ClickHouse. ### Creating Tables -- [Tables_001_CreateTableSingleNode.cs](Tables/Tables_001_CreateTableSingleNode.cs) - Creating tables with different engines and data types on a single-node deployment -- [Tables_002_CreateTableCluster.cs](Tables/Tables_002_CreateTableCluster.cs) - Creating ReplicatedMergeTree tables on an on-premises ClickHouse cluster with ON CLUSTER and macros -- [Tables_003_CreateTableCloud.cs](Tables/Tables_003_CreateTableCloud.cs) - Creating tables on ClickHouse Cloud (automatic replication, no ENGINE needed) +- [Tables_001_CreateTableSingleNode.cs](Http/Tables/Tables_001_CreateTableSingleNode.cs) - Creating tables with different engines and data types on a single-node deployment +- [Tables_002_CreateTableCluster.cs](Http/Tables/Tables_002_CreateTableCluster.cs) - Creating ReplicatedMergeTree tables on an on-premises ClickHouse cluster with ON CLUSTER and macros +- [Tables_003_CreateTableCloud.cs](Http/Tables/Tables_003_CreateTableCloud.cs) - Creating tables on ClickHouse Cloud (automatic replication, no ENGINE needed) ### Inserting Data -- [Insert_001_SimpleDataInsert.cs](Insert/Insert_001_SimpleDataInsert.cs) - Basic data insertion using parameterized queries -- [Insert_002_BulkInsert.cs](Insert/Insert_002_BulkInsert.cs) - High-performance bulk data insertion using `InsertBinaryAsync()` -- [Insert_003_AsyncInsert.cs](Insert/Insert_003_AsyncInsert.cs) - Server-side batching with async inserts for high-concurrency workloads -- [Insert_004_RawStreamInsert.cs](Insert/Insert_004_RawStreamInsert.cs) - Inserting raw data streams from files or memory (CSV, JSON, Parquet, etc.) -- [Insert_005_InsertFromSelect.cs](Insert/Insert_005_InsertFromSelect.cs) - Using INSERT FROM SELECT for ETL, data transformation, and loading from external sources (S3, URL, remote servers) -- [Insert_006_EphemeralColumns.cs](Insert/Insert_006_EphemeralColumns.cs) - Using EPHEMERAL columns to transform input data before storage -- [Insert_007_UpsertsWithReplacingMergeTree.cs](Insert/Insert_007_UpsertsWithReplacingMergeTree.cs) - Upsert patterns using ReplacingMergeTree with version and deleted columns -- [Insert_008_SchemaOptimization.cs](Insert/Insert_008_SchemaOptimization.cs) - Skipping the schema probe query with `ColumnTypes` or `UseSchemaCache` -- [Insert_009_PocoInsert.cs](Insert/Insert_009_PocoInsert.cs) - Inserting strongly-typed POCO objects with `InsertBinaryAsync`, attribute mapping, and schema probe optimization +- [Insert_001_SimpleDataInsert.cs](Http/Insert/Insert_001_SimpleDataInsert.cs) - Basic data insertion using parameterized queries +- [Insert_002_BulkInsert.cs](Http/Insert/Insert_002_BulkInsert.cs) - High-performance bulk data insertion using `InsertBinaryAsync()` +- [Insert_003_AsyncInsert.cs](Http/Insert/Insert_003_AsyncInsert.cs) - Server-side batching with async inserts for high-concurrency workloads +- [Insert_004_RawStreamInsert.cs](Http/Insert/Insert_004_RawStreamInsert.cs) - Inserting raw data streams from files or memory (CSV, JSON, Parquet, etc.) +- [Insert_005_InsertFromSelect.cs](Http/Insert/Insert_005_InsertFromSelect.cs) - Using INSERT FROM SELECT for ETL, data transformation, and loading from external sources (S3, URL, remote servers) +- [Insert_006_EphemeralColumns.cs](Http/Insert/Insert_006_EphemeralColumns.cs) - Using EPHEMERAL columns to transform input data before storage +- [Insert_007_UpsertsWithReplacingMergeTree.cs](Http/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs) - Upsert patterns using ReplacingMergeTree with version and deleted columns +- [Insert_008_SchemaOptimization.cs](Http/Insert/Insert_008_SchemaOptimization.cs) - Skipping the schema probe query with `ColumnTypes` or `UseSchemaCache` +- [Insert_009_PocoInsert.cs](Http/Insert/Insert_009_PocoInsert.cs) - Inserting strongly-typed POCO objects with `InsertBinaryAsync`, attribute mapping, and schema probe optimization ### Selecting Data -- [Select_001_BasicSelect.cs](Select/Select_001_BasicSelect.cs) - Basic SELECT queries and reading the results -- [Select_002_SelectMetadata.cs](Select/Select_002_SelectMetadata.cs) - Column metadata overview -- [Select_003_SelectWithParameterBinding.cs](Select/Select_003_SelectWithParameterBinding.cs) - Parameterized queries for safe and dynamic SQL construction -- [Select_004_ExportToFile.cs](Select/Select_004_ExportToFile.cs) - Exporting query results to files (JSONEachRow, Parquet, etc.) -- [Select_005_CompressedRawExport.cs](Select/Select_005_CompressedRawExport.cs) - Per-query `AcceptEncoding` override to stream a compressed raw export (e.g. LZ4-compressed Parquet) straight to a file with `ExecuteRawResultAsync` -- [Select_006_PocoSelect.cs](Select/Select_006_PocoSelect.cs) - Reading query results into strongly-typed POCO objects with `QueryAsync` (streaming) and `reader.MapTo` (per-row), including `[ClickHouseColumn(Name)]` aliases and `[ClickHouseNotMapped]` exclusion -- [Select_007_ResponseCompression.cs](Select/Select_007_ResponseCompression.cs) - Transport compression of responses: the codec the driver negotiates by default, how to override it with `AcceptEncoding`, and why raw exports are exempt +- [Select_001_BasicSelect.cs](Http/Select/Select_001_BasicSelect.cs) - Basic SELECT queries and reading the results +- [Select_002_SelectMetadata.cs](Http/Select/Select_002_SelectMetadata.cs) - Column metadata overview +- [Select_003_SelectWithParameterBinding.cs](Http/Select/Select_003_SelectWithParameterBinding.cs) - Parameterized queries for safe and dynamic SQL construction +- [Select_004_ExportToFile.cs](Http/Select/Select_004_ExportToFile.cs) - Exporting query results to files (JSONEachRow, Parquet, etc.) +- [Select_005_CompressedRawExport.cs](Http/Select/Select_005_CompressedRawExport.cs) - Per-query `AcceptEncoding` override to stream a compressed raw export (e.g. LZ4-compressed Parquet) straight to a file with `ExecuteRawResultAsync` +- [Select_006_PocoSelect.cs](Http/Select/Select_006_PocoSelect.cs) - Reading query results into strongly-typed POCO objects with `QueryAsync` (streaming) and `reader.MapTo` (per-row), including `[ClickHouseColumn(Name)]` aliases and `[ClickHouseNotMapped]` exclusion +- [Select_007_ResponseCompression.cs](Http/Select/Select_007_ResponseCompression.cs) - Transport compression of responses: the codec the driver negotiates by default, how to override it with `AcceptEncoding`, and why raw exports are exempt ### Data Types -- [DataTypes_001_SimpleTypes.cs](DataTypes/DataTypes_001_SimpleTypes.cs) - Simple/scalar data types: integers (Int8-Int256), floats, decimals (ClickHouseDecimal), boolean -- [DataTypes_002_DateTimeHandling.cs](DataTypes/DataTypes_002_DateTimeHandling.cs) - Comprehensive guide to DateTime, DateTime64, Date, Date32, timezones, DateTime.Kind behavior, and DateTimeOffset -- [DataTypes_003_ComplexTypes.cs](DataTypes/DataTypes_003_ComplexTypes.cs) - Working with complex data types: Arrays, Maps, Tuples, IP addresses, and Nested structures -- [DataTypes_004_StringHandling.cs](DataTypes/DataTypes_004_StringHandling.cs) - String and FixedString handling, binary data, ReadStringsAsByteArrays setting, and writing from Streams -- [DataTypes_005_JsonType.cs](DataTypes/DataTypes_005_JsonType.cs) - Working with JSON type: reading as JsonObject or string, writing from various sources, and configuring JsonReadMode/JsonWriteMode -- [DataTypes_006_Geometry.cs](DataTypes/DataTypes_006_Geometry.cs) - Geometry types: Point, Polygon, WKT parsing, H3 geospatial indexing, point-in-polygon checks, and great circle distance calculations -- [Vector_001_QBitSimilaritySearch.cs](DataTypes/Vector_001_QBitSimilaritySearch.cs) - Vector similarity search using quantized binary embeddings +- [DataTypes_001_SimpleTypes.cs](Http/DataTypes/DataTypes_001_SimpleTypes.cs) - Simple/scalar data types: integers (Int8-Int256), floats, decimals (ClickHouseDecimal), boolean +- [DataTypes_002_DateTimeHandling.cs](Http/DataTypes/DataTypes_002_DateTimeHandling.cs) - Comprehensive guide to DateTime, DateTime64, Date, Date32, timezones, DateTime.Kind behavior, and DateTimeOffset +- [DataTypes_003_ComplexTypes.cs](Http/DataTypes/DataTypes_003_ComplexTypes.cs) - Working with complex data types: Arrays, Maps, Tuples, IP addresses, and Nested structures +- [DataTypes_004_StringHandling.cs](Http/DataTypes/DataTypes_004_StringHandling.cs) - String and FixedString handling, binary data, ReadStringsAsByteArrays setting, and writing from Streams +- [DataTypes_005_JsonType.cs](Http/DataTypes/DataTypes_005_JsonType.cs) - Working with JSON type: reading as JsonObject or string, writing from various sources, and configuring JsonReadMode/JsonWriteMode +- [DataTypes_006_Geometry.cs](Http/DataTypes/DataTypes_006_Geometry.cs) - Geometry types: Point, Polygon, WKT parsing, H3 geospatial indexing, point-in-polygon checks, and great circle distance calculations +- [Vector_001_QBitSimilaritySearch.cs](Http/DataTypes/Vector_001_QBitSimilaritySearch.cs) - Vector similarity search using quantized binary embeddings ### ORM Integration -- [ORM_001_Dapper.cs](ORM/ORM_001_Dapper.cs) - Using Dapper and Dapper.Contrib with ClickHouse: queries, inserts, type handlers, and known limitations -- [ORM_002_Linq2Db.cs](ORM/ORM_002_Linq2Db.cs) - Using linq2db with ClickHouse: LINQ queries, inserts, BulkCopy, and entity mapping +- [ORM_001_Dapper.cs](Http/ORM/ORM_001_Dapper.cs) - Using Dapper and Dapper.Contrib with ClickHouse: queries, inserts, type handlers, and known limitations +- [ORM_002_Linq2Db.cs](Http/ORM/ORM_002_Linq2Db.cs) - Using linq2db with ClickHouse: LINQ queries, inserts, BulkCopy, and entity mapping ### Advanced Features -- [Advanced_001_QueryIdUsage.cs](Advanced/Advanced_001_QueryIdUsage.cs) - Using Query IDs to track and monitor query execution -- [Advanced_002_SessionIdUsage.cs](Advanced/Advanced_002_SessionIdUsage.cs) - Using Session IDs for temporary tables and session state (with important limitations) -- [Advanced_003_LongRunningQueries.cs](Advanced/Advanced_003_LongRunningQueries.cs) - Strategies for handling long-running queries (progress headers and fire-and-forget patterns) -- [Advanced_004_CustomSettings.cs](Advanced/Advanced_004_CustomSettings.cs) - Using custom ClickHouse server settings for resource limits and query optimization -- [Advanced_005_QueryStatistics.cs](Advanced/Advanced_005_QueryStatistics.cs) - Accessing and using query statistics for performance monitoring and optimization decisions -- [Advanced_006_Roles.cs](Advanced/Advanced_006_Roles.cs) - Using ClickHouse roles to control permissions at connection and command levels -- [Advanced_007_CustomHeaders.cs](Advanced/Advanced_007_CustomHeaders.cs) - Using custom HTTP headers for proxy authentication, distributed tracing, etc -- [Advanced_008_QueryCancellation.cs](Advanced/Advanced_008_QueryCancellation.cs) - Using CancellationToken to cancel long-running queries -- [Advanced_009_ReadOnlyUsers.cs](Advanced/Advanced_009_ReadOnlyUsers.cs) - Working with READONLY = 1 users and their limitations -- [Advanced_010_RetriesAndDeduplication.cs](Advanced/Advanced_010_RetriesAndDeduplication.cs) - Retry patterns with Polly and ReplacingMergeTree for exactly-once insert semantics -- [Advanced_011_Compression.cs](Advanced/Advanced_011_Compression.cs) - Understanding the UseCompression setting: how it works, when to disable it, and custom HttpClient requirements -- [Advanced_012_ParameterTypeResolver.cs](Advanced/Advanced_012_ParameterTypeResolver.cs) - Customizing default type mappings for @-style parameters (DateTime→DateTime64, decimal precision, custom resolvers) -- [Advanced_013_ParameterFormatter.cs](Advanced/Advanced_013_ParameterFormatter.cs) - Customizing how parameter values are serialized for HTTP transport (custom DateTime format, array-element formatting, escaping caveats for string-like types in composites) -- [Advanced_014_ReadValueConverter.cs](Advanced/Advanced_014_ReadValueConverter.cs) - Transforming values returned by the data reader (DateTime.Kind, string normalization, per-query overrides) +- [Advanced_001_QueryIdUsage.cs](Http/Advanced/Advanced_001_QueryIdUsage.cs) - Using Query IDs to track and monitor query execution +- [Advanced_002_SessionIdUsage.cs](Http/Advanced/Advanced_002_SessionIdUsage.cs) - Using Session IDs for temporary tables and session state (with important limitations) +- [Advanced_003_LongRunningQueries.cs](Http/Advanced/Advanced_003_LongRunningQueries.cs) - Strategies for handling long-running queries (progress headers and fire-and-forget patterns) +- [Advanced_004_CustomSettings.cs](Http/Advanced/Advanced_004_CustomSettings.cs) - Using custom ClickHouse server settings for resource limits and query optimization +- [Advanced_005_QueryStatistics.cs](Http/Advanced/Advanced_005_QueryStatistics.cs) - Accessing and using query statistics for performance monitoring and optimization decisions +- [Advanced_006_Roles.cs](Http/Advanced/Advanced_006_Roles.cs) - Using ClickHouse roles to control permissions at connection and command levels +- [Advanced_007_CustomHeaders.cs](Http/Advanced/Advanced_007_CustomHeaders.cs) - Using custom HTTP headers for proxy authentication, distributed tracing, etc +- [Advanced_008_QueryCancellation.cs](Http/Advanced/Advanced_008_QueryCancellation.cs) - Using CancellationToken to cancel long-running queries +- [Advanced_009_ReadOnlyUsers.cs](Http/Advanced/Advanced_009_ReadOnlyUsers.cs) - Working with READONLY = 1 users and their limitations +- [Advanced_010_RetriesAndDeduplication.cs](Http/Advanced/Advanced_010_RetriesAndDeduplication.cs) - Retry patterns with Polly and ReplacingMergeTree for exactly-once insert semantics +- [Advanced_011_Compression.cs](Http/Advanced/Advanced_011_Compression.cs) - Understanding the UseCompression setting: how it works, when to disable it, and custom HttpClient requirements +- [Advanced_012_ParameterTypeResolver.cs](Http/Advanced/Advanced_012_ParameterTypeResolver.cs) - Customizing default type mappings for @-style parameters (DateTime→DateTime64, decimal precision, custom resolvers) +- [Advanced_013_ParameterFormatter.cs](Http/Advanced/Advanced_013_ParameterFormatter.cs) - Customizing how parameter values are serialized for HTTP transport (custom DateTime format, array-element formatting, escaping caveats for string-like types in composites) +- [Advanced_014_ReadValueConverter.cs](Http/Advanced/Advanced_014_ReadValueConverter.cs) - Transforming values returned by the data reader (DateTime.Kind, string normalization, per-query overrides) ### Troubleshooting -- [Troubleshooting_001_LoggingConfiguration.cs](Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs) - Setting up logging with Microsoft.Extensions.Logging to view diagnostic information -- [Troubleshooting_002_NetworkTracing.cs](Troubleshooting/Troubleshooting_002_NetworkTracing.cs) - Enabling low-level .NET network tracing for debugging connection issues (HTTP, Sockets, DNS, TLS) -- [Troubleshooting_003_OpenTelemetryTracing.cs](Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs) - Collecting OpenTelemetry traces from the driver for distributed tracing and observability +- [Troubleshooting_001_LoggingConfiguration.cs](Http/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs) - Setting up logging with Microsoft.Extensions.Logging to view diagnostic information +- [Troubleshooting_002_NetworkTracing.cs](Http/Troubleshooting/Troubleshooting_002_NetworkTracing.cs) - Enabling low-level .NET network tracing for debugging connection issues (HTTP, Sockets, DNS, TLS) +- [Troubleshooting_003_OpenTelemetryTracing.cs](Http/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs) - Collecting OpenTelemetry traces from the driver for distributed tracing and observability ### Testing -- [Testing_001_Testcontainers.cs](Testing/Testing_001_Testcontainers.cs) - Using Testcontainers to spin up ephemeral ClickHouse instances for integration testing +- [Testing_001_Testcontainers.cs](Http/Testing/Testing_001_Testcontainers.cs) - Using Testcontainers to spin up ephemeral ClickHouse instances for integration testing ## How to run @@ -126,7 +128,7 @@ dotnet run -- --filter basicusage dotnet run -- basicusage ``` -The filter uses fuzzy matching - it matches against any substring of the example filename, ignoring case and underscores. For example, `core001`, `core_001`, `basicusage`, and `Basic` would all match `Core_001_BasicUsage`. +The filter matches the example's class name, which `--list` prints. A class name is the topic without the file's category prefix: `Core_001_BasicUsage.cs` declares `class BasicUsage`. Matching ignores case and underscores and accepts any substring, so `basicusage`, `basic` and `usage` all match it. The file's `core001` prefix does not. ### Connection configuration @@ -134,7 +136,7 @@ By default, examples connect to ClickHouse at `localhost:8123` with the `default 1. Modify the connection strings in the examples 2. Set up a local ClickHouse instance with default settings -3. Use environment variables or configuration files (see [Core_002_ConnectionStringConfiguration.cs](Core/Core_002_ConnectionStringConfiguration.cs)) +3. Use environment variables or configuration files (see [Core_002_ConnectionStringConfiguration.cs](Http/Core/Core_002_ConnectionStringConfiguration.cs)) ### ClickHouse Cloud From 305d785675f69fa236efe26c95da2853a1cec71c Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 11:04:10 +0200 Subject: [PATCH 02/16] Read the examples' server from one place ExampleConfig assembles the connection string from CLICKHOUSE_HOST, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_TCP_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD and CLICKHOUSE_DATABASE, each falling back to what a stock server container exposes on localhost, so the suite runs with nothing set and points somewhere else without editing 46 files. CLICKHOUSE_HTTP_CONNECTION_STRING replaces the whole string for an endpoint the pieces cannot describe. HttpBuilder() covers the examples that change one of the five keys the assembled string already sets; appending covers the rest, since no other key collides. Core_002_ConnectionStringConfiguration and Core_003_DependencyInjection keep their literals, because configuration is what they teach, and Testing_001_Testcontainers starts its own server. Co-Authored-By: Claude Opus 5 (1M context) --- examples/AGENTS.md | 16 ++++ examples/ExampleConfig.cs | 84 +++++++++++++++++++ .../Advanced/Advanced_001_QueryIdUsage.cs | 2 +- .../Advanced_003_LongRunningQueries.cs | 4 +- .../Advanced/Advanced_004_CustomSettings.cs | 6 +- .../Advanced/Advanced_005_QueryStatistics.cs | 2 +- examples/Http/Advanced/Advanced_006_Roles.cs | 4 +- .../Advanced/Advanced_007_CustomHeaders.cs | 2 +- .../Advanced_008_QueryCancellation.cs | 4 +- .../Advanced/Advanced_009_ReadOnlyUsers.cs | 8 +- .../Advanced_010_RetriesAndDeduplication.cs | 2 +- .../Http/Advanced/Advanced_011_Compression.cs | 4 +- .../Advanced_012_ParameterTypeResolver.cs | 10 +-- .../Advanced_013_ParameterFormatter.cs | 8 +- .../Advanced_014_ReadValueConverter.cs | 8 +- .../Http/AspNet/AspNet_001_HealthChecks.cs | 2 +- examples/Http/Core/Core_001_BasicUsage.cs | 4 +- .../DataTypes/DataTypes_001_SimpleTypes.cs | 2 +- .../DataTypes_002_DateTimeHandling.cs | 4 +- .../DataTypes/DataTypes_003_ComplexTypes.cs | 2 +- .../DataTypes/DataTypes_004_StringHandling.cs | 14 ++-- .../Http/DataTypes/DataTypes_005_JsonType.cs | 12 +-- .../Http/DataTypes/DataTypes_006_Geometry.cs | 2 +- .../Vector_001_QBitSimilaritySearch.cs | 2 +- .../Insert/Insert_001_SimpleDataInsert.cs | 4 +- examples/Http/Insert/Insert_002_BulkInsert.cs | 2 +- .../Http/Insert/Insert_003_AsyncInsert.cs | 4 +- .../Http/Insert/Insert_004_RawStreamInsert.cs | 2 +- .../Insert/Insert_005_InsertFromSelect.cs | 2 +- .../Insert/Insert_006_EphemeralColumns.cs | 2 +- ...nsert_007_UpsertsWithReplacingMergeTree.cs | 2 +- .../Insert/Insert_008_SchemaOptimization.cs | 2 +- examples/Http/Insert/Insert_009_PocoInsert.cs | 2 +- examples/Http/ORM/ORM_001_Dapper.cs | 2 +- examples/Http/ORM/ORM_002_Linq2Db.cs | 2 +- .../Http/Select/Select_001_BasicSelect.cs | 2 +- .../Http/Select/Select_002_SelectMetadata.cs | 2 +- .../Select_003_SelectWithParameterBinding.cs | 2 +- .../Http/Select/Select_004_ExportToFile.cs | 2 +- .../Select/Select_005_CompressedRawExport.cs | 2 +- examples/Http/Select/Select_006_PocoSelect.cs | 2 +- .../Select/Select_007_ResponseCompression.cs | 6 +- .../Tables_001_CreateTableSingleNode.cs | 2 +- .../Tables/Tables_002_CreateTableCluster.cs | 2 +- ...roubleshooting_001_LoggingConfiguration.cs | 2 +- .../Troubleshooting_002_NetworkTracing.cs | 2 +- ...roubleshooting_003_OpenTelemetryTracing.cs | 2 +- 47 files changed, 183 insertions(+), 79 deletions(-) create mode 100644 examples/ExampleConfig.cs diff --git a/examples/AGENTS.md b/examples/AGENTS.md index 3ca3c2313..b800d8e08 100644 --- a/examples/AGENTS.md +++ b/examples/AGENTS.md @@ -33,6 +33,22 @@ Five steps. Skip any one of them and the example does not run. Then run it (`dotnet run -- --filter `) and read the output. An example whose output does not teach the topic is not finished. +## Never hard-code a connection string + +Take the server from `ExampleConfig`, which resolves environment variables over localhost defaults: + +- `ExampleConfig.CreateHttpClient()` / `CreateHttpConnection()` for the common case. +- `ExampleConfig.HttpConnectionString` where a constructor takes the string itself. +- `$"{ExampleConfig.HttpConnectionString};SomeKey=value"` to add a key the assembled string does not + already set. It sets `Host`, `Port`, `Username`, `Password` and `Database`, so appending any of + those would produce a duplicate. +- `ExampleConfig.HttpBuilder()` to *change* one of those five. It returns a fresh builder each call. + +Three examples are exempt, because configuration is their subject or they start their own server: +`Core_002_ConnectionStringConfiguration`, `Core_003_DependencyInjection`, +`Testing_001_Testcontainers`. A literal connection string inside a comment, shown to teach the +reader what one looks like, is also fine. + ## Examples deliberately left out of `RunAllExamples` Three need infrastructure the CI server does not have, so they are registered nowhere and run only diff --git a/examples/ExampleConfig.cs b/examples/ExampleConfig.cs new file mode 100644 index 000000000..321a9bca9 --- /dev/null +++ b/examples/ExampleConfig.cs @@ -0,0 +1,84 @@ +using ClickHouse.Driver.ADO; +using ClickHouse.Driver.Utility; + +namespace ClickHouse.Driver.Examples; + +/// +/// The one place the examples get their server from, so that pointing the whole suite at a different +/// ClickHouse is a matter of environment variables rather than editing every file. +/// +/// +/// +/// Every value falls back to what a stock clickhouse/clickhouse-server container exposes on +/// localhost, so the examples run with nothing set. Override any of: +/// +/// +/// CLICKHOUSE_HOSTdefault localhost +/// CLICKHOUSE_HTTP_PORTdefault 8123 +/// CLICKHOUSE_TCP_PORTdefault 9000, the native protocol port +/// CLICKHOUSE_USERdefault default +/// CLICKHOUSE_PASSWORDdefault empty +/// CLICKHOUSE_DATABASEdefault default +/// +/// +/// CLICKHOUSE_HTTP_CONNECTION_STRING replaces the whole assembled string, for a server the +/// pieces above cannot describe — TLS, a cloud endpoint, an extra setting. +/// +/// +/// Three examples deliberately do not use this: Core_002_ConnectionStringConfiguration and +/// Core_003_DependencyInjection, whose subject is configuration itself, and +/// Testing_001_Testcontainers, which starts its own server. +/// +/// +public static class ExampleConfig +{ + /// The server host name or address. + public static string Host { get; } = Env("CLICKHOUSE_HOST") ?? "localhost"; + + /// The HTTP interface port. + public static ushort HttpPort { get; } = ushort.Parse(Env("CLICKHOUSE_HTTP_PORT") ?? "8123"); + + /// The native protocol port. Not interchangeable with . + public static ushort TcpPort { get; } = ushort.Parse(Env("CLICKHOUSE_TCP_PORT") ?? "9000"); + + /// The user to authenticate as. + public static string Username { get; } = Env("CLICKHOUSE_USER") ?? "default"; + + /// The password, empty for a server with no password set. + public static string Password { get; } = Env("CLICKHOUSE_PASSWORD") ?? string.Empty; + + /// The default database for queries. + public static string Database { get; } = Env("CLICKHOUSE_DATABASE") ?? "default"; + + /// The connection string for the HTTP transport. + public static string HttpConnectionString { get; } = + Env("CLICKHOUSE_HTTP_CONNECTION_STRING") ?? HttpBuilder().ConnectionString; + + /// + /// A builder pre-filled with the configured endpoint and credentials, for an example that has to + /// change one key. Each call returns a fresh builder. + /// + /// A builder describing the configured HTTP endpoint. + public static ClickHouseConnectionStringBuilder HttpBuilder() => new() + { + Host = Host, + Port = HttpPort, + Username = Username, + Password = Password, + Database = Database, + }; + + /// Creates a client against the configured server. The caller disposes it. + /// A client for the configured HTTP endpoint. + public static ClickHouseClient CreateHttpClient() => new(HttpConnectionString); + + /// Creates an ADO.NET connection against the configured server. The caller disposes it. + /// A connection for the configured HTTP endpoint. + public static ClickHouseConnection CreateHttpConnection() => new(HttpConnectionString); + + private static string Env(string name) + { + var value = Environment.GetEnvironmentVariable(name); + return string.IsNullOrWhiteSpace(value) ? null : value; + } +} diff --git a/examples/Http/Advanced/Advanced_001_QueryIdUsage.cs b/examples/Http/Advanced/Advanced_001_QueryIdUsage.cs index 3fc4b7ce6..4d4cf7306 100644 --- a/examples/Http/Advanced/Advanced_001_QueryIdUsage.cs +++ b/examples/Http/Advanced/Advanced_001_QueryIdUsage.cs @@ -16,7 +16,7 @@ public static class QueryIdUsage { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("Query ID Usage Examples\n"); diff --git a/examples/Http/Advanced/Advanced_003_LongRunningQueries.cs b/examples/Http/Advanced/Advanced_003_LongRunningQueries.cs index 8b9df6198..2858968e9 100644 --- a/examples/Http/Advanced/Advanced_003_LongRunningQueries.cs +++ b/examples/Http/Advanced/Advanced_003_LongRunningQueries.cs @@ -33,7 +33,7 @@ public static async Task Run() private static async Task Example1_ProgressHeaders() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine(" Configuring query with progress headers..."); Console.WriteLine(" This approach keeps the HTTP connection alive by sending periodic progress updates."); @@ -82,7 +82,7 @@ LIMIT 100 private static async Task Example2_FireAndForget() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine(" Fire-and-forget pattern for very long queries..."); Console.WriteLine(); diff --git a/examples/Http/Advanced/Advanced_004_CustomSettings.cs b/examples/Http/Advanced/Advanced_004_CustomSettings.cs index dc82e3abd..f4681e6ed 100644 --- a/examples/Http/Advanced/Advanced_004_CustomSettings.cs +++ b/examples/Http/Advanced/Advanced_004_CustomSettings.cs @@ -39,7 +39,7 @@ public static async Task Run() private static async Task Example1_ClientLevelSettings() { // Settings applied at the client level affect all queries - var settings = new ClickHouseClientSettings("Host=localhost"); + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString); // Add custom ClickHouse settings settings.CustomSettings.Add("max_threads", 4); @@ -71,7 +71,7 @@ ORDER BY name private static async Task Example2_QueryLevelSettings() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine(" Applying settings to a specific query:"); @@ -106,7 +106,7 @@ ORDER BY name private static async Task Example3_ExecutionTimeLimits() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine(" Setting max_execution_time to limit query duration:"); diff --git a/examples/Http/Advanced/Advanced_005_QueryStatistics.cs b/examples/Http/Advanced/Advanced_005_QueryStatistics.cs index 953895256..4dda3e39a 100644 --- a/examples/Http/Advanced/Advanced_005_QueryStatistics.cs +++ b/examples/Http/Advanced/Advanced_005_QueryStatistics.cs @@ -21,7 +21,7 @@ public static class QueryStatistics { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); + using var connection = ExampleConfig.CreateHttpConnection(); await connection.OpenAsync(); Console.WriteLine("Query Statistics Examples\n"); diff --git a/examples/Http/Advanced/Advanced_006_Roles.cs b/examples/Http/Advanced/Advanced_006_Roles.cs index 3e60ee9e4..05d1d898d 100644 --- a/examples/Http/Advanced/Advanced_006_Roles.cs +++ b/examples/Http/Advanced/Advanced_006_Roles.cs @@ -26,7 +26,7 @@ public static async Task Run() Console.WriteLine("This example demonstrates role-based access control.\n"); // Setup: Create tables, roles, and user using default connection - using var defaultClient = new ClickHouseConnection("Host=localhost"); + using var defaultClient = ExampleConfig.CreateHttpConnection(); await defaultClient.OpenAsync(); await CreateOrReplaceUser(defaultClient, Username, Password); @@ -40,7 +40,7 @@ public static async Task Run() try { // Create a client using a role that only has permission to query table1 - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { Username = Username, Password = Password, diff --git a/examples/Http/Advanced/Advanced_007_CustomHeaders.cs b/examples/Http/Advanced/Advanced_007_CustomHeaders.cs index 3d90b038c..e535e5684 100644 --- a/examples/Http/Advanced/Advanced_007_CustomHeaders.cs +++ b/examples/Http/Advanced/Advanced_007_CustomHeaders.cs @@ -17,7 +17,7 @@ public static async Task Run() // Add custom headers for proxy authentication // Useful when connecting through a proxy that requires specific headers - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { CustomHeaders = new Dictionary { diff --git a/examples/Http/Advanced/Advanced_008_QueryCancellation.cs b/examples/Http/Advanced/Advanced_008_QueryCancellation.cs index 1a7c87b2f..4b21ba872 100644 --- a/examples/Http/Advanced/Advanced_008_QueryCancellation.cs +++ b/examples/Http/Advanced/Advanced_008_QueryCancellation.cs @@ -25,7 +25,7 @@ private static async Task CancelWithTimeout() { Console.WriteLine("1. Cancel query after timeout:"); - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); // Create a cancellation token that will cancel after 1 second using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1)); @@ -60,7 +60,7 @@ private static async Task CancelManually() { Console.WriteLine("\n2. Cancel query manually from another task:"); - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); using var cts = new CancellationTokenSource(); Console.WriteLine(" Starting a 3-second query..."); diff --git a/examples/Http/Advanced/Advanced_009_ReadOnlyUsers.cs b/examples/Http/Advanced/Advanced_009_ReadOnlyUsers.cs index 57703b49f..553f21e34 100644 --- a/examples/Http/Advanced/Advanced_009_ReadOnlyUsers.cs +++ b/examples/Http/Advanced/Advanced_009_ReadOnlyUsers.cs @@ -21,7 +21,7 @@ public static async Task Run() Console.WriteLine("This example demonstrates the limitations of READONLY = 1 users.\n"); // Setup using the default (non-read-only) user - using var defaultClient = new ClickHouseClient("Host=localhost"); + using var defaultClient = ExampleConfig.CreateHttpClient(); // Create a unique read-only user for this example var guid = Guid.NewGuid().ToString("N"); @@ -29,7 +29,11 @@ public static async Task Run() var readOnlyPassword = $"{guid}_pwd"; // The default JsonWriteMode sets a parameter which cannot be used by readonly users, so it must be changed - string readOnlyConnectionString = $"Host=localhost;Username={readOnlyUsername};Password={readOnlyPassword};JsonWriteMode=None"; + var readOnlyBuilder = ExampleConfig.HttpBuilder(); + readOnlyBuilder.Username = readOnlyUsername; + readOnlyBuilder.Password = readOnlyPassword; + readOnlyBuilder["JsonWriteMode"] = "None"; + string readOnlyConnectionString = readOnlyBuilder.ConnectionString; await SetupReadOnlyUser(defaultClient, readOnlyUsername, readOnlyPassword); await SetupTestTable(defaultClient); diff --git a/examples/Http/Advanced/Advanced_010_RetriesAndDeduplication.cs b/examples/Http/Advanced/Advanced_010_RetriesAndDeduplication.cs index 5c48a7222..f4a46010b 100644 --- a/examples/Http/Advanced/Advanced_010_RetriesAndDeduplication.cs +++ b/examples/Http/Advanced/Advanced_010_RetriesAndDeduplication.cs @@ -23,7 +23,7 @@ public static class RetriesAndDeduplication public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); // Create a ReplacingMergeTree table for deduplication await SetupReplacingMergeTreeTable(client); diff --git a/examples/Http/Advanced/Advanced_011_Compression.cs b/examples/Http/Advanced/Advanced_011_Compression.cs index a2be9f4ac..4e55a713d 100644 --- a/examples/Http/Advanced/Advanced_011_Compression.cs +++ b/examples/Http/Advanced/Advanced_011_Compression.cs @@ -61,7 +61,7 @@ public static async Task Run() // Default: compression enabled Console.WriteLine("1. Default behavior (compression enabled):"); - using (var client = new ClickHouseClient("Host=localhost")) + using (var client = ExampleConfig.CreateHttpClient()) { // The driver will: // - Request compressed responses via enable_http_compression=true @@ -74,7 +74,7 @@ public static async Task Run() // Compression disabled Console.WriteLine("2. Compression disabled:"); - using (var client = new ClickHouseClient("Host=localhost;Compression=false")) + using (var client = new ClickHouseClient($"{ExampleConfig.HttpConnectionString};Compression=false")) { // The driver will: // - Set enable_http_compression=false and advertise no codec (uncompressed responses) diff --git a/examples/Http/Advanced/Advanced_012_ParameterTypeResolver.cs b/examples/Http/Advanced/Advanced_012_ParameterTypeResolver.cs index 405c22f28..8c1b2b31f 100644 --- a/examples/Http/Advanced/Advanced_012_ParameterTypeResolver.cs +++ b/examples/Http/Advanced/Advanced_012_ParameterTypeResolver.cs @@ -39,7 +39,7 @@ private static async Task DictionaryResolverExample() { Console.WriteLine("1. DictionaryParameterTypeResolver - DateTime mapped to DateTime64(3):"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterTypeResolver = new DictionaryParameterTypeResolver(new Dictionary { @@ -78,7 +78,7 @@ private static async Task CustomResolverExample() { Console.WriteLine("\n2. Custom IParameterTypeResolver - value-aware resolution:"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterTypeResolver = new SmartDecimalResolver(), }; @@ -106,7 +106,7 @@ private static async Task ExplicitTypeOverrideExample() { Console.WriteLine("\n3. Explicit ClickHouseType overrides the resolver:"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterTypeResolver = new DictionaryParameterTypeResolver(new Dictionary { @@ -141,7 +141,7 @@ private static async Task AdoNetExample() { Console.WriteLine("\n4. Works with ClickHouseConnection (ADO.NET):"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterTypeResolver = new DictionaryParameterTypeResolver(new Dictionary { @@ -169,7 +169,7 @@ private static async Task PerQueryResolverExample() Console.WriteLine("\n5. Per-query resolver via QueryOptions:"); // Client-level: int → Int64 - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterTypeResolver = new DictionaryParameterTypeResolver(new Dictionary { diff --git a/examples/Http/Advanced/Advanced_013_ParameterFormatter.cs b/examples/Http/Advanced/Advanced_013_ParameterFormatter.cs index 405d52e7d..e6c2fe5df 100644 --- a/examples/Http/Advanced/Advanced_013_ParameterFormatter.cs +++ b/examples/Http/Advanced/Advanced_013_ParameterFormatter.cs @@ -40,7 +40,7 @@ private static async Task DictionaryFormatterExample() { Console.WriteLine("1. DictionaryParameterFormatter - custom DateTime serialization:"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterFormatter = new DictionaryParameterFormatter(new Dictionary> { @@ -76,7 +76,7 @@ private static async Task CompositeElementExample() { Console.WriteLine("\n2. Formatter runs on array elements too:"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterFormatter = new DictionaryParameterFormatter(new Dictionary> { @@ -109,7 +109,7 @@ private static async Task CustomFormatterExample() { Console.WriteLine("\n3. Custom IParameterFormatter:"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterFormatter = new DoublingDecimalFormatter(), }; @@ -134,7 +134,7 @@ private static async Task PerQueryFormatterExample() { Console.WriteLine("\n4. Per-query formatter via QueryOptions:"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ParameterFormatter = new DictionaryParameterFormatter(new Dictionary> { diff --git a/examples/Http/Advanced/Advanced_014_ReadValueConverter.cs b/examples/Http/Advanced/Advanced_014_ReadValueConverter.cs index 923eeb975..a14135d25 100644 --- a/examples/Http/Advanced/Advanced_014_ReadValueConverter.cs +++ b/examples/Http/Advanced/Advanced_014_ReadValueConverter.cs @@ -39,7 +39,7 @@ private static async Task DictionaryConverterExample() .For(dt => DateTime.SpecifyKind(dt, DateTimeKind.Utc)) .For(s => s.Trim()); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ReadValueConverter = converter, }; @@ -66,7 +66,7 @@ private static async Task CustomConverterExample() { Console.WriteLine("\n2. Custom IReadValueConverter - dispatch on ClickHouse-side type:"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ReadValueConverter = new UtcOnlyForNoTzDateTimeConverter(), }; @@ -91,7 +91,7 @@ private static async Task PerQueryConverterExample() { Console.WriteLine("\n3. Per-query converter override via QueryOptions:"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ReadValueConverter = new DictionaryReadValueConverter() .For(dt => DateTime.SpecifyKind(dt, DateTimeKind.Utc)), @@ -132,7 +132,7 @@ private static async Task AdoNetExample() { Console.WriteLine("\n4. Works with ClickHouseConnection (ADO.NET):"); - var settings = new ClickHouseClientSettings("Host=localhost") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { ReadValueConverter = new DictionaryReadValueConverter() .For(dt => DateTime.SpecifyKind(dt, DateTimeKind.Utc)), diff --git a/examples/Http/AspNet/AspNet_001_HealthChecks.cs b/examples/Http/AspNet/AspNet_001_HealthChecks.cs index 309e75a99..3a1765131 100644 --- a/examples/Http/AspNet/AspNet_001_HealthChecks.cs +++ b/examples/Http/AspNet/AspNet_001_HealthChecks.cs @@ -23,7 +23,7 @@ public static async Task Run() { Console.WriteLine("ClickHouse ASP.NET Health Checks Example\n"); - var connectionString = "Host=localhost;Port=8123;Protocol=http;Username=default;Password=;Database=default"; + var connectionString = ExampleConfig.HttpConnectionString; // ======================================================================= // OPTION 1: Using ClickHouseClient (recommended for direct operations) diff --git a/examples/Http/Core/Core_001_BasicUsage.cs b/examples/Http/Core/Core_001_BasicUsage.cs index a6f9f68e4..6e19b73d9 100644 --- a/examples/Http/Core/Core_001_BasicUsage.cs +++ b/examples/Http/Core/Core_001_BasicUsage.cs @@ -32,7 +32,7 @@ public static async Task Run() private static async Task UsingClickHouseClient() { // ClickHouseClient is thread-safe and designed for singleton usage - var settings = new ClickHouseClientSettings("Host=localhost"); + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString); using var client = new ClickHouseClient(settings); var version = await client.ExecuteScalarAsync("SELECT version()"); @@ -80,7 +80,7 @@ ORDER BY (id) private static async Task UsingClickHouseConnection() { // ClickHouseConnection provides ADO.NET compatibility for Dapper, EF Core, etc. - var settings = new ClickHouseClientSettings("Host=localhost"); + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString); using var connection = new ClickHouseConnection(settings); await connection.OpenAsync(); diff --git a/examples/Http/DataTypes/DataTypes_001_SimpleTypes.cs b/examples/Http/DataTypes/DataTypes_001_SimpleTypes.cs index fbbb1e60e..22026c5b4 100644 --- a/examples/Http/DataTypes/DataTypes_001_SimpleTypes.cs +++ b/examples/Http/DataTypes/DataTypes_001_SimpleTypes.cs @@ -12,7 +12,7 @@ public static class SimpleTypes { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("Simple Data Types Examples\n"); diff --git a/examples/Http/DataTypes/DataTypes_002_DateTimeHandling.cs b/examples/Http/DataTypes/DataTypes_002_DateTimeHandling.cs index d04e7a09b..72685b9f0 100644 --- a/examples/Http/DataTypes/DataTypes_002_DateTimeHandling.cs +++ b/examples/Http/DataTypes/DataTypes_002_DateTimeHandling.cs @@ -18,7 +18,7 @@ public static class DateTimeHandling { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); + using var connection = ExampleConfig.CreateHttpConnection(); await connection.OpenAsync(); Console.WriteLine("DateTime Handling Examples\n"); @@ -302,7 +302,7 @@ dt_amsterdam DateTime('Europe/Amsterdam') ENGINE = Memory "); - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); // Unspecified DateTime values are treated as wall-clock time in the column's timezone var columns = new[] { "id", "dt_utc", "dt_amsterdam" }; diff --git a/examples/Http/DataTypes/DataTypes_003_ComplexTypes.cs b/examples/Http/DataTypes/DataTypes_003_ComplexTypes.cs index 36b5df803..20c2c0b2c 100644 --- a/examples/Http/DataTypes/DataTypes_003_ComplexTypes.cs +++ b/examples/Http/DataTypes/DataTypes_003_ComplexTypes.cs @@ -17,7 +17,7 @@ public static class ComplexTypes { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("Complex Data Types Examples\n"); diff --git a/examples/Http/DataTypes/DataTypes_004_StringHandling.cs b/examples/Http/DataTypes/DataTypes_004_StringHandling.cs index da27c59ee..b57a68a19 100644 --- a/examples/Http/DataTypes/DataTypes_004_StringHandling.cs +++ b/examples/Http/DataTypes/DataTypes_004_StringHandling.cs @@ -16,7 +16,7 @@ public static class StringHandling { public static async Task Run() { - using var connection = new ClickHouseConnection("Host=localhost"); + using var connection = ExampleConfig.CreateHttpConnection(); await connection.OpenAsync(); Console.WriteLine("String and FixedString Handling Examples\n"); @@ -109,7 +109,7 @@ fixed_str FixedString(5) ENGINE = Memory "); - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); // Insert strings of different lengths var columns = new[] { "fixed_str" }; @@ -122,7 +122,7 @@ fixed_str FixedString(5) await client.InsertBinaryAsync(tableName, columns, data); // Read back and show the actual bytes - var cb = new ClickHouseConnectionStringBuilder("Host=localhost") + var cb = new ClickHouseConnectionStringBuilder(ExampleConfig.HttpConnectionString) { ReadStringsAsByteArrays = true }; @@ -157,7 +157,7 @@ data String ENGINE = Memory "); - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); // Write binary data that is NOT valid UTF-8 var binaryData = new byte[] { 0xFF, 0xFE, 0x00, 0x01, 0x02 }; @@ -174,7 +174,7 @@ data String Console.WriteLine($" Inserted {data.Count} rows with binary data"); // Read back as byte[] to preserve the binary data - var cb = new ClickHouseConnectionStringBuilder("Host=localhost") + var cb = new ClickHouseConnectionStringBuilder(ExampleConfig.HttpConnectionString) { ReadStringsAsByteArrays = true }; @@ -202,7 +202,7 @@ private static async Task Example4_ReadStringsAsByteArrays() { // Default behavior: returns string Console.WriteLine(" Default (ReadStringsAsByteArrays=false):"); - using (var connection = new ClickHouseConnection("Host=localhost")) + using (var connection = ExampleConfig.CreateHttpConnection()) { var result = await connection.ExecuteScalarAsync("SELECT 'Hello'"); Console.WriteLine($" Type: {result.GetType().Name}, Value: \"{result}\""); @@ -210,7 +210,7 @@ private static async Task Example4_ReadStringsAsByteArrays() // With setting enabled: returns byte[] Console.WriteLine(" With ReadStringsAsByteArrays=true:"); - using (var connection = new ClickHouseConnection("Host=localhost;ReadStringsAsByteArrays=true")) + using (var connection = new ClickHouseConnection($"{ExampleConfig.HttpConnectionString};ReadStringsAsByteArrays=true")) { var result = await connection.ExecuteScalarAsync("SELECT 'Hello'"); var bytes = (byte[])result; diff --git a/examples/Http/DataTypes/DataTypes_005_JsonType.cs b/examples/Http/DataTypes/DataTypes_005_JsonType.cs index 4b5e804b3..d7aa6b13b 100644 --- a/examples/Http/DataTypes/DataTypes_005_JsonType.cs +++ b/examples/Http/DataTypes/DataTypes_005_JsonType.cs @@ -48,7 +48,7 @@ private static async Task Example1_InsertStringMode() Console.WriteLine("-".PadRight(50, '-')); // JsonWriteMode.String is the default, so we can omit it - using var client = new ClickHouseClient("Host=localhost;set_allow_experimental_json_type=1"); + using var client = new ClickHouseClient($"{ExampleConfig.HttpConnectionString};set_allow_experimental_json_type=1"); var tableName = "example_insert_string_mode"; await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); @@ -115,7 +115,7 @@ private static async Task Example2_InsertBinaryMode() Console.WriteLine("-".PadRight(50, '-')); // Must explicitly set Binary mode - using var client = new ClickHouseClient("Host=localhost;JsonWriteMode=Binary;set_allow_experimental_json_type=1"); + using var client = new ClickHouseClient($"{ExampleConfig.HttpConnectionString};JsonWriteMode=Binary;set_allow_experimental_json_type=1"); // Register POCO types before using them client.RegisterJsonSerializationType(); @@ -167,7 +167,7 @@ private static async Task Example3_ReadBinaryMode() Console.WriteLine("-".PadRight(50, '-')); // JsonReadMode.Binary is the default, so we can omit it - using var connection = new ClickHouseConnection("Host=localhost"); + using var connection = ExampleConfig.CreateHttpConnection(); await connection.OpenAsync(); connection.CustomSettings["allow_experimental_json_type"] = 1; @@ -199,7 +199,7 @@ private static async Task Example4_ReadStringMode() Console.WriteLine("\n4. READ WITH STRING MODE"); Console.WriteLine("-".PadRight(50, '-')); - using var connection = new ClickHouseConnection("Host=localhost;JsonReadMode=String"); + using var connection = new ClickHouseConnection($"{ExampleConfig.HttpConnectionString};JsonReadMode=String"); await connection.OpenAsync(); connection.CustomSettings["allow_experimental_json_type"] = 1; @@ -245,7 +245,7 @@ private static async Task Example5_QueryingJsonPaths() Console.WriteLine("\n5. QUERYING JSON PATHS"); Console.WriteLine("-".PadRight(50, '-')); - using var connection = new ClickHouseConnection("Host=localhost;"); + using var connection = new ClickHouseConnection(ExampleConfig.HttpConnectionString); await connection.OpenAsync(); connection.CustomSettings["allow_experimental_json_type"] = 1; @@ -269,7 +269,7 @@ tags Array(String), Console.WriteLine(" - metadata.created: DateTime"); // Insert data - hints ensure proper type handling - using var client = new ClickHouseClient("Host=localhost;set_allow_experimental_json_type=1;set_date_time_input_format=best_effort"); + using var client = new ClickHouseClient($"{ExampleConfig.HttpConnectionString};set_allow_experimental_json_type=1;set_date_time_input_format=best_effort"); var columns = new[] { "id", "data" }; var rows = new[] diff --git a/examples/Http/DataTypes/DataTypes_006_Geometry.cs b/examples/Http/DataTypes/DataTypes_006_Geometry.cs index e48c449e4..bcb6890df 100644 --- a/examples/Http/DataTypes/DataTypes_006_Geometry.cs +++ b/examples/Http/DataTypes/DataTypes_006_Geometry.cs @@ -16,7 +16,7 @@ public static class GeometryTypes { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("Geometry Types Examples\n"); diff --git a/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs b/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs index fe7073562..621d629fe 100644 --- a/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs +++ b/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs @@ -11,7 +11,7 @@ public static class QBitSimilaritySearch { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("=== QBit Similarity Search with Different Precision Levels ===\n"); diff --git a/examples/Http/Insert/Insert_001_SimpleDataInsert.cs b/examples/Http/Insert/Insert_001_SimpleDataInsert.cs index 351a75a77..a6e67d8cb 100644 --- a/examples/Http/Insert/Insert_001_SimpleDataInsert.cs +++ b/examples/Http/Insert/Insert_001_SimpleDataInsert.cs @@ -16,7 +16,7 @@ public static class SimpleDataInsert public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); await SetupTable(client); @@ -109,7 +109,7 @@ private static async Task InsertUsingAdoCommand() { Console.WriteLine("3. ADO.NET Command pattern:"); - using var connection = new ClickHouseConnection("Host=localhost"); + using var connection = ExampleConfig.CreateHttpConnection(); await connection.OpenAsync(); using var command = connection.CreateCommand(); diff --git a/examples/Http/Insert/Insert_002_BulkInsert.cs b/examples/Http/Insert/Insert_002_BulkInsert.cs index 6c264fe2b..fdd008c5a 100644 --- a/examples/Http/Insert/Insert_002_BulkInsert.cs +++ b/examples/Http/Insert/Insert_002_BulkInsert.cs @@ -11,7 +11,7 @@ public static class BulkInsert { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); var tableName = "example_bulk_insert"; diff --git a/examples/Http/Insert/Insert_003_AsyncInsert.cs b/examples/Http/Insert/Insert_003_AsyncInsert.cs index bfc393ee4..4042fd54e 100644 --- a/examples/Http/Insert/Insert_003_AsyncInsert.cs +++ b/examples/Http/Insert/Insert_003_AsyncInsert.cs @@ -74,7 +74,7 @@ private static async Task Example1_AsyncInsertWithWait() // // "Host=localhost;set_async_insert=1;set_wait_for_async_insert=1;set_async_insert_busy_timeout_ms=1000" // - var settings = new ClickHouseClientSettings("Host=localhost"); + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString); settings.CustomSettings["async_insert"] = 1; settings.CustomSettings["wait_for_async_insert"] = 1; settings.CustomSettings["async_insert_max_data_size"] = 1_000_000; @@ -150,7 +150,7 @@ ORDER BY id private static async Task Example2_AsyncInsertWithoutWait() { // Configure async inserts WITHOUT waiting - var settings = new ClickHouseClientSettings("Host=localhost"); + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString); settings.CustomSettings["async_insert"] = 1; settings.CustomSettings["wait_for_async_insert"] = 0; // Fire and forget settings.CustomSettings["async_insert_max_data_size"] = 1_000_000; diff --git a/examples/Http/Insert/Insert_004_RawStreamInsert.cs b/examples/Http/Insert/Insert_004_RawStreamInsert.cs index 42a8bd3a7..413ad2c5b 100644 --- a/examples/Http/Insert/Insert_004_RawStreamInsert.cs +++ b/examples/Http/Insert/Insert_004_RawStreamInsert.cs @@ -13,7 +13,7 @@ public static class RawStreamInsert { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); await InsertFromFile(client); await InsertFromMemory(client); diff --git a/examples/Http/Insert/Insert_005_InsertFromSelect.cs b/examples/Http/Insert/Insert_005_InsertFromSelect.cs index 1ed963e76..90fb97419 100644 --- a/examples/Http/Insert/Insert_005_InsertFromSelect.cs +++ b/examples/Http/Insert/Insert_005_InsertFromSelect.cs @@ -39,7 +39,7 @@ public static class InsertFromSelect { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("INSERT FROM SELECT Examples\n"); diff --git a/examples/Http/Insert/Insert_006_EphemeralColumns.cs b/examples/Http/Insert/Insert_006_EphemeralColumns.cs index a9ab7ed1e..dde1fe08a 100644 --- a/examples/Http/Insert/Insert_006_EphemeralColumns.cs +++ b/examples/Http/Insert/Insert_006_EphemeralColumns.cs @@ -20,7 +20,7 @@ public static class EphemeralColumns { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("Ephemeral Columns Examples\n"); diff --git a/examples/Http/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs b/examples/Http/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs index 0eb61049c..d0b71fe32 100644 --- a/examples/Http/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs +++ b/examples/Http/Insert/Insert_007_UpsertsWithReplacingMergeTree.cs @@ -24,7 +24,7 @@ public static class UpsertsWithReplacingMergeTree public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); await SetupTable(client); diff --git a/examples/Http/Insert/Insert_008_SchemaOptimization.cs b/examples/Http/Insert/Insert_008_SchemaOptimization.cs index d89156912..dcf1d706b 100644 --- a/examples/Http/Insert/Insert_008_SchemaOptimization.cs +++ b/examples/Http/Insert/Insert_008_SchemaOptimization.cs @@ -9,7 +9,7 @@ public static class SchemaOptimization { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); var tableName = "example_schema_optimization"; diff --git a/examples/Http/Insert/Insert_009_PocoInsert.cs b/examples/Http/Insert/Insert_009_PocoInsert.cs index 026e7c1fb..450ddb8ed 100644 --- a/examples/Http/Insert/Insert_009_PocoInsert.cs +++ b/examples/Http/Insert/Insert_009_PocoInsert.cs @@ -8,7 +8,7 @@ public static class PocoInsert { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); await BasicPocoInsert(client); await AttributeMappingInsert(client); diff --git a/examples/Http/ORM/ORM_001_Dapper.cs b/examples/Http/ORM/ORM_001_Dapper.cs index c8a5d2dc1..8ff4c9381 100644 --- a/examples/Http/ORM/ORM_001_Dapper.cs +++ b/examples/Http/ORM/ORM_001_Dapper.cs @@ -36,7 +36,7 @@ public static async Task Run() { // Create a DataSource - in a real app, this would be a singleton (register in DI) // The DataSource manages HttpClient pooling internally - var dataSource = new ClickHouseDataSource("Host=localhost"); + var dataSource = new ClickHouseDataSource(ExampleConfig.HttpConnectionString); // Create a connection from the DataSource // Connections are lightweight - create them per operation diff --git a/examples/Http/ORM/ORM_002_Linq2Db.cs b/examples/Http/ORM/ORM_002_Linq2Db.cs index 6efb933fa..eae341d66 100644 --- a/examples/Http/ORM/ORM_002_Linq2Db.cs +++ b/examples/Http/ORM/ORM_002_Linq2Db.cs @@ -16,7 +16,7 @@ public static class Linq2DbExample public static async Task Run() { // Connect using linq2db's DataConnection with ClickHouseDriver provider - var connectionString = "Host=localhost"; + var connectionString = ExampleConfig.HttpConnectionString; var options = new DataOptions().UseClickHouse(connectionString, ClickHouseProvider.ClickHouseDriver); await using var db = new DataConnection(options); diff --git a/examples/Http/Select/Select_001_BasicSelect.cs b/examples/Http/Select/Select_001_BasicSelect.cs index 522df48fe..60c771b33 100644 --- a/examples/Http/Select/Select_001_BasicSelect.cs +++ b/examples/Http/Select/Select_001_BasicSelect.cs @@ -10,7 +10,7 @@ public static class BasicSelect { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); var tableName = "example_select_basic"; diff --git a/examples/Http/Select/Select_002_SelectMetadata.cs b/examples/Http/Select/Select_002_SelectMetadata.cs index fcd923807..a3c590e77 100644 --- a/examples/Http/Select/Select_002_SelectMetadata.cs +++ b/examples/Http/Select/Select_002_SelectMetadata.cs @@ -9,7 +9,7 @@ public static class SelectMetadata { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); var tableName = "example_formats"; diff --git a/examples/Http/Select/Select_003_SelectWithParameterBinding.cs b/examples/Http/Select/Select_003_SelectWithParameterBinding.cs index c8b3d82ee..30cc11ddf 100644 --- a/examples/Http/Select/Select_003_SelectWithParameterBinding.cs +++ b/examples/Http/Select/Select_003_SelectWithParameterBinding.cs @@ -11,7 +11,7 @@ public static class SelectWithParameterBinding { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); var tableName = "example_parameter_binding"; diff --git a/examples/Http/Select/Select_004_ExportToFile.cs b/examples/Http/Select/Select_004_ExportToFile.cs index 8a926e4e3..fdaa5be55 100644 --- a/examples/Http/Select/Select_004_ExportToFile.cs +++ b/examples/Http/Select/Select_004_ExportToFile.cs @@ -11,7 +11,7 @@ public static class ExportToFile { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); var tableName = "example_export"; diff --git a/examples/Http/Select/Select_005_CompressedRawExport.cs b/examples/Http/Select/Select_005_CompressedRawExport.cs index ef865f182..d378fd844 100644 --- a/examples/Http/Select/Select_005_CompressedRawExport.cs +++ b/examples/Http/Select/Select_005_CompressedRawExport.cs @@ -17,7 +17,7 @@ public static class CompressedRawExport { public static async Task Run() { - var connectionString = "Host=localhost"; + var connectionString = ExampleConfig.HttpConnectionString; var tableName = "example_compressed_export"; // Nothing to configure for this to work: the driver decompresses responses itself, so a raw diff --git a/examples/Http/Select/Select_006_PocoSelect.cs b/examples/Http/Select/Select_006_PocoSelect.cs index 6225eb74e..708106ced 100644 --- a/examples/Http/Select/Select_006_PocoSelect.cs +++ b/examples/Http/Select/Select_006_PocoSelect.cs @@ -9,7 +9,7 @@ public static class PocoSelect { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); await BasicQueryAsync(client); await AttributeMappingQuery(client); diff --git a/examples/Http/Select/Select_007_ResponseCompression.cs b/examples/Http/Select/Select_007_ResponseCompression.cs index 9cbefbfab..52b89c4e6 100644 --- a/examples/Http/Select/Select_007_ResponseCompression.cs +++ b/examples/Http/Select/Select_007_ResponseCompression.cs @@ -20,7 +20,7 @@ public static async Task Run() var tableName = "example_response_compression"; // A default client. No custom HttpClient and no compression settings needed. - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); await client.ExecuteNonQueryAsync($@" CREATE TABLE IF NOT EXISTS {tableName} @@ -83,7 +83,7 @@ ORDER BY (id) // Example 3: overriding the codec client-wide. Brotli is decoded whenever it arrives but is // not advertised by default, so it is only used when a caller names it. Console.WriteLine("3. Choosing brotli client-wide:"); - using (var brotliClient = new ClickHouseClient(new ClickHouseClientSettings("Host=localhost") + using (var brotliClient = new ClickHouseClient(new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { AcceptEncoding = "br", })) @@ -94,7 +94,7 @@ ORDER BY (id) // The same thing through a connection string, for ORM users (Dapper, EF Core, linq2db) // who never touch ClickHouseClientSettings directly. - using (var csClient = new ClickHouseClient("Host=localhost;AcceptEncoding=br")) + using (var csClient = new ClickHouseClient($"{ExampleConfig.HttpConnectionString};AcceptEncoding=br")) { var count = await csClient.ExecuteScalarAsync($"SELECT count() FROM {tableName}"); Console.WriteLine($" Rows read with AcceptEncoding=br in the connection string: {count}\n"); diff --git a/examples/Http/Tables/Tables_001_CreateTableSingleNode.cs b/examples/Http/Tables/Tables_001_CreateTableSingleNode.cs index 7806a11d6..2a35dfdb6 100644 --- a/examples/Http/Tables/Tables_001_CreateTableSingleNode.cs +++ b/examples/Http/Tables/Tables_001_CreateTableSingleNode.cs @@ -10,7 +10,7 @@ public static class CreateTableSingleNode { public static async Task Run() { - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("Creating tables on a single ClickHouse node\n"); diff --git a/examples/Http/Tables/Tables_002_CreateTableCluster.cs b/examples/Http/Tables/Tables_002_CreateTableCluster.cs index cf52a88d9..37fe8375e 100644 --- a/examples/Http/Tables/Tables_002_CreateTableCluster.cs +++ b/examples/Http/Tables/Tables_002_CreateTableCluster.cs @@ -11,7 +11,7 @@ public static class CreateTableCluster public static async Task Run() { // For cluster operations, connect to any node in the cluster - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); Console.WriteLine("Creating tables on a ClickHouse cluster\n"); diff --git a/examples/Http/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs b/examples/Http/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs index d34b9df9c..b7432246b 100644 --- a/examples/Http/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs +++ b/examples/Http/Troubleshooting/Troubleshooting_001_LoggingConfiguration.cs @@ -24,7 +24,7 @@ public static async Task Run() Console.WriteLine("Creating client with Trace-level logging enabled...\n"); // Create client settings with logger factory - var settings = new ClickHouseClientSettings("Host=localhost;Port=8123;Username=default;Database=default") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { LoggerFactory = loggerFactory, }; diff --git a/examples/Http/Troubleshooting/Troubleshooting_002_NetworkTracing.cs b/examples/Http/Troubleshooting/Troubleshooting_002_NetworkTracing.cs index 2dbe203b9..effbf7e47 100644 --- a/examples/Http/Troubleshooting/Troubleshooting_002_NetworkTracing.cs +++ b/examples/Http/Troubleshooting/Troubleshooting_002_NetworkTracing.cs @@ -38,7 +38,7 @@ public static async Task Run() }); // Step 2: Configure ClickHouse client with EnableDebugMode - var settings = new ClickHouseClientSettings("Host=localhost;Port=8123;Username=default;Database=default") + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { LoggerFactory = loggerFactory, EnableDebugMode = true, // Enable low-level network tracing diff --git a/examples/Http/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs b/examples/Http/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs index f4e480955..64f27afe8 100644 --- a/examples/Http/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs +++ b/examples/Http/Troubleshooting/Troubleshooting_003_OpenTelemetryTracing.cs @@ -39,7 +39,7 @@ public static async Task Run() Console.WriteLine($"Listening to ActivitySource: {ClickHouseDiagnosticsOptions.ActivitySourceName}"); Console.WriteLine("SQL in traces: enabled\n"); - using var client = new ClickHouseClient("Host=localhost"); + using var client = ExampleConfig.CreateHttpClient(); // Query with results - shows read statistics await ExecuteQueryWithResults(client); From 1f41b2917b5848e2bc92ef0c2fb3a71617e3f5c3 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 11:10:16 +0200 Subject: [PATCH 03/16] Make the examples project ready for native-protocol examples ClickHouse.Driver references ClickHouse.Driver.Tcp with PrivateAssets="all", so the native client's types do not reach a consumer of the driver project; the examples reference it directly. CHTCP0001 is suppressed project-wide, and Tcp/README.md explains the opt-in a consumer has to make for themselves. An example's transport comes from its class name. Every example shares one namespace, so a native-protocol example cannot reuse an HTTP example's class name, and the Tcp prefix that keeps them apart also says which endpoint it needs. --http and --tcp select on it, and --list takes it too. ExamplePreflight reaches those endpoints once before anything runs and reports the endpoint, the reason, and the variables that change it. It exits non-zero rather than skipping, because CI runs the suite with no filter and a skip would leave the run green having exercised nothing. Asking for a transport checks that transport, so --tcp reports whether port 9000 answers even with no example written yet. The workflow's server publishes 9000 alongside 8123, and its paths filter covers the Tcp project. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/examples.yml | 4 + examples/AGENTS.md | 5 + examples/ClickHouse.Driver.Examples.csproj | 6 ++ examples/ExampleConfig.cs | 28 +++++- examples/ExamplePreflight.cs | 106 +++++++++++++++++++++ examples/ExampleRunner.cs | 28 +++++- examples/ExampleTransport.cs | 14 +++ examples/Program.cs | 56 ++++++++++- examples/README.md | 37 +++++-- examples/Tcp/README.md | 59 ++++++++++++ 10 files changed, 329 insertions(+), 14 deletions(-) create mode 100644 examples/ExamplePreflight.cs create mode 100644 examples/ExampleTransport.cs create mode 100644 examples/Tcp/README.md diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 10e855a73..a363807b7 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -7,6 +7,7 @@ on: - "examples/**" - "ClickHouse.Driver/**" - "ClickHouse.Driver.Common/**" + - "ClickHouse.Driver.Tcp/**" - ".github/workflows/examples.yml" pull_request: branches: [main] @@ -14,6 +15,7 @@ on: - "examples/**" - "ClickHouse.Driver/**" - "ClickHouse.Driver.Common/**" + - "ClickHouse.Driver.Tcp/**" - ".github/workflows/examples.yml" workflow_dispatch: inputs: @@ -36,6 +38,8 @@ jobs: image: clickhouse/clickhouse-server:latest ports: - 8123:8123 + # The native protocol listens separately; the Tcp examples need it. + - 9000:9000 env: CLICKHOUSE_DB: test CLICKHOUSE_SKIP_USER_SETUP: "1" diff --git a/examples/AGENTS.md b/examples/AGENTS.md index b800d8e08..be3fe7071 100644 --- a/examples/AGENTS.md +++ b/examples/AGENTS.md @@ -23,6 +23,11 @@ Five steps. Skip any one of them and the example does not run. The class name is what `--list` prints and what `--filter` matches, so name it after the topic (`BasicUsage`), not after the file (`Core_001_BasicUsage`). + **A native-protocol example's class name must start with `Tcp`** (`TcpBasicUsage`). Every example + shares one namespace, so it could not reuse an HTTP example's name anyway, and that prefix is how + `ExampleInfo.Transport` knows which endpoint to check before running it — and what `--http` and + `--tcp` select on. + 4. **Add it to `RunAllExamples` in `Program.cs`**, under its category banner, in file-number order. That list is hand-maintained so the run order and the banners stay meaningful. An example missing from it still compiles, still appears in `--list`, and still runs under `--filter` — it just never diff --git a/examples/ClickHouse.Driver.Examples.csproj b/examples/ClickHouse.Driver.Examples.csproj index 7336ed000..0ccfe4344 100644 --- a/examples/ClickHouse.Driver.Examples.csproj +++ b/examples/ClickHouse.Driver.Examples.csproj @@ -5,6 +5,9 @@ net10.0 enable enable + + $(NoWarn);CHTCP0001 @@ -12,6 +15,9 @@ + + diff --git a/examples/ExampleConfig.cs b/examples/ExampleConfig.cs index 321a9bca9..785b786cc 100644 --- a/examples/ExampleConfig.cs +++ b/examples/ExampleConfig.cs @@ -1,4 +1,5 @@ using ClickHouse.Driver.ADO; +using ClickHouse.Driver.Tcp; using ClickHouse.Driver.Utility; namespace ClickHouse.Driver.Examples; @@ -54,6 +55,10 @@ public static class ExampleConfig public static string HttpConnectionString { get; } = Env("CLICKHOUSE_HTTP_CONNECTION_STRING") ?? HttpBuilder().ConnectionString; + /// The connection string for the native protocol. + public static string TcpConnectionString { get; } = + Env("CLICKHOUSE_TCP_CONNECTION_STRING") ?? TcpBuilder().ToString(); + /// /// A builder pre-filled with the configured endpoint and credentials, for an example that has to /// change one key. Each call returns a fresh builder. @@ -68,15 +73,36 @@ public static class ExampleConfig Database = Database, }; + /// + /// A builder pre-filled with the configured endpoint and credentials for the native protocol, for + /// an example that has to change one key. Each call returns a fresh builder. + /// + /// A builder describing the configured native endpoint. + public static ClickHouseTcpConnectionStringBuilder TcpBuilder() => new() + { + Host = Host, + Port = TcpPort, + Username = Username, + Password = Password, + Database = Database, + }; + /// Creates a client against the configured server. The caller disposes it. /// A client for the configured HTTP endpoint. public static ClickHouseClient CreateHttpClient() => new(HttpConnectionString); + /// + /// Creates a native-protocol client against the configured server. The caller disposes it, + /// asynchronously where it can. + /// + /// A client for the configured native endpoint. + public static ClickHouseTcpClient CreateTcpClient() => new(TcpConnectionString); + /// Creates an ADO.NET connection against the configured server. The caller disposes it. /// A connection for the configured HTTP endpoint. public static ClickHouseConnection CreateHttpConnection() => new(HttpConnectionString); - private static string Env(string name) + private static string? Env(string name) { var value = Environment.GetEnvironmentVariable(name); return string.IsNullOrWhiteSpace(value) ? null : value; diff --git a/examples/ExamplePreflight.cs b/examples/ExamplePreflight.cs new file mode 100644 index 000000000..eb51cc7d3 --- /dev/null +++ b/examples/ExamplePreflight.cs @@ -0,0 +1,106 @@ +using ClickHouse.Driver.Tcp; +using ClickHouse.Driver.Utility; + +namespace ClickHouse.Driver.Examples; + +/// +/// Reaches the server once before any example runs, so that an unreachable or misconfigured endpoint +/// is reported as itself rather than as a failure inside whichever example happened to run first. +/// +/// +/// A failure exits non-zero instead of skipping. CI runs the whole suite with no filter, and a skip +/// would leave the run green while nothing had been exercised. +/// +public static class ExamplePreflight +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); + + /// + /// Checks the endpoints the given examples need, and reports what to fix if one is unreachable. + /// + /// The examples about to run. Only their transports are checked. + /// True when every needed endpoint answered. + public static Task CheckAsync(IEnumerable examples) + => CheckAsync(examples.Select(e => e.Transport).Distinct().ToArray()); + + /// + /// Checks the named endpoints, and reports what to fix if one is unreachable. + /// + /// The transports to check. Duplicates are checked once. + /// True when every named endpoint answered. + public static async Task CheckAsync(params ExampleTransport[] transports) + { + var ok = true; + + foreach (var transport in transports.Distinct().OrderBy(t => t)) + { + string? failure = transport == ExampleTransport.Http + ? await CheckHttpAsync() + : await CheckTcpAsync(); + + if (failure is not null) + { + Report(transport, failure); + ok = false; + } + } + + return ok; + } + + private static async Task CheckHttpAsync() + { + try + { + using var cancellation = new CancellationTokenSource(Timeout); + using var client = ExampleConfig.CreateHttpClient(); + await client.ExecuteScalarAsync("SELECT 1", cancellationToken: cancellation.Token); + return null; + } + catch (Exception ex) + { + return ex.Message; + } + } + + private static async Task CheckTcpAsync() + { + try + { + using var cancellation = new CancellationTokenSource(Timeout); + await using var client = ExampleConfig.CreateTcpClient(); + await client.PingAsync(cancellation.Token); + return null; + } + catch (Exception ex) + { + return ex.Message; + } + } + + private static void Report(ExampleTransport transport, string failure) + { + var (name, endpoint, port, source) = transport == ExampleTransport.Http + ? ("HTTP interface", $"{ExampleConfig.Host}:{ExampleConfig.HttpPort}", "CLICKHOUSE_HTTP_PORT", "CLICKHOUSE_HTTP_CONNECTION_STRING") + : ("native protocol", $"{ExampleConfig.Host}:{ExampleConfig.TcpPort}", "CLICKHOUSE_TCP_PORT", "CLICKHOUSE_TCP_CONNECTION_STRING"); + + Console.WriteLine(); + Console.WriteLine($"Cannot reach ClickHouse on the {name} at {endpoint} as user '{ExampleConfig.Username}'."); + Console.WriteLine($" {failure}"); + Console.WriteLine(); + + if (transport == ExampleTransport.Tcp) + { + Console.WriteLine($" The native protocol listens on port 9000 by default, not on the HTTP port ({ExampleConfig.HttpPort})."); + Console.WriteLine(); + } + + Console.WriteLine(" Start a server with both ports published:"); + Console.WriteLine(" docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server"); + Console.WriteLine(); + Console.WriteLine(" Or point the examples somewhere else:"); + Console.WriteLine($" CLICKHOUSE_HOST, {port}, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD, CLICKHOUSE_DATABASE"); + Console.WriteLine($" {source} replaces the whole connection string."); + Console.WriteLine(); + } +} diff --git a/examples/ExampleRunner.cs b/examples/ExampleRunner.cs index 02c6c72ae..6bd14aa5f 100644 --- a/examples/ExampleRunner.cs +++ b/examples/ExampleRunner.cs @@ -19,6 +19,15 @@ public record ExampleInfo(string ClassName, Type Type, MethodInfo RunMethod) /// Normalized form for matching (lowercase, no underscores). /// public string NormalizedName { get; } = Normalize(ClassName); + + /// + /// Which transport the example needs a server on. Read from the class name, because every + /// example shares one namespace and so a native-protocol example cannot reuse an HTTP + /// example's class name — the Tcp prefix that keeps them apart is the signal. + /// + public ExampleTransport Transport { get; } = ClassName.StartsWith("Tcp", StringComparison.Ordinal) + ? ExampleTransport.Tcp + : ExampleTransport.Http; } /// @@ -26,6 +35,14 @@ public record ExampleInfo(string ClassName, Type Type, MethodInfo RunMethod) /// public static IReadOnlyList Examples => _examples; + /// + /// Gets the examples that use one transport. + /// + /// The transport to select. + /// The matching examples, in class-name order. + public static List ForTransport(ExampleTransport transport) + => _examples.Where(e => e.Transport == transport).ToList(); + /// /// Finds examples matching the given filter using fuzzy matching. /// Matches against any substring of the normalized class name. @@ -50,11 +67,13 @@ public static async Task RunExample(ExampleInfo example) /// /// Lists all available examples to the console. /// - public static void ListExamples() + public static void ListExamples(ExampleTransport? transport = null) { - Console.WriteLine("Available examples:\n"); + var listed = transport is { } only ? ForTransport(only) : _examples.ToList(); + + Console.WriteLine(transport is { } named ? $"Available {named} examples:\n" : "Available examples:\n"); - foreach (var example in _examples.OrderBy(e => e.ClassName)) + foreach (var example in listed.OrderBy(e => e.ClassName)) { Console.WriteLine($" - {example.ClassName}"); } @@ -62,7 +81,10 @@ public static void ListExamples() Console.WriteLine(); Console.WriteLine("Usage:"); Console.WriteLine(" dotnet run Run all examples"); + Console.WriteLine(" dotnet run -- --http Run only the HTTP examples"); + Console.WriteLine(" dotnet run -- --tcp Run only the native protocol examples"); Console.WriteLine(" dotnet run -- --list List available examples"); + Console.WriteLine(" dotnet run -- --list --tcp List one transport's examples"); Console.WriteLine(" dotnet run -- --filter Run examples matching pattern"); Console.WriteLine(" dotnet run -- Shorthand for --filter"); Console.WriteLine(); diff --git a/examples/ExampleTransport.cs b/examples/ExampleTransport.cs new file mode 100644 index 000000000..d70f9d83d --- /dev/null +++ b/examples/ExampleTransport.cs @@ -0,0 +1,14 @@ +namespace ClickHouse.Driver.Examples; + +/// +/// Which ClickHouse interface an example talks to. The two use different ports, so an example can +/// fail for no reason other than the other one's port being closed. +/// +public enum ExampleTransport +{ + /// The HTTP interface, port 8123 by default. + Http, + + /// The native protocol, port 9000 by default. + Tcp, +} diff --git a/examples/Program.cs b/examples/Program.cs index 9bf3bced0..244a951d9 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -12,11 +12,11 @@ static async Task Main(string[] args) try { - var filter = ParseArgs(args, out bool showList); + var filter = ParseArgs(args, out bool showList, out ExampleTransport? transport); if (showList) { - ExampleRunner.ListExamples(); + ExampleRunner.ListExamples(transport); return; } @@ -24,8 +24,17 @@ static async Task Main(string[] args) { await RunFiltered(filter, isInteractive); } + else if (transport is { } only) + { + await RunTransport(only, isInteractive); + } else { + if (!await ExamplePreflight.CheckAsync(ExampleRunner.Examples)) + { + Environment.Exit(1); + } + await RunAllExamples(isInteractive); } } @@ -50,6 +59,11 @@ private static async Task RunFiltered(string filter, bool isInteractive) Console.WriteLine($"Found {matches.Count} matching example(s):\n"); + if (!await ExamplePreflight.CheckAsync(matches)) + { + Environment.Exit(1); + } + foreach (var example in matches) { await ExampleRunner.RunExample(example); @@ -58,6 +72,27 @@ private static async Task RunFiltered(string filter, bool isInteractive) } } + private static async Task RunTransport(ExampleTransport transport, bool isInteractive) + { + var selected = ExampleRunner.ForTransport(transport); + + Console.WriteLine($"Running {selected.Count} {transport} example(s):\n"); + + // The named transport, not the selection's, so that asking for one with none written yet + // still reports whether its endpoint answers. + if (!await ExamplePreflight.CheckAsync(transport)) + { + Environment.Exit(1); + } + + foreach (var example in selected) + { + await ExampleRunner.RunExample(example); + WaitForUser(isInteractive); + Console.WriteLine("\n"); + } + } + private static async Task RunAllExamples(bool isInteractive) { // Core Usage & Configuration @@ -308,9 +343,10 @@ private static async Task RunAllExamples(bool isInteractive) Console.WriteLine(new string('=', 70)); } - private static string? ParseArgs(string[] args, out bool showList) + private static string? ParseArgs(string[] args, out bool showList, out ExampleTransport? transport) { showList = false; + transport = null; string? filter = null; for (int i = 0; i < args.Length; i++) @@ -320,7 +356,19 @@ private static async Task RunAllExamples(bool isInteractive) if (arg == "--list" || arg == "-l") { showList = true; - return null; + continue; + } + + if (arg == "--http") + { + transport = ExampleTransport.Http; + continue; + } + + if (arg == "--tcp") + { + transport = ExampleTransport.Tcp; + continue; } if (arg == "--filter" || arg == "-f") diff --git a/examples/README.md b/examples/README.md index 69b4a054e..def5dde27 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,7 +8,7 @@ We aim to cover various scenarios of driver usage with these examples. You shoul If something is missing, or you found a mistake in one of these examples, please open an issue or a pull request. [AGENTS.md](AGENTS.md) has the checklist for adding one. -Examples are grouped by transport. Everything under [Http/](Http) uses `ClickHouseClient` or `ClickHouseConnection` over HTTP. +Examples are grouped by transport. Everything under [Http/](Http) uses `ClickHouseClient` or `ClickHouseConnection` over HTTP; everything under [Tcp/](Tcp) uses `ClickHouseTcpClient` over the native protocol, and needs port 9000 rather than 8123 — see [Tcp/README.md](Tcp/README.md). ## Examples @@ -118,8 +118,13 @@ cd examples # Run all examples dotnet run -# List available examples +# Run only one transport's examples +dotnet run -- --http +dotnet run -- --tcp + +# List available examples, optionally for one transport dotnet run -- --list +dotnet run -- --list --tcp # Run specific example(s) using a filter dotnet run -- --filter basicusage @@ -128,15 +133,35 @@ dotnet run -- --filter basicusage dotnet run -- basicusage ``` +Before running anything, the runner reaches the endpoints the selected examples need and reports what to fix if one does not answer, rather than letting the first example fail with a connection error. + The filter matches the example's class name, which `--list` prints. A class name is the topic without the file's category prefix: `Core_001_BasicUsage.cs` declares `class BasicUsage`. Matching ignores case and underscores and accepts any substring, so `basicusage`, `basic` and `usage` all match it. The file's `core001` prefix does not. ### Connection configuration -By default, examples connect to ClickHouse at `localhost:8123` with the `default` user and no password. If your setup is different, you can: +Every example takes its server from [ExampleConfig.cs](ExampleConfig.cs), so one environment variable +points the whole suite somewhere else. The defaults are what a stock server container exposes on +localhost, and the examples run with nothing set. + +| Variable | Default | +| --- | --- | +| `CLICKHOUSE_HOST` | `localhost` | +| `CLICKHOUSE_HTTP_PORT` | `8123` | +| `CLICKHOUSE_TCP_PORT` | `9000` | +| `CLICKHOUSE_USER` | `default` | +| `CLICKHOUSE_PASSWORD` | empty | +| `CLICKHOUSE_DATABASE` | `default` | + +For an endpoint those pieces cannot describe — TLS, a cloud host, an extra setting — set +`CLICKHOUSE_HTTP_CONNECTION_STRING` or `CLICKHOUSE_TCP_CONNECTION_STRING` to replace the whole string: + +```bash +CLICKHOUSE_HOST=my-server CLICKHOUSE_PASSWORD=secret dotnet run -- basicusage +``` -1. Modify the connection strings in the examples -2. Set up a local ClickHouse instance with default settings -3. Use environment variables or configuration files (see [Core_002_ConnectionStringConfiguration.cs](Http/Core/Core_002_ConnectionStringConfiguration.cs)) +Two examples keep literal connection strings, because configuration is what they teach: +[Core_002_ConnectionStringConfiguration.cs](Http/Core/Core_002_ConnectionStringConfiguration.cs) and +[Core_003_DependencyInjection.cs](Http/Core/Core_003_DependencyInjection.cs). ### ClickHouse Cloud diff --git a/examples/Tcp/README.md b/examples/Tcp/README.md new file mode 100644 index 000000000..a086a5fa5 --- /dev/null +++ b/examples/Tcp/README.md @@ -0,0 +1,59 @@ +# Native protocol examples + +These use `ClickHouseTcpClient`, which speaks ClickHouse's native TCP protocol, rather than the +`ClickHouseClient` / `ClickHouseConnection` pair in [../Http](../Http) that speaks HTTP. The index of +every example, and how to run one, is in [the top-level README](../README.md). + +## Before you run them + +**They need port 9000, not 8123.** The two interfaces are separate listeners, so a server reachable +over HTTP is not necessarily reachable here: + +```bash +docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server +``` + +`dotnet run -- --tcp` runs only these, and checks the endpoint before starting. + +## The API is experimental + +Every public type of the native client carries `[Experimental("CHTCP0001")]`, so using one is a +compile error until you acknowledge that the surface may change in a future release: + +```csharp +#pragma warning disable CHTCP0001 // The native protocol client's API is not yet stable. +``` + +Per file as above, or once for a project: + +```xml +$(NoWarn);CHTCP0001 +``` + +This examples project takes the project-wide route, which is why no file here opens with the pragma. + +## What the native client does not do + +Reach for the HTTP client instead when you need: + +- **A format other than Native** — the protocol carries columnar blocks, so there is no CSV, JSONEachRow + or Parquet ingestion or export, and no raw stream insert. +- **ADO.NET, and so any ORM.** There is no `DbConnection` implementation over this transport, so Dapper, + EF Core and linq2db do not work with it. +- **JWT or bearer authentication.** Username and password only. +- **Custom HTTP headers**, which have no equivalent on the wire. +- **A parameter type resolver, a parameter formatter, or a read value converter.** The native client + has no hook for any of the three; a parameter's type comes from the `{name:Type}` placeholder in + the query or from `ClickHouseTcpParameter.ClickHouseType`. + +## What only the native client does + +- **Blocks and columns.** `StreamAsync` yields a `Block` whose typed columns expose `ReadOnlySpan` + over the server's own layout, so a read can avoid materializing rows at all — and a column read out + of a block re-inserts without being rebuilt. +- **Real sessions.** `OpenSessionAsync` pins one connection, so a temporary table or a `SET` survives + from one operation to the next without the caveats an HTTP session carries. +- **Progress, profile info and profile events while a query runs**, through + `ClickHouseTcpQueryCallbacks`, rather than as headers after the fact. +- **Block compression** on the wire, LZ4 by default. +- **Bit-plane access to `QBit` columns**, through `IQBitColumn`. From aea6293691e63b154636b203f4d4126e655bd7f5 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 11:34:30 +0200 Subject: [PATCH 04/16] Add the native protocol's core usage examples Four under examples/Tcp/Core/: constructing a client and running the four operations; the connection string's native key set and deriving an options variant; AddClickHouseTcpDataSource with keyed registrations and who disposes the pool; and the same task written over both transports, with the call-for-call mapping and an honest account of what the native client cannot do. An example that talks to both interfaces needs both endpoints checked, which its class-name prefix does not say, so ExampleInfo carries RequiredTransports alongside the transport it is filed under. Tcp/README.md said every public type of the native client is experimental. Six are; the options record, the builder, Block, the columns and the exceptions are not. It also now records that per-query roles and databases have no native equivalent, and that a timestamp column reaches the row and block tiers as the integer the wire carried. Co-Authored-By: Claude Opus 5 (1M context) --- examples/ExamplePreflight.cs | 2 +- examples/ExampleRunner.cs | 19 +- examples/Program.cs | 30 ++- examples/README.md | 9 + examples/Tcp/Core/Tcp_001_BasicUsage.cs | 120 +++++++++++ examples/Tcp/Core/Tcp_002_ConnectionString.cs | 165 +++++++++++++++ .../Tcp/Core/Tcp_003_DependencyInjection.cs | 160 +++++++++++++++ .../Tcp/Core/Tcp_004_MigratingFromHttp.cs | 194 ++++++++++++++++++ examples/Tcp/README.md | 14 +- 9 files changed, 706 insertions(+), 7 deletions(-) create mode 100644 examples/Tcp/Core/Tcp_001_BasicUsage.cs create mode 100644 examples/Tcp/Core/Tcp_002_ConnectionString.cs create mode 100644 examples/Tcp/Core/Tcp_003_DependencyInjection.cs create mode 100644 examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs diff --git a/examples/ExamplePreflight.cs b/examples/ExamplePreflight.cs index eb51cc7d3..e1913e595 100644 --- a/examples/ExamplePreflight.cs +++ b/examples/ExamplePreflight.cs @@ -21,7 +21,7 @@ public static class ExamplePreflight /// The examples about to run. Only their transports are checked. /// True when every needed endpoint answered. public static Task CheckAsync(IEnumerable examples) - => CheckAsync(examples.Select(e => e.Transport).Distinct().ToArray()); + => CheckAsync(examples.SelectMany(e => e.RequiredTransports).Distinct().ToArray()); /// /// Checks the named endpoints, and reports what to fix if one is unreachable. diff --git a/examples/ExampleRunner.cs b/examples/ExampleRunner.cs index 6bd14aa5f..3486392ce 100644 --- a/examples/ExampleRunner.cs +++ b/examples/ExampleRunner.cs @@ -8,6 +8,15 @@ namespace ClickHouse.Driver.Examples; /// public static class ExampleRunner { + /// + /// Examples that talk to both interfaces, so their class-name prefix understates what they need. + /// Declared before _examples: static initializers run in order, and discovery reads this. + /// + private static readonly HashSet _crossTransport = new(StringComparer.Ordinal) + { + "TcpMigratingFromHttp", + }; + private static readonly List _examples = DiscoverExamples(); /// @@ -21,13 +30,21 @@ public record ExampleInfo(string ClassName, Type Type, MethodInfo RunMethod) public string NormalizedName { get; } = Normalize(ClassName); /// - /// Which transport the example needs a server on. Read from the class name, because every + /// Which transport the example is filed under. Read from the class name, because every /// example shares one namespace and so a native-protocol example cannot reuse an HTTP /// example's class name — the Tcp prefix that keeps them apart is the signal. /// public ExampleTransport Transport { get; } = ClassName.StartsWith("Tcp", StringComparison.Ordinal) ? ExampleTransport.Tcp : ExampleTransport.Http; + + /// + /// Every endpoint the example needs to reach, which is not always the one it is filed under: + /// an example comparing the two transports needs both. + /// + public IReadOnlyList RequiredTransports { get; } = _crossTransport.Contains(ClassName) + ? [ExampleTransport.Http, ExampleTransport.Tcp] + : [ClassName.StartsWith("Tcp", StringComparison.Ordinal) ? ExampleTransport.Tcp : ExampleTransport.Http]; } /// diff --git a/examples/Program.cs b/examples/Program.cs index 244a951d9..813e61f1c 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -78,9 +78,12 @@ private static async Task RunTransport(ExampleTransport transport, bool isIntera Console.WriteLine($"Running {selected.Count} {transport} example(s):\n"); - // The named transport, not the selection's, so that asking for one with none written yet - // still reports whether its endpoint answers. - if (!await ExamplePreflight.CheckAsync(transport)) + // The named transport plus whatever the selection needs beyond it, so that asking for one + // with none written yet still reports whether its endpoint answers, and an example + // comparing the two transports still gets both checked. + var needed = selected.SelectMany(e => e.RequiredTransports).Append(transport).Distinct().ToArray(); + + if (!await ExamplePreflight.CheckAsync(needed)) { Environment.Exit(1); } @@ -338,6 +341,27 @@ private static async Task RunAllExamples(bool isInteractive) await Testcontainers.Run(); WaitForUser(isInteractive); + // Native Protocol: Core Usage & Configuration + Console.WriteLine("\n\n" + new string('=', 70)); + Console.WriteLine("NATIVE PROTOCOL: CORE USAGE & CONFIGURATION"); + Console.WriteLine(new string('=', 70) + "\n"); + + Console.WriteLine($"Running: {nameof(TcpBasicUsage)}"); + await TcpBasicUsage.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpConnectionString)}"); + await TcpConnectionString.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpDependencyInjection)}"); + await TcpDependencyInjection.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpMigratingFromHttp)}"); + await TcpMigratingFromHttp.Run(); + WaitForUser(isInteractive); + Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ALL EXAMPLES COMPLETED SUCCESSFULLY!"); Console.WriteLine(new string('=', 70)); diff --git a/examples/README.md b/examples/README.md index def5dde27..393503e7c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -97,6 +97,15 @@ Examples are grouped by transport. Everything under [Http/](Http) uses `ClickHou - [Testing_001_Testcontainers.cs](Http/Testing/Testing_001_Testcontainers.cs) - Using Testcontainers to spin up ephemeral ClickHouse instances for integration testing +### Native Protocol: Core Usage & Configuration + +These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/README.md) first. + +- [Tcp_001_BasicUsage.cs](Tcp/Core/Tcp_001_BasicUsage.cs) - Constructing `ClickHouseTcpClient`, DDL with `ExecuteAsync`, inserting with `InsertRowsAsync`, reading with `QueryAsync` and `ExecuteScalarAsync`, and disposal +- [Tcp_002_ConnectionString.cs](Tcp/Core/Tcp_002_ConnectionString.cs) - The native key set (compression codec, pool keys, TLS keys, no `Protocol`), `ClickHouseTcpConnectionStringBuilder`, and deriving an options variant with a `with` expression +- [Tcp_003_DependencyInjection.cs](Tcp/Core/Tcp_003_DependencyInjection.cs) - `AddClickHouseTcpDataSource`, injecting `IClickHouseTcpClient`, keyed registrations for two clusters, and who disposes the shared pool +- [Tcp_004_MigratingFromHttp.cs](Tcp/Core/Tcp_004_MigratingFromHttp.cs) - The same task over both transports, the call-for-call API mapping, the `CHTCP0001` opt-in, and what the native client cannot do + ## How to run ### Prerequisites diff --git a/examples/Tcp/Core/Tcp_001_BasicUsage.cs b/examples/Tcp/Core/Tcp_001_BasicUsage.cs new file mode 100644 index 000000000..667a16a8c --- /dev/null +++ b/examples/Tcp/Core/Tcp_001_BasicUsage.cs @@ -0,0 +1,120 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// The native-protocol client from end to end: construct a , run DDL with +/// ExecuteAsync, insert rows with InsertRowsAsync, read them back with QueryAsync, read one +/// value with ExecuteScalarAsync, and dispose it. +/// +/// +/// This client speaks ClickHouse's own TCP protocol on port 9000, and it is not an ADO.NET provider. See +/// Tcp_004_MigratingFromHttp for how each HTTP-client call maps onto it, and for what it cannot do. +/// +/// +public static class TcpBasicUsage +{ + // These examples are not the test suite, so a fixed name is fine. It is dropped even if a step throws. + private const string TableName = "example_tcp_basic_usage"; + + public static async Task Run() + { + // One client per endpoint, kept for the life of the application: it owns a connection pool, is safe to + // share across threads, and runs as many operations at once as the pool is wide. Building one per + // operation would pay for a connect and a handshake every time. + // + // 'await using', not 'using': the client is IAsyncDisposable, and disposal closes sockets. + await using var client = ExampleConfig.CreateTcpClient(); + + Console.WriteLine($"Native protocol endpoint: {ExampleConfig.Host}:{ExampleConfig.TcpPort}, user '{ExampleConfig.Username}'"); + + // Read out of the handshake the connection already made, so this costs no query. + var server = await client.GetServerInfoAsync(); + Console.WriteLine($"Server: {server} (protocol revision {server.ProtocolRevision}, timezone {server.Timezone})"); + + try + { + await CreateTable(client); + await InsertRows(client); + await ReadRows(client); + await ReadOneValue(client); + ShowTheReadTiers(); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped '{TableName}'. Disposing the client closes its pooled connections."); + } + } + + private static async Task CreateTable(ClickHouseTcpClient client) + { + // ExecuteAsync is for anything that returns no rows: DDL, and DML other than INSERT ... VALUES. + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + id UInt64, + name String, + score Float64 + ) + ENGINE = MergeTree() + ORDER BY id"); + + Console.WriteLine($"\nCreated '{TableName}' (id UInt64, name String, score Float64)"); + } + + private static async Task InsertRows(ClickHouseTcpClient client) + { + // The statement ends at VALUES: the rows travel after it as native blocks, never as SQL text. Each + // object[] is matched to the column list by position. + // + // A column takes the CLR type of its first non-null value, so keep one type per column: ulong for + // UInt64, string for String, double for Float64. + var rows = new List + { + new object[] { 1UL, "Ada", 99.5 }, + new object[] { 2UL, "Grace", 97.25 }, + new object[] { 3UL, "Alan", 91.0 }, + }; + + await client.InsertRowsAsync($"INSERT INTO {TableName} (id, name, score) VALUES", rows); + + Console.WriteLine($"Inserted {rows.Count} rows with InsertRowsAsync"); + } + + private static async Task ReadRows(ClickHouseTcpClient client) + { + Console.WriteLine("\nQueryAsync yields one object[] per row, values in the order the SELECT names them:"); + Console.WriteLine(" ID Name Score"); + Console.WriteLine(" -- ----- -----"); + + // Rows arrive as they are read off the connection rather than after the whole result is buffered. Each + // object[] is yours to keep; the enumeration holds a connection until it ends, so read it to the end. + await foreach (object[] row in client.QueryAsync($"SELECT id, name, score FROM {TableName} ORDER BY id")) + { + Console.WriteLine($" {(ulong)row[0],2} {(string)row[1],-5} {(double)row[2],5}"); + } + } + + private static async Task ReadOneValue(ClickHouseTcpClient client) + { + // ExecuteScalarAsync returns the first column of the first row, boxed. It reads the whole result before + // returning, so write a query that produces one row. + object count = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + + // count() is UInt64, so the box holds a ulong: a value's CLR type follows the column's ClickHouse type. + Console.WriteLine($"\nExecuteScalarAsync(\"SELECT count() ...\") = {count} (boxed {count.GetType().Name})"); + } + + private static void ShowTheReadTiers() + { + Console.WriteLine("\nThree read tiers, all on this client:"); + Console.WriteLine(" QueryAsync one object[] per row, every value boxed"); + Console.WriteLine(" QueryAsync one POCO per row, filled by column name"); + Console.WriteLine(" StreamAsync whole Blocks, typed columns, no per-row boxing"); + Console.WriteLine(); + Console.WriteLine("The row tier boxes the value the wire carried, which is not always the CLR type the column"); + Console.WriteLine("name suggests: a DateTime column arrives as uint epoch seconds. Read date and time columns"); + Console.WriteLine("through QueryAsync into a DateTime or DateTimeOffset property."); + } +} diff --git a/examples/Tcp/Core/Tcp_002_ConnectionString.cs b/examples/Tcp/Core/Tcp_002_ConnectionString.cs new file mode 100644 index 000000000..67f52061f --- /dev/null +++ b/examples/Tcp/Core/Tcp_002_ConnectionString.cs @@ -0,0 +1,165 @@ +using ClickHouse.Driver.Compression; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// Configuring the native-protocol client: what a native connection string holds, how +/// builds one, and how it becomes a +/// — the record every client is built from. +/// +/// +/// The key set is not the HTTP one. There is no Protocol key, Compression names a codec instead of +/// switching a boolean, and the pool and TLS keys have no HTTP counterpart at all. +/// +/// +/// +/// Configuration is this example's subject, so the connection strings in its output are literals. The client it +/// connects with still comes from ExampleConfig. +/// +/// +public static class TcpConnectionString +{ + public static async Task Run() + { + ShowTheKeys(); + ClickHouseTcpClientOptions options = BuildOptions(); + DeriveAVariant(options); + ShowTlsKeys(); + await ConnectWithThem(options); + } + + private static void ShowTheKeys() + { + Console.WriteLine("1. A native-protocol connection string:\n"); + Console.WriteLine(" Host=localhost;Port=9000;Username=default;Password=secret;Database=default"); + Console.WriteLine(); + Console.WriteLine(" Every key is optional: Host defaults to localhost, Username to default, Database to"); + Console.WriteLine(" default, Password to empty. Port is the one to watch — the native protocol listens on"); + Console.WriteLine(" 9000, not on the HTTP interface's 8123."); + Console.WriteLine(); + Console.WriteLine(" There is no Protocol key. UseTls=true selects TLS, and an unset Port then resolves to"); + Console.WriteLine(" 9440, the secure native port, instead of 9000."); + + Console.WriteLine("\n2. The keys that are not in the HTTP set:\n"); + Console.WriteLine(" Compression=lz4|zstd|none A codec name, where the HTTP client's Compression is a"); + Console.WriteLine(" boolean. lz4 is the default, so wire blocks are"); + Console.WriteLine(" compressed in both directions unless this says none."); + Console.WriteLine(" Pool MinPoolSize, MaxPoolSize, PoolTimeout, IdleTimeout,"); + Console.WriteLine(" MaxConnectionLifetime, SweepInterval, PoolReusePolicy"); + Console.WriteLine(" TLS UseTls, TlsServerName, TlsCaCertificatePath,"); + Console.WriteLine(" TlsAllowInvalidCertificates"); + Console.WriteLine(" Deadlines DialTimeout, ReadTimeout (both in seconds)"); + Console.WriteLine(" Other QuotaKey, MaxSendBufferBytes, and set_="); + Console.WriteLine(" for a ClickHouse setting sent with every operation"); + } + + private static ClickHouseTcpClientOptions BuildOptions() + { + // Every key the builder knows has a typed property, so a name is checked at compile time rather than + // kept as an unknown key and ignored. An unreadable UseTls, TLS-authority or PoolReusePolicy value throws; + // an unreadable number falls back to its default. + var builder = ExampleConfig.TcpBuilder(); + builder.Compression = "zstd"; + builder.MaxPoolSize = 4; + builder.IdleTimeout = TimeSpan.FromSeconds(60); + + // Custom settings have no typed property: any set_ key becomes a client-level ClickHouse setting. + builder["set_max_threads"] = 2; + + Console.WriteLine("\n3. ClickHouseTcpConnectionStringBuilder:\n"); + Console.WriteLine($" Host {builder.Host}"); + Console.WriteLine($" Port {builder.Port?.ToString() ?? "(unset: resolved from UseTls)"}"); + Console.WriteLine($" Username {builder.Username}"); + Console.WriteLine($" Password {(builder.Password.Length == 0 ? "(empty)" : "(set — not printed)")}"); + Console.WriteLine($" Database {builder.Database}"); + Console.WriteLine($" Compression {builder.Compression}"); + Console.WriteLine($" MaxPoolSize {builder.MaxPoolSize}"); + Console.WriteLine($" IdleTimeout {builder.IdleTimeout.TotalSeconds}s"); + Console.WriteLine($" UseTls {builder.UseTls}"); + Console.WriteLine(); + Console.WriteLine(" builder.ToString() would render all of that back as a connection string, password"); + Console.WriteLine(" included, so it is not something to log."); + + // ToOptions() materializes the keys; FromConnectionString(text) is the same thing in one call for a string + // that came from configuration. + ClickHouseTcpClientOptions options = builder.ToOptions(); + ClickHouseTcpClientOptions fromText = ClickHouseTcpClientOptions.FromConnectionString(ExampleConfig.TcpConnectionString); + + Console.WriteLine("\n4. ClickHouseTcpClientOptions — what a client is really built from:\n"); + + // The record's generated ToString would print the password; this override names only the safe properties, + // so options are safe to log. The port it shows is the resolved one. + Console.WriteLine($" builder.ToOptions() {options}"); + Console.WriteLine($" Compressor {Describe(options.Compressor)}"); + Console.WriteLine($" MaxPoolSize {options.MaxPoolSize}"); + Console.WriteLine($" IdleTimeout {options.IdleTimeout}"); + Console.WriteLine($" CustomSettings {string.Join(", ", options.CustomSettings.Select(s => $"{s.Key}={s.Value}"))}"); + Console.WriteLine(); + Console.WriteLine($" FromConnectionString(ExampleConfig.TcpConnectionString) {fromText}"); + Console.WriteLine($" Compressor {Describe(fromText.Compressor)}"); + Console.WriteLine(" That string carries no Compression key, so the lz4 default stands. Only 'none'"); + Console.WriteLine(" leaves the codec null, and null means the query asks for no compression at all."); + + return options; + } + + private static string Describe(IClickHouseCompressor compressor) + => compressor?.GetType().Name ?? "(none)"; + + private static void DeriveAVariant(ClickHouseTcpClientOptions options) + { + // Options are an init-only record, so one instance can hold what every client shares and a 'with' + // expression derives the variant. The original is untouched. + ClickHouseTcpClientOptions wide = options with { MaxPoolSize = 32, Database = "system" }; + + Console.WriteLine("\n5. Options is a record, so 'with' derives a variant:\n"); + Console.WriteLine($" options with {{ MaxPoolSize = 32, Database = \"system\" }}"); + Console.WriteLine($" original MaxPoolSize={options.MaxPoolSize}, Database={options.Database}"); + Console.WriteLine($" variant MaxPoolSize={wide.MaxPoolSize}, Database={wide.Database}"); + } + + private static void ShowTlsKeys() + { + Console.WriteLine("\n6. The TLS keys, and how they are checked:\n"); + Console.WriteLine(" UseTls=true encrypt the transport, and dial 9440 unless Port says"); + Console.WriteLine(" otherwise. The handshake carries the password in the"); + Console.WriteLine(" clear, so this is what protects it."); + Console.WriteLine(" TlsServerName=host the name to match the certificate against, when Host"); + Console.WriteLine(" is an address or an internal alias"); + Console.WriteLine(" TlsCaCertificatePath=ca.pem validate against these authorities instead of the"); + Console.WriteLine(" host trust store"); + Console.WriteLine(" TlsAllowInvalidCertificates=true accept any certificate — development only"); + + // A TLS key with UseTls left false is refused at construction. Silently ignoring it is how a connection + // meant to be encrypted ends up in the clear. + try + { + _ = new ClickHouseTcpClient(new ClickHouseTcpClientOptions + { + Host = ExampleConfig.Host, + TlsAllowInvalidCertificates = true, + }); + } + catch (ArgumentException ex) + { + Console.WriteLine(); + Console.WriteLine(" A TLS key set while UseTls is false is rejected, not ignored:"); + Console.WriteLine($" {ex.Message}"); + } + } + + private static async Task ConnectWithThem(ClickHouseTcpClientOptions options) + { + Console.WriteLine("\n7. Running with those options:\n"); + + await using var client = new ClickHouseTcpClient(options); + + var server = await client.GetServerInfoAsync(); + Console.WriteLine($" Connected to {server}, blocks framed with {Describe(options.Compressor)}"); + + // set_max_threads became a client-level setting, so the server sees it on every operation. + object maxThreads = await client.ExecuteScalarAsync("SELECT getSetting('max_threads')"); + Console.WriteLine($" getSetting('max_threads') = {maxThreads} — the set_max_threads key reached the server"); + } +} diff --git a/examples/Tcp/Core/Tcp_003_DependencyInjection.cs b/examples/Tcp/Core/Tcp_003_DependencyInjection.cs new file mode 100644 index 000000000..74d16e5ef --- /dev/null +++ b/examples/Tcp/Core/Tcp_003_DependencyInjection.cs @@ -0,0 +1,160 @@ +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Driver.Examples; + +/// +/// Registering the native-protocol client in an with +/// AddClickHouseTcpDataSource: the connection-string and options overloads, injecting +/// into a consumer, the keyed overload for two clusters, and the one rule that +/// matters — the pool is a singleton, so nothing injected may dispose it. +/// +public static class TcpDependencyInjection +{ + public static async Task Run() + { + Console.WriteLine("One call registers three services, all singletons:\n"); + Console.WriteLine(" ClickHouseTcpDataSource owns the client and its connection pool; the container disposes it"); + Console.WriteLine(" IClickHouseTcpClient the client that data source owns — queries, inserts, sessions"); + Console.WriteLine(" IClickHouseTcpOperations the same object again, for code that only runs operations"); + + await FromConnectionString(); + await FromOptions(); + await TwoClusters(); + await WhoDisposesWhat(); + } + + private static async Task FromConnectionString() + { + Console.WriteLine("\n1. From a connection string:\n"); + + var services = new ServiceCollection(); + services.AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString); + + // A consumer takes the interface. Registered through a factory here only because this example's consumer + // is a private nested type; a normal AddSingleton() reaches the same client, and a keyed one + // is reached with [FromKeyedServices("key")] on the constructor parameter. + services.AddSingleton(sp => new ServerProbe(sp.GetRequiredService())); + + await using ServiceProvider provider = services.BuildServiceProvider(); + + var probe = provider.GetRequiredService(); + Console.WriteLine($" ServerProbe (injected IClickHouseTcpClient): {await probe.DescribeAsync()}"); + + // Every registration resolves the one client the data source owns, so there is one pool per registration + // however many consumers there are. + var dataSource = provider.GetRequiredService(); + var client = provider.GetRequiredService(); + var operations = provider.GetRequiredService(); + + Console.WriteLine($" IClickHouseTcpClient is dataSource.GetClient(): {ReferenceEquals(client, dataSource.GetClient())}"); + Console.WriteLine($" IClickHouseTcpOperations is the same object: {ReferenceEquals(client, operations)}"); + } + + private static async Task FromOptions() + { + Console.WriteLine("\n2. From options, with the container's logging:\n"); + + var services = new ServiceCollection(); + services.AddLogging(logging => logging.AddConsole().SetMinimumLevel(LogLevel.Warning)); + + // Options are a record, so the shape that differs from the connection string is a 'with' away. + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions() with + { + MaxPoolSize = 4, + IdleTimeout = TimeSpan.FromSeconds(30), + }; + + services.AddClickHouseTcpDataSource(options); + + await using ServiceProvider provider = services.BuildServiceProvider(); + + var dataSource = provider.GetRequiredService(); + Console.WriteLine($" {dataSource.Options}"); + Console.WriteLine($" MaxPoolSize {dataSource.Options.MaxPoolSize}, IdleTimeout {dataSource.Options.IdleTimeout}"); + + // The registration fills in a null LoggerFactory from the container, on a copy: the options object passed + // in is left alone, so the two are no longer the same instance. + Console.WriteLine($" LoggerFactory on the options passed in: {options.LoggerFactory?.GetType().Name ?? "(null)"}"); + Console.WriteLine($" LoggerFactory the data source runs with: {dataSource.Options.LoggerFactory?.GetType().Name ?? "(null)"}"); + + // There is also an overload taking Func, for options that + // need something else out of the container, and one taking a Func that builds the data source itself. + object value = await provider.GetRequiredService().ExecuteScalarAsync("SELECT 'registered from options'"); + Console.WriteLine($" SELECT returned: {value}"); + } + + private static async Task TwoClusters() + { + Console.WriteLine("\n3. Two clusters, told apart by service key:\n"); + + var services = new ServiceCollection(); + + // A second unkeyed call would be a no-op: every service is added with TryAdd, so the first registration + // of a service and key wins. A key is what makes the second registration a different service. + services.AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString, serviceKey: "ingest"); + services.AddClickHouseTcpDataSource( + ExampleConfig.TcpBuilder().ToOptions() with { MaxPoolSize = 2 }, + serviceKey: "reporting"); + + // Both point at this example's one server; in a real application they would be different endpoints. + services.AddSingleton(sp => new ServerProbe(sp.GetRequiredKeyedService("reporting"))); + + await using ServiceProvider provider = services.BuildServiceProvider(); + + var ingest = provider.GetRequiredKeyedService("ingest"); + var reporting = provider.GetRequiredKeyedService("reporting"); + + Console.WriteLine($" 'ingest' MaxPoolSize {ingest.Options.MaxPoolSize}, pool of its own: {!ReferenceEquals(ingest, reporting)}"); + Console.WriteLine($" 'reporting' MaxPoolSize {reporting.Options.MaxPoolSize}"); + Console.WriteLine($" ServerProbe holding the 'reporting' client: {await provider.GetRequiredService().DescribeAsync()}"); + + // Keyed registrations are not also unkeyed, so plain injection finds nothing. Key every consumer, or + // register one of the endpoints without a key as well. + Console.WriteLine($" An unkeyed IClickHouseTcpClient is registered: {provider.GetService() is not null}"); + } + + private static async Task WhoDisposesWhat() + { + Console.WriteLine("\n4. Who disposes what:\n"); + Console.WriteLine(" The data source owns the pool and the container owns the data source, so the pool"); + Console.WriteLine(" closes once, at shutdown, when the provider is disposed. Prefer DisposeAsync where the"); + Console.WriteLine(" call site can await it, as a generic host does."); + Console.WriteLine(); + Console.WriteLine(" Never dispose an injected client. It offers DisposeAsync because a session needs one,"); + Console.WriteLine(" but disposing it closes the shared pool and every other consumer's next operation"); + Console.WriteLine(" fails. A session from OpenSessionAsync is the opposite: it is yours to dispose."); + + ServiceProvider provider = new ServiceCollection() + .AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString) + .BuildServiceProvider(); + + var client = provider.GetRequiredService(); + await client.PingAsync(); + Console.WriteLine("\n Ping before shutdown: answered"); + + await provider.DisposeAsync(); + + // What a consumer that disposed the client would leave behind for everyone else. + try + { + await client.PingAsync(); + } + catch (ObjectDisposedException) + { + Console.WriteLine(" Ping after the provider was disposed: ObjectDisposedException, as it should be"); + } + } + + // Takes IClickHouseTcpClient rather than the concrete client, so a test can substitute a double. Holds no + // disposal logic: the container owns the client's lifetime. + private sealed class ServerProbe(IClickHouseTcpClient client) + { + public async Task DescribeAsync() + { + ClickHouseTcpServerInfo info = await client.GetServerInfoAsync(); + return $"{info}, protocol revision {info.ProtocolRevision}"; + } + } +} diff --git a/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs b/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs new file mode 100644 index 000000000..5448b0a70 --- /dev/null +++ b/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs @@ -0,0 +1,194 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// Moving code from the HTTP client to the native-protocol one: the experimental opt-in a consumer has to make, +/// the same task written both ways against one server, the call-for-call mapping, and an honest list of what the +/// native client does not do. +/// +/// +/// The list matters more than the mapping. Most HTTP calls have a native counterpart, but a few capabilities have +/// none at all, and they are the ones that decide whether a migration is possible. +/// +/// +public static class TcpMigratingFromHttp +{ + private const string TableName = "example_tcp_migrating_from_http"; + + public static async Task Run() + { + ShowTheOptIn(); + + // Two clients, one server: the HTTP interface on 8123 and the native protocol on 9000. + using var http = ExampleConfig.CreateHttpClient(); + await using var tcp = ExampleConfig.CreateTcpClient(); + + await SameTaskBothWays(http, tcp); + + ShowTheMapping(); + ShowWhatIsMissing(); + } + + private static void ShowTheOptIn() + { + Console.WriteLine("1. The experimental opt-in\n"); + Console.WriteLine(" ClickHouseTcpClient, ClickHouseTcpDataSource, the three IClickHouseTcp* interfaces and"); + Console.WriteLine(" AddClickHouseTcpDataSource carry [Experimental(\"CHTCP0001\")], so naming any of them is a"); + Console.WriteLine(" compile error until you acknowledge that the surface may still change."); + Console.WriteLine(); + Console.WriteLine(" The types around them — ClickHouseTcpClientOptions, the connection-string builder, Block,"); + Console.WriteLine(" the columns, the exceptions — do not carry it, so holding one raises no diagnostic even"); + Console.WriteLine(" though it is just as experimental."); + Console.WriteLine(); + Console.WriteLine(" Per file:"); + Console.WriteLine(" #pragma warning disable CHTCP0001 // The native protocol client's API is not yet stable."); + Console.WriteLine(); + Console.WriteLine(" Or once for a project:"); + Console.WriteLine(" $(NoWarn);CHTCP0001"); + Console.WriteLine(); + Console.WriteLine(" This examples project takes the project-wide route, which is why no file under Tcp/"); + Console.WriteLine(" opens with the pragma."); + } + + private static async Task SameTaskBothWays(ClickHouseClient http, ClickHouseTcpClient tcp) + { + Console.WriteLine("\n2. The same task, both ways\n"); + + try + { + await CompareTransports(http, tcp); + } + finally + { + await tcp.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\n Dropped '{TableName}'"); + } + } + + private static async Task CompareTransports(ClickHouseClient http, ClickHouseTcpClient tcp) + { + // DDL. HTTP: ExecuteNonQueryAsync, which returns the affected-row count ADO.NET expects. + await http.ExecuteNonQueryAsync($@" + CREATE TABLE {TableName} (id UInt64, name String, source String) + ENGINE = MergeTree() ORDER BY id"); + Step("http.ExecuteNonQueryAsync(\"CREATE TABLE ...\")", "table created"); + + // The native equivalent returns nothing: there is no row count on this path, only acknowledgement. + await tcp.ExecuteAsync($"ALTER TABLE {TableName} MODIFY COMMENT 'written over both transports'"); + Step("tcp.ExecuteAsync(\"ALTER TABLE ...\")", "comment set"); + + // Insert. HTTP names the table and the columns as arguments. + await http.InsertBinaryAsync( + TableName, + new[] { "id", "name", "source" }, + new List + { + new object[] { 1UL, "Ada", "http" }, + new object[] { 2UL, "Grace", "http" }, + }); + Step("http.InsertBinaryAsync(table, columns, rows)", "2 rows"); + + // The native client takes the statement instead, ending at VALUES, and the rows follow it as blocks. + await tcp.InsertRowsAsync( + $"INSERT INTO {TableName} (id, name, source) VALUES", + new List + { + new object[] { 3UL, "Alan", "tcp" }, + new object[] { 4UL, "Edsger", "tcp" }, + }); + Step("tcp.InsertRowsAsync(\"INSERT ... VALUES\", rows)", "2 rows"); + + Console.WriteLine(); + Console.WriteLine(" Reading the same four rows through each client:\n"); + Console.WriteLine(" ID Name Source read by"); + Console.WriteLine(" -- ------ ------ -------"); + + // HTTP reads through a DbDataReader, pulled row by row. + using (var reader = await http.ExecuteReaderAsync($"SELECT id, name, source FROM {TableName} ORDER BY id")) + { + while (reader.Read()) + { + Console.WriteLine($" {reader.GetFieldValue(0),2} {reader.GetString(1),-6} {reader.GetString(2),-6} ExecuteReaderAsync"); + } + } + + // The native client streams object[] rows instead. There is no DbDataReader on this transport. + await foreach (object[] row in tcp.QueryAsync($"SELECT id, name, source FROM {TableName} ORDER BY id")) + { + Console.WriteLine($" {(ulong)row[0],2} {(string)row[1],-6} {(string)row[2],-6} QueryAsync"); + } + + // One scalar call, spelled the same on both, and the same boxed CLR type comes back. + object httpCount = await http.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + object tcpCount = await tcp.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + Console.WriteLine(); + Console.WriteLine($" http.ExecuteScalarAsync(\"SELECT count() ...\") = {httpCount} ({httpCount.GetType().Name})"); + Console.WriteLine($" tcp.ExecuteScalarAsync(\"SELECT count() ...\") = {tcpCount} ({tcpCount.GetType().Name})"); + Console.WriteLine($" The two transports agree: {httpCount.Equals(tcpCount)}"); + } + + private static void ShowTheMapping() + { + Console.WriteLine("\n3. Call for call\n"); + + Map("ClickHouseClient", "ClickHouseTcpClient"); + Map("ClickHouseClientSettings", "ClickHouseTcpClientOptions (an init-only record)"); + Map("ClickHouseConnectionStringBuilder", "ClickHouseTcpConnectionStringBuilder"); + Map("ExecuteNonQueryAsync(sql) -> int", "ExecuteAsync(sql)"); + Map("ExecuteScalarAsync(sql)", "ExecuteScalarAsync(sql)"); + Map("ExecuteReaderAsync(sql) -> DbDataReader", "QueryAsync(sql) -> IAsyncEnumerable"); + Map(string.Empty, "QueryAsync(sql) -> IAsyncEnumerable"); + Map(string.Empty, "StreamAsync(sql) -> IAsyncEnumerable"); + Map("InsertBinaryAsync(table, columns, rows)", "InsertRowsAsync(\"INSERT INTO t (cols) VALUES\", rows)"); + Map("InsertBinaryAsync(table, rows)", "InsertRowsAsync(\"INSERT INTO t (cols) VALUES\", rows)"); + Map(string.Empty, "InsertAsync(sql, IColumn[]) — columnar, no per-row boxing"); + Map("PingAsync()", "PingAsync() — a protocol ping, not a SELECT 1"); + Map("QueryOptions", "ClickHouseTcpQueryOptions / ClickHouseTcpInsertOptions"); + Map("QueryOptions.CustomSettings (object values)", "Settings (string values)"); + Map("ClickHouseParameterCollection", "ClickHouseTcpParameterCollection"); + Map("@name, rewritten client-side", "{name:Type} only — nothing is rewritten"); + Map("UseSession / SessionId", "OpenSessionAsync() -> IClickHouseTcpSession"); + Map("AddClickHouseDataSource(...)", "AddClickHouseTcpDataSource(...)"); + Map("using (IDisposable)", "await using (IAsyncDisposable, and IDisposable)"); + Map("Port=8123, Protocol=https", "Port=9000, UseTls=true (9440 when Port is unset)"); + Map("Compression=true", "Compression=lz4|zstd|none"); + Map("ClickHouseConnection / ClickHouseCommand", "(nothing — see below)"); + } + + private static void ShowWhatIsMissing() + { + Console.WriteLine("\n4. What the native client does not do\n"); + Console.WriteLine(" A format other than Native. The protocol carries columnar blocks, so there is no CSV,"); + Console.WriteLine(" JSONEachRow or Parquet ingestion or export, and no raw stream insert."); + Console.WriteLine(); + Console.WriteLine(" ADO.NET, and so any ORM. There is no DbConnection over this transport, so Dapper, EF"); + Console.WriteLine(" Core and linq2db need the HTTP client."); + Console.WriteLine(); + Console.WriteLine(" JWT or bearer authentication. Username and password only, plus QuotaKey."); + Console.WriteLine(); + Console.WriteLine(" Custom HTTP headers, which have no equivalent on the wire."); + Console.WriteLine(); + Console.WriteLine(" A parameter type resolver, a parameter formatter, or a read value converter. There is no"); + Console.WriteLine(" hook for any of the three: a parameter's type comes from its {name:Type} placeholder or"); + Console.WriteLine(" from ClickHouseTcpParameter.ClickHouseType."); + Console.WriteLine(); + Console.WriteLine(" Per-query Roles or Database. Run SET ROLE inside a session for the first; qualify the"); + Console.WriteLine(" name, or use a client per database, for the second."); + + Console.WriteLine("\n5. What only the native client does\n"); + Console.WriteLine(" Blocks and typed columns, so a read can skip materializing rows at all, and a column"); + Console.WriteLine(" read out of one block re-inserts without being rebuilt."); + Console.WriteLine(" Sessions that are one pinned connection, so a temporary table or a SET survives."); + Console.WriteLine(" Progress, profile info and profile events while the query runs, through callbacks."); + Console.WriteLine(" Block compression on the wire, LZ4 by default."); + Console.WriteLine(); + Console.WriteLine(" Both lists in full: examples/Tcp/README.md"); + } + + private static void Step(string call, string result) + => Console.WriteLine($" {call,-46} {result}"); + + private static void Map(string http, string tcp) + => Console.WriteLine($" {http,-44} {tcp}"); +} diff --git a/examples/Tcp/README.md b/examples/Tcp/README.md index a086a5fa5..afdbbfff6 100644 --- a/examples/Tcp/README.md +++ b/examples/Tcp/README.md @@ -17,8 +17,11 @@ docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clic ## The API is experimental -Every public type of the native client carries `[Experimental("CHTCP0001")]`, so using one is a -compile error until you acknowledge that the surface may change in a future release: +The client, the data source, the session and the three `IClickHouseTcp*` interfaces carry +`[Experimental("CHTCP0001")]`, so touching one is a compile error until you acknowledge that the +surface may change in a future release. The types around them — the options record, the connection +string builder, `Block`, the column interfaces and the exceptions — carry nothing, so they can be +named without the suppression. ```csharp #pragma warning disable CHTCP0001 // The native protocol client's API is not yet stable. @@ -45,6 +48,13 @@ Reach for the HTTP client instead when you need: - **A parameter type resolver, a parameter formatter, or a read value converter.** The native client has no hook for any of the three; a parameter's type comes from the `{name:Type}` placeholder in the query or from `ClickHouseTcpParameter.ClickHouseType`. +- **A per-query role or database.** HTTP's `QueryOptions` carries both; `ClickHouseTcpQueryOptions` + carries only `QueryId`, `Settings`, `Parameters` and `Callbacks`. Set the database on the client, + and change roles with `SET ROLE` inside a session. + +Also worth knowing before you read a timestamp: a `DateTime`, `DateTime64` or `Time` column reaches +the row and block tiers as the integer the wire carried, not as a calendar type. `QueryAsync` into +a POCO is the only tier that converts. ## What only the native client does From f357960a705b1075d95749da82de86ed74fd14d6 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 11:44:05 +0200 Subject: [PATCH 05/16] Record that the block tier converts a temporal column IDateTimeColumn and ITimeColumn reach the timezone and scale, so the row tier is the only one that hands back the raw count. Co-Authored-By: Claude Opus 5 (1M context) --- examples/Tcp/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/Tcp/README.md b/examples/Tcp/README.md index afdbbfff6..b43a78bbc 100644 --- a/examples/Tcp/README.md +++ b/examples/Tcp/README.md @@ -52,9 +52,11 @@ Reach for the HTTP client instead when you need: carries only `QueryId`, `Settings`, `Parameters` and `Callbacks`. Set the database on the client, and change roles with `SET ROLE` inside a session. -Also worth knowing before you read a timestamp: a `DateTime`, `DateTime64` or `Time` column reaches -the row and block tiers as the integer the wire carried, not as a calendar type. `QueryAsync` into -a POCO is the only tier that converts. +Also worth knowing before you read a timestamp: a `DateTime`, `DateTime64`, `Time` or `Time64` column +reaches the **row** tier as the integer the wire carried, not as a calendar type, because that is the +value the server sent. `QueryAsync` into a POCO converts, and on the block tier the column +pattern-matches to `IDateTimeColumn` or `ITimeColumn`, which convert and report the timezone and +scale the column type declared. ## What only the native client does From 3e3471c63ad39b8f9558fc74bf8d30bd251208c9 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 12:14:39 +0200 Subject: [PATCH 06/16] Add the native protocol's read examples Four under examples/Tcp/Read/: the three read tiers with what each allocates, measured over 200,000 rows; the block tier in depth, including the borrowed-span contract and the temporal column interfaces; parameter binding with the three traps that cost real time; and POCO reads and writes over one class. Tcp_001 presented QueryAsync as the only way to get a calendar value out of a timestamp column, which the block tier's IDateTimeColumn now also does. Tcp/README.md's list of experimental types omitted the AddClickHouseTcpDataSource overloads and counted the session twice. Co-Authored-By: Claude Opus 5 (1M context) --- examples/Program.cs | 21 + examples/README.md | 7 + examples/Tcp/Core/Tcp_001_BasicUsage.cs | 5 +- examples/Tcp/README.md | 11 +- examples/Tcp/Read/Tcp_005_ReadTiers.cs | 302 ++++++++++++ examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs | 433 ++++++++++++++++++ examples/Tcp/Read/Tcp_007_Parameters.cs | 407 ++++++++++++++++ examples/Tcp/Read/Tcp_008_Poco.cs | 281 ++++++++++++ 8 files changed, 1460 insertions(+), 7 deletions(-) create mode 100644 examples/Tcp/Read/Tcp_005_ReadTiers.cs create mode 100644 examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs create mode 100644 examples/Tcp/Read/Tcp_007_Parameters.cs create mode 100644 examples/Tcp/Read/Tcp_008_Poco.cs diff --git a/examples/Program.cs b/examples/Program.cs index 813e61f1c..01dc0fbef 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -362,6 +362,27 @@ private static async Task RunAllExamples(bool isInteractive) await TcpMigratingFromHttp.Run(); WaitForUser(isInteractive); + // Native Protocol: Reading Data + Console.WriteLine("\n\n" + new string('=', 70)); + Console.WriteLine("NATIVE PROTOCOL: READING DATA"); + Console.WriteLine(new string('=', 70) + "\n"); + + Console.WriteLine($"Running: {nameof(TcpReadTiers)}"); + await TcpReadTiers.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpBlocksAndColumns)}"); + await TcpBlocksAndColumns.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpParameters)}"); + await TcpParameters.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpPoco)}"); + await TcpPoco.Run(); + WaitForUser(isInteractive); + Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ALL EXAMPLES COMPLETED SUCCESSFULLY!"); Console.WriteLine(new string('=', 70)); diff --git a/examples/README.md b/examples/README.md index 393503e7c..1905f5211 100644 --- a/examples/README.md +++ b/examples/README.md @@ -106,6 +106,13 @@ These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/READ - [Tcp_003_DependencyInjection.cs](Tcp/Core/Tcp_003_DependencyInjection.cs) - `AddClickHouseTcpDataSource`, injecting `IClickHouseTcpClient`, keyed registrations for two clusters, and who disposes the shared pool - [Tcp_004_MigratingFromHttp.cs](Tcp/Core/Tcp_004_MigratingFromHttp.cs) - The same task over both transports, the call-for-call API mapping, the `CHTCP0001` opt-in, and what the native client cannot do +### Native Protocol: Reading Data + +- [Tcp_005_ReadTiers.cs](Tcp/Read/Tcp_005_ReadTiers.cs) - The three read tiers side by side — `QueryAsync` (boxed `object[]`), `QueryAsync` (POCO, converted), `StreamAsync` (columnar blocks) — what each allocates, and which to pick +- [Tcp_006_BlocksAndColumns.cs](Tcp/Read/Tcp_006_BlocksAndColumns.cs) - The block tier in depth: `Block.ColumnNames`, the indexers, `Column`, `IColumn` metadata, `ReadOnlySpan` values, `IDateTimeColumn`/`ITimeColumn`, `IArrayColumn`, and the borrowed-lifetime contract +- [Tcp_007_Parameters.cs](Tcp/Read/Tcp_007_Parameters.cs) - `ClickHouseTcpParameterCollection` and `ClickHouseTcpQueryOptions.Parameters`, plus the three traps: `{name:Type}` is required, an instant needs a declared timezone, and a parameter named after a server setting +- [Tcp_008_Poco.cs](Tcp/Read/Tcp_008_Poco.cs) - `QueryAsync` and `InsertRowsAsync` over one class, the name-matching rules, `[ClickHouseTcpColumn]`, `[ClickHouseTcpNotMapped]`, and what a mapping mismatch reports + ## How to run ### Prerequisites diff --git a/examples/Tcp/Core/Tcp_001_BasicUsage.cs b/examples/Tcp/Core/Tcp_001_BasicUsage.cs index 667a16a8c..a26aff45d 100644 --- a/examples/Tcp/Core/Tcp_001_BasicUsage.cs +++ b/examples/Tcp/Core/Tcp_001_BasicUsage.cs @@ -114,7 +114,8 @@ private static void ShowTheReadTiers() Console.WriteLine(" StreamAsync whole Blocks, typed columns, no per-row boxing"); Console.WriteLine(); Console.WriteLine("The row tier boxes the value the wire carried, which is not always the CLR type the column"); - Console.WriteLine("name suggests: a DateTime column arrives as uint epoch seconds. Read date and time columns"); - Console.WriteLine("through QueryAsync into a DateTime or DateTimeOffset property."); + Console.WriteLine("name suggests: a DateTime column arrives as uint epoch seconds. For a calendar value, read"); + Console.WriteLine("it through QueryAsync into a DateTime or DateTimeOffset property, or on the block tier"); + Console.WriteLine("match the column to IDateTimeColumn (ITimeColumn for Time) and call GetDateTimeOffset."); } } diff --git a/examples/Tcp/README.md b/examples/Tcp/README.md index b43a78bbc..fd4a0b270 100644 --- a/examples/Tcp/README.md +++ b/examples/Tcp/README.md @@ -17,11 +17,12 @@ docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clic ## The API is experimental -The client, the data source, the session and the three `IClickHouseTcp*` interfaces carry -`[Experimental("CHTCP0001")]`, so touching one is a compile error until you acknowledge that the -surface may change in a future release. The types around them — the options record, the connection -string builder, `Block`, the column interfaces and the exceptions — carry nothing, so they can be -named without the suppression. +`ClickHouseTcpClient`, `ClickHouseTcpDataSource`, the three `IClickHouseTcp*` interfaces +(`IClickHouseTcpClient`, `IClickHouseTcpOperations`, `IClickHouseTcpSession`) and the +`AddClickHouseTcpDataSource` overloads carry `[Experimental("CHTCP0001")]`, so touching one is a +compile error until you acknowledge that the surface may change in a future release. The types around +them — the options record, the connection string builder, `Block`, the column interfaces and the +exceptions — carry nothing, so they can be named without the suppression. ```csharp #pragma warning disable CHTCP0001 // The native protocol client's API is not yet stable. diff --git a/examples/Tcp/Read/Tcp_005_ReadTiers.cs b/examples/Tcp/Read/Tcp_005_ReadTiers.cs new file mode 100644 index 000000000..cf4bc512e --- /dev/null +++ b/examples/Tcp/Read/Tcp_005_ReadTiers.cs @@ -0,0 +1,302 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// The three ways the native client reads a result, run one after another over the same query: QueryAsync +/// (one object[] per row, every value boxed), QueryAsync<T> (one POCO per row, values +/// converted) and StreamAsync (whole s, typed columns, no per-row work at all). +/// +/// +/// The tiers are not three spellings of one thing. They differ in what they allocate, and they differ in what a +/// timestamp looks like when it arrives — the row tier hands back the integer the wire carried, while the other +/// two convert it. Section 4 measures the first difference and section 1 shows the second. +/// +/// +public static class TcpReadTiers +{ + private const string TableName = "example_tcp_read_tiers"; + + // Rows for the allocation measurement in section 4. Large enough that the tiers separate, small enough that + // the whole example stays under a second. + private const int MeasuredRows = 200_000; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await Seed(client); + await RowTier(client); + await PocoTier(client); + await BlockTier(client); + await WhatEachCosts(client); + ShowTheChoice(); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped '{TableName}'"); + } + } + + private static async Task Seed(ClickHouseTcpClient client) + { + // recorded_at declares its timezone. A bare DateTime would take the server's, which is what the block + // tier reports as the column's TimeZone; naming UTC makes this example's output the same everywhere. + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + id UInt64, + city String, + temperature Float64, + recorded_at DateTime('UTC') + ) + ENGINE = MergeTree() + ORDER BY id"); + + var midnight = new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc); + + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, city, temperature, recorded_at) VALUES", + new List + { + new object[] { 1UL, "Amsterdam", 17.5, midnight.AddHours(6) }, + new object[] { 2UL, "Amsterdam", 21.0, midnight.AddHours(14) }, + new object[] { 3UL, "Reykjavik", 9.5, midnight.AddHours(6) }, + new object[] { 4UL, "Reykjavik", 11.25, midnight.AddHours(14) }, + new object[] { 5UL, "Singapore", 28.0, midnight.AddHours(6) }, + new object[] { 6UL, "Singapore", 31.75, midnight.AddHours(14) }, + }); + + Console.WriteLine($"Seeded '{TableName}' with 6 rows (id UInt64, city String, temperature Float64, recorded_at DateTime('UTC'))"); + } + + private static string Sql => $"SELECT id, city, temperature, recorded_at FROM {TableName} ORDER BY id"; + + private static async Task RowTier(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. QueryAsync — one object[] per row, every value boxed\n"); + Console.WriteLine(" ID City Temp recorded_at CLR types"); + Console.WriteLine(" -- --------- ----- ----------- ---------"); + + object[]? first = null; + + // Values arrive in the order the SELECT names them; there are no names on this tier. Each array is yours + // to keep, so collecting rows into a list is safe. + await foreach (object[] row in client.QueryAsync(Sql)) + { + first ??= row; + Console.WriteLine( + $" {(ulong)row[0],2} {(string)row[1],-9} {(double)row[2],5} {row[3],11} " + + string.Join(", ", row.Select(v => v.GetType().Name))); + } + + Console.WriteLine(); + Console.WriteLine(" The last column is the trap. recorded_at is a DateTime('UTC'), but a DateTime column"); + Console.WriteLine(" is stored as a count of epoch seconds and that count is what the box holds:"); + + // Reading a calendar value off this tier means converting the count by hand, which needs the timezone the + // column declared — and nothing on this tier reports it. The other two tiers do the conversion for you. + uint seconds = (uint)first![3]; + Console.WriteLine($" row[3] is {first[3].GetType().Name} = {seconds}"); + + try + { + _ = (DateTime)first[3]; + } + catch (InvalidCastException ex) + { + Console.WriteLine($" (DateTime)row[3] throws: {ex.Message}"); + } + + Console.WriteLine($" converted by hand: {DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime:yyyy-MM-dd HH:mm:ss} UTC"); + Console.WriteLine(" Date, DateTime64, Time and Time64 behave the same way. Read them through one of the"); + Console.WriteLine(" next two tiers."); + } + + private static async Task PocoTier(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. QueryAsync — one POCO per row, values converted\n"); + Console.WriteLine(" Each column fills the property of the same name (case- and underscore-insensitively,"); + Console.WriteLine(" so recorded_at reaches RecordedAt), converting to the property's type on the way:\n"); + Console.WriteLine(" ID City Temp RecordedAt (DateTime) Kind"); + Console.WriteLine(" -- --------- ----- --------------------- ----"); + + await foreach (Reading reading in client.QueryAsync(Sql)) + { + Console.WriteLine( + $" {reading.Id,2} {reading.City,-9} {reading.Temperature,5} " + + $"{reading.RecordedAt.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture),-21} {reading.RecordedAt.Kind}"); + } + + Console.WriteLine(); + Console.WriteLine(" Kind is Utc because the column declares UTC. A column in a zone with an offset yields"); + Console.WriteLine(" Kind=Unspecified — the wall-clock reading in that zone — so declare the property as a"); + Console.WriteLine(" DateTimeOffset when the offset matters."); + Console.WriteLine(); + Console.WriteLine(" A column no property maps to is skipped, and a property no column maps to keeps its"); + Console.WriteLine(" default. Tcp_008_Poco covers the mapping rules and the insert direction."); + } + + private static async Task BlockTier(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. StreamAsync — whole blocks, typed columns, nothing boxed\n"); + + await foreach (Block block in client.StreamAsync(Sql)) + { + Console.WriteLine($" Block of {block.RowCount} rows x {block.ColumnCount} columns: {string.Join(", ", block.ColumnNames)}"); + + // Bound once, outside any row loop: a name lookup is a scan of the block's columns. + IColumn temperature = block.Column("temperature"); + IColumn ids = block.Column("id"); + + // A span over the block's own buffer. Read into a local and iterate that; the property recomputes the + // span on every access, and it cannot be cached in a field because it is a ref struct. + ReadOnlySpan values = temperature.Values; + double total = 0; + foreach (double value in values) + { + total += value; + } + + Console.WriteLine($" temperature is {temperature.TypeName} -> ReadOnlySpan<{temperature.ElementType.Name}>, mean {total / values.Length:0.###}"); + Console.WriteLine($" id is {ids.TypeName} -> ReadOnlySpan<{ids.ElementType.Name}>, {ids.RowCount} values, first {ids[0]}"); + + // The typed view of a DateTime column is IColumn — the same count the row tier boxed. The + // calendar reading lives on IDateTimeColumn, which the column also implements. + IColumn recordedAt = block["recorded_at"]; + Console.WriteLine($" recorded_at is {recordedAt.TypeName} -> ReadOnlySpan<{recordedAt.ElementType.Name}>, the raw epoch seconds"); + + if (recordedAt is IDateTimeColumn instants) + { + Console.WriteLine($" ... and it pattern-matches to IDateTimeColumn: TimeZone {instants.TimeZone.Id}, Scale {instants.Scale}"); + Console.WriteLine($" GetDateTimeOffset(0) = {instants.GetDateTimeOffset(0).ToString("yyyy-MM-dd HH:mm:ss zzz", CultureInfo.InvariantCulture)}"); + + // ToDateTimeOffsets allocates, and the array it returns is the caller's: unlike Values it stays + // valid after the block is released. + DateTimeOffset[] all = instants.ToDateTimeOffsets(); + Console.WriteLine($" ToDateTimeOffsets() = {all.Length} instants, last {all[^1].ToString("HH:mm:ss zzz", CultureInfo.InvariantCulture)}"); + } + } + + Console.WriteLine(); + Console.WriteLine(" A yielded block is borrowed: it is released when the loop advances. Copy out what has"); + Console.WriteLine(" to outlive the iteration. Tcp_006_BlocksAndColumns is the whole contract."); + } + + private static async Task WhatEachCosts(ClickHouseTcpClient client) + { + Console.WriteLine($"\n4. What each costs, summing one Float64 column over {MeasuredRows:N0} rows\n"); + + // Two numeric columns and no strings, so the measurement is the tier's own overhead rather than the cost + // of materializing values every tier has to materialize anyway. + string sql = $"SELECT number AS id, number * 0.5 AS temperature FROM system.numbers LIMIT {MeasuredRows}"; + + // Warm up: the first read of a result compiles the POCO plan and grows the pooled buffers, and charging + // that to whichever tier ran first would be the whole difference at this size. + await SumWithRows(client, sql); + await SumWithPoco(client, sql); + await SumWithBlocks(client, sql); + + await Measure("QueryAsync object[] per row, 2 boxes per row", () => SumWithRows(client, sql)); + await Measure("QueryAsync one POCO per row, no boxing", () => SumWithPoco(client, sql)); + await Measure("StreamAsync spans over the block's buffers", () => SumWithBlocks(client, sql)); + + Console.WriteLine(); + Console.WriteLine(" Allocation is measured process-wide (GC.GetTotalAllocatedBytes), so it includes the"); + Console.WriteLine(" client's own read buffers — which is why the block tier is not zero rather than why it"); + Console.WriteLine(" is small. Absolute numbers move with the machine; the ratio is the point."); + Console.WriteLine(); + Console.WriteLine(" A String column narrows the gap, because every tier materializes one string per value."); + } + + private static async Task Measure(string label, Func> read) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + + long before = GC.GetTotalAllocatedBytes(precise: true); + var started = System.Diagnostics.Stopwatch.StartNew(); + double sum = await read(); + started.Stop(); + long allocated = GC.GetTotalAllocatedBytes(precise: true) - before; + + Console.WriteLine($" {label,-52} {allocated / 1024.0 / 1024.0,7:0.00} MB {started.ElapsedMilliseconds,4} ms (sum {sum:0})"); + } + + private static async Task SumWithRows(ClickHouseTcpClient client, string sql) + { + double sum = 0; + await foreach (object[] row in client.QueryAsync(sql)) + { + sum += (double)row[1]; + } + + return sum; + } + + private static async Task SumWithPoco(ClickHouseTcpClient client, string sql) + { + double sum = 0; + await foreach (Reading reading in client.QueryAsync(sql)) + { + sum += reading.Temperature; + } + + return sum; + } + + private static async Task SumWithBlocks(ClickHouseTcpClient client, string sql) + { + double sum = 0; + await foreach (Block block in client.StreamAsync(sql)) + { + ReadOnlySpan values = block.Column("temperature").Values; + for (int i = 0; i < values.Length; i++) + { + sum += values[i]; + } + } + + return sum; + } + + private static void ShowTheChoice() + { + Console.WriteLine("\n5. Which to pick\n"); + Console.WriteLine(" QueryAsync A result whose shape you do not know at compile time, or a few rows"); + Console.WriteLine(" where the boxing does not matter. No names — pair it with"); + Console.WriteLine(" Block.ColumnNames if you need them. Date and time columns arrive raw."); + Console.WriteLine(); + Console.WriteLine(" QueryAsync The default for application code. One object per row instead of an"); + Console.WriteLine(" array plus a box per value, values converted to the property's type,"); + Console.WriteLine(" and each row owns its values, so a row can be kept or returned."); + Console.WriteLine(); + Console.WriteLine(" StreamAsync Aggregating, scanning, or handing a column to something that wants a"); + Console.WriteLine(" span. Nothing is materialized per row, and a column read out of one"); + Console.WriteLine(" block re-inserts without being rebuilt. The cost is the borrowing"); + Console.WriteLine(" contract: nothing may outlive the iteration unless you copy it."); + Console.WriteLine(); + Console.WriteLine(" All three hold a connection until the enumeration ends, so read to the end (or stop"); + Console.WriteLine(" with a break, which tells the server the result is abandoned and drops the connection"); + Console.WriteLine(" rather than returning it to the pool)."); + } + + // One property per column. String is initialized because the project enables nullable reference types and + // the materializer assigns every mapped property anyway. + private sealed class Reading + { + public ulong Id { get; set; } + + public string City { get; set; } = string.Empty; + + public double Temperature { get; set; } + + // The DateTime('UTC') column's epoch-second count, converted with the column's timezone. Declare this as + // a DateTimeOffset instead to keep the offset. + public DateTime RecordedAt { get; set; } + } +} diff --git a/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs b/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs new file mode 100644 index 000000000..cd1baf856 --- /dev/null +++ b/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs @@ -0,0 +1,433 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// The block tier in depth: what StreamAsync yields, how a is addressed +/// (ColumnNames, the two indexers, TryGetColumn, the typed Column<T>), what an +/// reports about itself, and how its values are read as a . +/// +/// +/// Section 7 is the part to read twice. A yielded block is borrowed: its storage is returned to a pool +/// when the iteration moves on, so a column, a span, or the block itself is invalid the moment the loop +/// advances. Everything else here is convenience; this one is correctness. +/// +/// +public static class TcpBlocksAndColumns +{ + private const string TableName = "example_tcp_blocks_and_columns"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await Seed(client); + await OneResultManyBlocks(client); + await AddressingAColumn(client); + await WhatAColumnReports(client); + await ValuesAsSpans(client); + await DateAndTimeColumns(client); + await ArrayColumns(client); + await TheBorrowingContract(client); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped '{TableName}'"); + } + } + + private static async Task Seed(ClickHouseTcpClient client) + { + // captured_at names a zone with an offset and a daylight-saving rule, so the block tier has something to + // report; uptime is a Time, which is a count from midnight and has no zone at all. + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + id UInt64, + sensor String, + voltage Float64, + readings Array(Float64), + captured_at DateTime64(3, 'Europe/Amsterdam'), + uptime Time + ) + ENGINE = MergeTree() + ORDER BY id"); + + var baseline = new DateTime(2026, 6, 1, 10, 0, 0, DateTimeKind.Utc); + + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, sensor, voltage, readings, captured_at, uptime) VALUES", + new List + { + new object[] { 1UL, "north", 3.31, new[] { 0.5, 0.75, 1.0 }, baseline.AddMilliseconds(125), TimeSpan.FromMinutes(90) }, + new object[] { 2UL, "north", 3.28, new[] { 1.25, 1.5 }, baseline.AddMilliseconds(250), TimeSpan.FromMinutes(150) }, + new object[] { 3UL, "south", 3.35, Array.Empty(), baseline.AddMilliseconds(375), TimeSpan.FromMinutes(210) }, + new object[] { 4UL, "south", 3.30, new[] { 2.0 }, baseline.AddMilliseconds(500), TimeSpan.FromMinutes(270) }, + new object[] { 5UL, "west", 3.22, new[] { 2.25, 2.5, 2.75, 3.0 }, baseline.AddMilliseconds(625), TimeSpan.FromMinutes(330) }, + new object[] { 6UL, "west", 3.40, new[] { 3.25 }, baseline.AddMilliseconds(750), TimeSpan.FromMinutes(390) }, + }); + + Console.WriteLine($"Seeded '{TableName}' with 6 rows:"); + Console.WriteLine(" id UInt64, sensor String, voltage Float64, readings Array(Float64),"); + Console.WriteLine(" captured_at DateTime64(3, 'Europe/Amsterdam'), uptime Time"); + } + + private static async Task OneResultManyBlocks(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. One result is a sequence of blocks\n"); + Console.WriteLine(" How many, and how tall, is the server's decision. This table's six rows fit in one"); + Console.WriteLine(" granule, so they arrive together:\n"); + Console.WriteLine(" Block Rows Columns Name"); + Console.WriteLine(" ----- ---- ------- ----"); + await ShowShapes(client, $"SELECT id, sensor FROM {TableName}", null); + + // A generator does honour max_block_size row for row, which a six-row MergeTree read does not: the part is + // read whole and the setting only caps it. + var capped = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["max_block_size"] = "3" }, + }; + + Console.WriteLine("\n The same shape of query over a generator, with max_block_size = 3, splits:\n"); + Console.WriteLine(" Block Rows Columns Name"); + Console.WriteLine(" ----- ---- ------- ----"); + await ShowShapes(client, "SELECT number, toString(number) AS text FROM system.numbers LIMIT 8", capped); + + Console.WriteLine(); + Console.WriteLine(" A result block carries no name. A named block is how the server labels the extras a"); + Console.WriteLine(" query can produce — WITH TOTALS, extremes — which reach a caller through"); + Console.WriteLine(" ClickHouseTcpQueryOptions.Callbacks rather than through this stream."); + Console.WriteLine(); + Console.WriteLine(" So write the loop for any number of blocks of any height, and never for one."); + } + + private static async Task ShowShapes(ClickHouseTcpClient client, string sql, ClickHouseTcpQueryOptions? options) + { + int index = 0; + await foreach (Block block in client.StreamAsync(sql, options)) + { + Console.WriteLine($" {++index,5} {block.RowCount,4} {block.ColumnCount,7} {(block.Name.Length == 0 ? "(empty)" : block.Name)}"); + } + } + + private static async Task AddressingAColumn(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. Addressing a column\n"); + + await foreach (Block block in client.StreamAsync(Sql)) + { + // Owned, unlike the columns: computed once, cached, and safe to keep after the block is released. + Console.WriteLine($" block.ColumnNames = [{string.Join(", ", block.ColumnNames)}]"); + Console.WriteLine($" block.ColumnCount = {block.ColumnCount}, block.RowCount = {block.RowCount}"); + Console.WriteLine($" block[0] = '{block[0].Name}' by position"); + Console.WriteLine($" block[\"sensor\"] = '{block["sensor"].Name}' by name — ordinal and case-sensitive, like ClickHouse itself"); + + // The name lookup is a scan of the block's columns, so bind a column once and then loop over rows, + // never the other way round. + Console.WriteLine($" TryGetColumn(\"sensor\", out _) = {block.TryGetColumn("sensor", out _)}"); + Console.WriteLine($" TryGetColumn(\"Sensor\", out _) = {block.TryGetColumn("Sensor", out _)} (capital S is a different name)"); + + try + { + _ = block["missing"]; + } + catch (ArgumentException ex) + { + Console.WriteLine($" block[\"missing\"] throws: {ex.Message.Split(" (Parameter")[0]}"); + } + + // The typed overload is the same lookup plus a cast to IColumn, which is where the values live. + IColumn voltage = block.Column("voltage"); + Console.WriteLine($" block.Column(\"voltage\") = IColumn over '{voltage.TypeName}'"); + + try + { + _ = block.Column("captured_at"); + } + catch (InvalidCastException ex) + { + Console.WriteLine($" block.Column(\"captured_at\") throws: {ex.Message}"); + Console.WriteLine(" T must be the type the column's values are stored as, not the type you want them in."); + } + + // The first block is enough for the sections that follow. Breaking out is allowed: the client tells the + // server the result is abandoned, and drops that connection instead of returning it to the pool. + break; + } + } + + private static async Task WhatAColumnReports(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. What a column reports about itself\n"); + Console.WriteLine(" Name TypeName ElementType Rows Extra interface"); + Console.WriteLine(" ----------- ----------------------------------- ----------- ---- ---------------"); + + await foreach (Block block in client.StreamAsync(Sql)) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine( + $" {column.Name,-11} {column.TypeName,-35} {Describe(column.ElementType),-11} {column.RowCount,4} {ExtraInterface(column)}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" TypeName is the header text the server sent, so it is the type as ClickHouse spells it."); + Console.WriteLine(" ElementType is the T of the column's IColumn — what Values hands back. Where those"); + Console.WriteLine(" two disagree in kind, a second interface bridges them (sections 5 and 6)."); + } + + private static async Task ValuesAsSpans(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. Values, as a span over the server's own layout\n"); + + await foreach (Block block in client.StreamAsync(Sql)) + { + IColumn ids = block.Column("id"); + IColumn voltage = block.Column("voltage"); + IColumn sensor = block.Column("sensor"); + + // Read the span into a local: the property recomputes it on every access, and being a ref struct it + // cannot be stored in a field. Do not let it escape this iteration. + ReadOnlySpan volts = voltage.Values; + + double min = double.MaxValue; + double max = double.MinValue; + for (int i = 0; i < volts.Length; i++) + { + min = Math.Min(min, volts[i]); + max = Math.Max(max, volts[i]); + } + + Console.WriteLine($" voltage.Values is ReadOnlySpan of {volts.Length}: min {min}, max {max} — no allocation, no boxing"); + Console.WriteLine($" id.Values is ReadOnlySpan of {ids.Values.Length}: {string.Join(", ", ids.Values.ToArray())}"); + Console.WriteLine($" voltage[2] = {voltage[2]} (the indexer, for one value)"); + Console.WriteLine($" block[\"voltage\"].GetValue(2) = {block["voltage"].GetValue(2)} boxed — the untyped escape hatch"); + Console.WriteLine(); + Console.WriteLine($" A String column is a span too, of references: sensor.Values = [{string.Join(", ", sensor.Values.ToArray())}]"); + Console.WriteLine(" Reading it decodes one string per value, so the block tier saves less on String than"); + Console.WriteLine(" on a fixed-width type. It still saves the object[] and the boxes."); + + break; + } + } + + private static async Task DateAndTimeColumns(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. Date and time columns: a count, plus an interface that reads it\n"); + Console.WriteLine(" These types are stored as a plain integer, so IColumn hands back that integer — the"); + Console.WriteLine(" layout the wire carried, at no conversion cost. Turning it into a calendar value needs"); + Console.WriteLine(" the column's timezone and scale, which only these two interfaces report.\n"); + + await foreach (Block block in client.StreamAsync(Sql)) + { + IColumn capturedAt = block["captured_at"]; + Console.WriteLine($" captured_at {capturedAt.TypeName}"); + Console.WriteLine($" as IColumn: {string.Join(", ", block.Column("captured_at").Values[..3].ToArray())}, ... (milliseconds since the epoch)"); + + if (capturedAt is IDateTimeColumn instants) + { + Console.WriteLine($" as IDateTimeColumn: TimeZone {instants.TimeZone.Id}, Scale {instants.Scale}"); + Console.WriteLine($" GetDateTimeOffset(0) = {Format(instants.GetDateTimeOffset(0))}"); + Console.WriteLine($" GetDateTimeOffset(5) = {Format(instants.GetDateTimeOffset(5))}"); + + // Allocates one array, and that array is the caller's: it stays valid after the block is gone. + DateTimeOffset[] all = instants.ToDateTimeOffsets(); + Console.WriteLine($" ToDateTimeOffsets() = {all.Length} instants, and the array outlives the block"); + Console.WriteLine(" The +02:00 offset is the zone's, in June. The same column read in January"); + Console.WriteLine(" would report +01:00, which is why the timezone and not a fixed offset is what"); + Console.WriteLine(" the interface exposes."); + } + + IColumn uptime = block["uptime"]; + Console.WriteLine($" uptime {uptime.TypeName}"); + Console.WriteLine($" as IColumn: {string.Join(", ", block.Column("uptime").Values[..3].ToArray())}, ... (seconds from midnight)"); + + if (uptime is ITimeColumn times) + { + Console.WriteLine($" as ITimeColumn: Scale {times.Scale}, no timezone — a Time is a time of day, not an instant"); + Console.WriteLine($" GetTimeSpan(0) = {times.GetTimeSpan(0)}"); + Console.WriteLine($" ToTimeSpans() = {string.Join(", ", times.ToTimeSpans().Take(3))}, ... (also caller-owned)"); + Console.WriteLine(" The count is signed and is not clamped to one day, so a TimeSpan here can be"); + Console.WriteLine(" negative or longer than 24 hours."); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" Pattern-match rather than test the type name: DateTime and DateTime64 both give an"); + Console.WriteLine(" IDateTimeColumn, Time and Time64 both give an ITimeColumn, and neither interface is"); + Console.WriteLine(" generic, so one branch handles both widths."); + Console.WriteLine(); + Console.WriteLine(" A Nullable(DateTime) does not match, though — the wrapper is a column in its own right"); + Console.WriteLine(" and it is the wrapped column that reads the calendar. Go through INullableColumn:\n"); + + // The inner column holds one entry per row, with a placeholder where the row is null, so the null map and + // the inner column are indexed by the same row number. + await foreach (Block block in client.StreamAsync( + "SELECT if(number = 1, NULL, toDateTime(1780308000 + number, 'UTC')) AS maybe_at FROM system.numbers LIMIT 3")) + { + IColumn maybeAt = block["maybe_at"]; + Console.WriteLine($" {maybeAt.TypeName}, ElementType {Describe(maybeAt.ElementType)}"); + Console.WriteLine($" is IDateTimeColumn: {maybeAt is IDateTimeColumn,-5} (the Nullable wrapper itself)"); + + // INullableColumn because a DateTime is stored as uint: the wrapper's T is the inner storage + // type, so reaching Inner means knowing that type. + if (maybeAt is INullableColumn nullable && nullable.Inner is IDateTimeColumn inner) + { + Console.WriteLine($" is IDateTimeColumn: {true,-5} (INullableColumn.Inner)"); + ReadOnlySpan nulls = nullable.NullMap; + for (int row = 0; row < maybeAt.RowCount; row++) + { + string reading = nulls[row] != 0 ? "NULL" : Format(inner.GetDateTimeOffset(row)); + Console.WriteLine($" row {row}: NullMap {nulls[row]} -> {reading}"); + } + } + } + } + + private static async Task ArrayColumns(ClickHouseTcpClient client) + { + Console.WriteLine("\n6. An Array(T) column has two views, and they cost different things\n"); + + await foreach (Block block in client.StreamAsync(Sql)) + { + IColumn readings = block["readings"]; + Console.WriteLine($" readings is {readings.TypeName}, ElementType {Describe(readings.ElementType)}"); + + if (readings is IArrayColumn arrays) + { + // The wire layout: every row's elements end to end, plus one offset per row boundary. Both spans + // are borrowed, and this is the view that costs nothing to produce. + ReadOnlySpan flat = arrays.InnerValues; + ReadOnlySpan offsets = arrays.Offsets; + + Console.WriteLine(); + Console.WriteLine($" Borrowed view — InnerValues + Offsets, no allocation at all:"); + Console.WriteLine($" InnerValues ({flat.Length} elements) = {string.Join(", ", flat.ToArray())}"); + Console.WriteLine($" Offsets ({offsets.Length} entries, one more than the rows) = {string.Join(", ", offsets.ToArray())}"); + Console.WriteLine(" Row i is InnerValues.Slice(Offsets[i], Offsets[i + 1] - Offsets[i]):"); + + for (int row = 0; row < readings.RowCount; row++) + { + ReadOnlySpan slice = flat.Slice(offsets[row], offsets[row + 1] - offsets[row]); + double sum = 0; + foreach (double value in slice) + { + sum += value; + } + + Console.WriteLine($" row {row}: {slice.Length} element(s), sum {sum}"); + } + + Console.WriteLine(); + Console.WriteLine($" Inner is that same flat run as a column rather than a span — IColumn<{Describe(arrays.Inner.ElementType)}> here."); + Console.WriteLine(" Use it for an Array(Tuple(...)) or an Array(Array(T)), where the inner column"); + Console.WriteLine(" pattern-matches to ITupleColumn or IArrayColumn in turn, so a nested composite"); + Console.WriteLine(" can be walked all the way down without materializing a level."); + } + + // The other view. Each row is copied into a fresh double[], so these arrays are the caller's and stay + // valid after the block is released — at one allocation per row. + Console.WriteLine(); + Console.WriteLine(" Allocating view — Values and the indexer materialize one double[] per row:"); + IColumn rows = block.Column("readings"); + Console.WriteLine($" rows[4] = [{string.Join(", ", rows[4])}] (the indexer: one double[], allocated here)"); + Console.WriteLine($" Values[0] = [{string.Join(", ", rows.Values[0])}] (Values: every row's array, built at once)"); + Console.WriteLine(" Those arrays outlive the block. The span holding them does not, being a span."); + Console.WriteLine(" So prefer the indexer when only a few rows out of a tall block are wanted."); + + break; + } + + Console.WriteLine(); + Console.WriteLine(" The same split runs through the other composites: Map, Tuple, Nested, Nullable and"); + Console.WriteLine(" LowCardinality each expose a borrowed columnar view plus a materializing one."); + } + + private static async Task TheBorrowingContract(ClickHouseTcpClient client) + { + Console.WriteLine("\n7. The borrowing contract\n"); + Console.WriteLine(" Valid only for the current iteration — released when the loop advances, you stop"); + Console.WriteLine(" enumerating, or the enumerator is disposed:"); + Console.WriteLine(" the Block, every IColumn on it, IColumn.Values,"); + Console.WriteLine(" IArrayColumn.InnerValues / Offsets / Inner, INullableColumn.NullMap / Inner,"); + Console.WriteLine(" and the other composites' views."); + Console.WriteLine(); + Console.WriteLine(" Yours to keep:"); + Console.WriteLine(" Block.ColumnNames, a string or a struct value you read out,"); + Console.WriteLine(" the per-row arrays from an Array(T) column's Values or indexer,"); + Console.WriteLine(" IDateTimeColumn.ToDateTimeOffsets() and ITimeColumn.ToTimeSpans(),"); + Console.WriteLine(" and anything you copy: Values.ToArray(), a slice's ToArray()."); + Console.WriteLine(); + Console.WriteLine(" Do not dispose a yielded block. Block is IDisposable because the reader that produced"); + Console.WriteLine(" it disposes it; doing so yourself returns pooled storage the reader still manages."); + Console.WriteLine(); + Console.WriteLine(" So the shape of a correct loop is: read, aggregate, or copy — inside the body.\n"); + + // The aggregate is a value type, and the copies are arrays of our own, so both are safe to use after the + // enumeration has finished and every block has been released. + long rowsSeen = 0; + double voltageTotal = 0; + var strongestSensor = string.Empty; + double strongest = double.MinValue; + double[]? firstRowReadings = null; + + await foreach (Block block in client.StreamAsync(Sql)) + { + IColumn sensors = block.Column("sensor"); + ReadOnlySpan volts = block.Column("voltage").Values; + + for (int row = 0; row < block.RowCount; row++) + { + rowsSeen++; + voltageTotal += volts[row]; + if (volts[row] > strongest) + { + strongest = volts[row]; + + // A string read out of the block is a reference to an object the block does not own, so + // holding it is fine. A span is not. + strongestSensor = sensors[row]; + } + } + + // ToArray inside the loop is the copy. Taking the span out of the loop instead would be reading + // storage the next iteration has already handed back to the pool. + firstRowReadings ??= block.Column("readings")[0].ToArray(); + } + + Console.WriteLine($" After the loop, from copies only: {rowsSeen} rows, mean voltage {voltageTotal / rowsSeen:0.####},"); + Console.WriteLine($" highest on sensor '{strongestSensor}' at {strongest}, first row's readings [{string.Join(", ", firstRowReadings!)}]"); + } + + private static string Sql + => $"SELECT id, sensor, voltage, readings, captured_at, uptime FROM {TableName} ORDER BY id"; + + private static string Format(DateTimeOffset value) + => value.ToString("yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture); + + private static string Describe(Type type) => type switch + { + _ when type == typeof(double[]) => "double[]", + _ when type == typeof(uint?) => "uint?", + _ => type.Name, + }; + + // Which of the block tier's extra read surfaces a column offers, found by pattern-matching rather than by + // reading TypeName. + private static string ExtraInterface(IColumn column) => column switch + { + IDateTimeColumn => "IDateTimeColumn", + ITimeColumn => "ITimeColumn", + IArrayColumn => "IArrayColumn", + _ => "-", + }; +} diff --git a/examples/Tcp/Read/Tcp_007_Parameters.cs b/examples/Tcp/Read/Tcp_007_Parameters.cs new file mode 100644 index 000000000..0b039fbf5 --- /dev/null +++ b/examples/Tcp/Read/Tcp_007_Parameters.cs @@ -0,0 +1,407 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// Binding values into a query with and +/// ClickHouseTcpQueryOptions.Parameters — and the three ways it goes wrong, which are worth more of your +/// attention than the happy path. +/// +/// +/// The query text must carry each parameter's type ({id:Int32}): there is no @name rewriting on this +/// transport. A value that names an instant is refused unless the placeholder declares a timezone. And a +/// parameter named after a server setting is applied as that setting rather than bound, which fails with an error +/// that names neither. +/// +/// +public static class TcpParameters +{ + private const string TableName = "example_tcp_parameters"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await Seed(client); + await BindingValues(client); + await TheCollection(client); + await NoAtNameRewriting(client); + await Identifiers(client); + await InstantsNeedATimezone(client); + await NamesThatCollideWithSettings(client); + ShowWhatIsAbsent(); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped '{TableName}'"); + } + } + + private static async Task Seed(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + id UInt64, + city String, + temperature Float64, + recorded_at DateTime('UTC') + ) + ENGINE = MergeTree() + ORDER BY id"); + + var midnight = new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc); + + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, city, temperature, recorded_at) VALUES", + new List + { + new object[] { 1UL, "Amsterdam", 17.5, midnight.AddHours(6) }, + new object[] { 2UL, "Amsterdam", 21.0, midnight.AddHours(14) }, + new object[] { 3UL, "Reykjavik", 9.5, midnight.AddHours(6) }, + new object[] { 4UL, "Reykjavik", 11.25, midnight.AddHours(14) }, + new object[] { 5UL, "Singapore", 28.0, midnight.AddHours(6) }, + new object[] { 6UL, "Singapore", 31.75, midnight.AddHours(14) }, + }); + + Console.WriteLine($"Seeded '{TableName}' with 6 rows (id, city, temperature, recorded_at DateTime('UTC'))"); + + // Parameters travel in the Query packet's settings list, which is why they need a protocol revision that + // knows about them. An older server rejects the query rather than run it unparameterized. + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + Console.WriteLine($"Server protocol revision {server.ProtocolRevision}; parameters need 54459 or above"); + } + + private static async Task BindingValues(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. Binding values\n"); + + // Names, not positions. The collection keeps insertion order, but the query refers to each by name. + var parameters = new ClickHouseTcpParameterCollection(); + parameters.Add("city", "Amsterdam"); + parameters.Add("floor", 18.0); + parameters.Add("wanted", new[] { 1UL, 2UL, 5UL }); + + // Every placeholder states its type. That is what the server parses the value as, and what the client + // formats it as, so the two cannot disagree. + string sql = $@" + SELECT id, city, temperature + FROM {TableName} + WHERE (city = {{city:String}} OR temperature >= {{floor:Float64}}) + AND id IN {{wanted:Array(UInt64)}} + ORDER BY id"; + + Console.WriteLine(" SELECT ... WHERE (city = {city:String} OR temperature >= {floor:Float64})"); + Console.WriteLine(" AND id IN {wanted:Array(UInt64)}"); + Console.WriteLine(" city='Amsterdam', floor=18.0, wanted=[1, 2, 5]\n"); + + await foreach (object[] row in client.QueryAsync(sql, new ClickHouseTcpQueryOptions { Parameters = parameters })) + { + Console.WriteLine($" id {row[0],2} {(string)row[1],-9} {row[2]}"); + } + + Console.WriteLine(); + Console.WriteLine(" A collection works on any operation that takes ClickHouseTcpQueryOptions, so the same"); + Console.WriteLine(" parameters bind on ExecuteAsync, ExecuteScalarAsync, QueryAsync, StreamAsync and"); + Console.WriteLine(" InsertAsync — an INSERT ... SELECT can be parameterized too."); + + object count = await client.ExecuteScalarAsync( + $"SELECT count() FROM {TableName} WHERE city = {{city:String}}", + new ClickHouseTcpQueryOptions { Parameters = new ClickHouseTcpParameterCollection { { "city", "Reykjavik" } } }); + Console.WriteLine($" ExecuteScalarAsync(count() WHERE city = {{city:String}}) with city='Reykjavik' = {count}"); + } + + private static async Task TheCollection(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. The collection itself\n"); + + var parameters = new ClickHouseTcpParameterCollection + { + { "city", "Singapore" }, + { "floor", 30.0 }, + }; + + Console.WriteLine($" Count {parameters.Count}"); + Console.WriteLine($" Contains(\"city\") {parameters.Contains("city")}"); + Console.WriteLine($" Contains(\"City\") {parameters.Contains("City")} (names are ordinal, like the server's)"); + Console.WriteLine($" this[\"floor\"].Value {parameters["floor"].Value}"); + Console.WriteLine($" TryGetValue(\"nope\", out _) {parameters.TryGetValue("nope", out _)}"); + Console.WriteLine($" enumerates in order {string.Join(", ", parameters.Select(p => p.Name))}"); + + // The wire format is a name/value list, so a repeated name has no meaning and is refused rather than + // silently taking one of the two values. + try + { + parameters.Add("city", "Reykjavik"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" Adding 'city' twice throws: {ex.Message.Split(" (Parameter")[0]}"); + } + + Console.WriteLine(); + Console.WriteLine(" The collection is mutable and not thread-safe, and a client is meant to be shared, so"); + Console.WriteLine(" build one per operation and then leave it alone."); + + // The options record makes that cheap: keep the shared settings in one instance and derive the variant. + var shared = new ClickHouseTcpQueryOptions { Settings = new Dictionary { ["max_threads"] = "2" } }; + object hottest = await client.ExecuteScalarAsync( + $"SELECT max(temperature) FROM {TableName} WHERE city = {{city:String}}", + shared with { Parameters = parameters }); + + Console.WriteLine($" shared with {{ Parameters = parameters }} -> max(temperature) in Singapore = {hottest}"); + } + + private static async Task NoAtNameRewriting(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. Trap one: the query text carries the type, and @name is not rewritten\n"); + Console.WriteLine(" The HTTP client rewrites @city into {city:String} before sending. Nothing rewrites"); + Console.WriteLine(" anything here — the text goes to the server as you wrote it:\n"); + + var parameters = new ClickHouseTcpParameterCollection { { "city", "Amsterdam" } }; + var options = new ClickHouseTcpQueryOptions { Parameters = parameters }; + + try + { + await client.ExecuteScalarAsync($"SELECT count() FROM {TableName} WHERE city = @city", options); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" WHERE city = @city -> {Describe(ex)}"); + } + + // Without the type the server cannot parse the placeholder either. + try + { + await client.ExecuteScalarAsync($"SELECT count() FROM {TableName} WHERE city = {{city}}", options); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" WHERE city = {{city}} -> {Describe(ex)}"); + } + + object ok = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName} WHERE city = {{city:String}}", options); + Console.WriteLine($" WHERE city = {{city:String}} -> {ok}"); + Console.WriteLine(); + Console.WriteLine(" So a query written for Dapper does not port over unchanged, and neither does one that"); + Console.WriteLine(" relied on the HTTP client inferring a type from the .NET value."); + } + + private static async Task Identifiers(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. Where the type comes from, and the Identifier placeholder\n"); + Console.WriteLine(" Three places, first match wins:"); + Console.WriteLine(" 1. ClickHouseTcpParameter.ClickHouseType, set on the parameter"); + Console.WriteLine(" 2. the query's {name:Type} placeholder"); + Console.WriteLine(" 3. the value's CLR type — which only ever applies to a parameter the query does"); + Console.WriteLine(" not name, because a query that does name it must state the type for the server"); + Console.WriteLine(); + Console.WriteLine(" So rung 1 exists for the case where the placeholder is not the format the value should"); + Console.WriteLine(" be written in. The server still reads the type from the query text, so an override that"); + Console.WriteLine(" disagrees with the placeholder makes the server parse text it did not expect: most"); + Console.WriteLine(" queries want rung 2 and nothing else."); + Console.WriteLine(); + + // Identifier is not a data type: the server splices the value in as a name rather than as a literal, so a + // table or column can be bound instead of concatenated into the query text. + var parameters = new ClickHouseTcpParameterCollection { { "tbl", TableName }, { "col", "temperature" } }; + object rows = await client.ExecuteScalarAsync( + "SELECT count({col:Identifier}) FROM {tbl:Identifier}", + new ClickHouseTcpQueryOptions { Parameters = parameters }); + + Console.WriteLine($" Identifier binds a name rather than a value — the one placeholder that is not a type:"); + Console.WriteLine($" SELECT count({{col:Identifier}}) FROM {{tbl:Identifier}} with tbl='{TableName}', col='temperature' = {rows}"); + } + + private static async Task InstantsNeedATimezone(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. Trap two: a value that names an instant needs a timezone in the placeholder\n"); + Console.WriteLine(" The wire carries a wall-clock time and no timezone, so the server reads the value in"); + Console.WriteLine(" its session timezone. For a value that names a point in time that silently moves the"); + Console.WriteLine(" instant, so the client refuses to send it rather than let it move:\n"); + + var noon = new DateTime(2026, 6, 1, 12, 0, 0, DateTimeKind.Utc); + + // Kind=Utc names an instant, and DateTime with no timezone argument declares none. + await Refused("DateTime Kind=Utc into {t:DateTime}", client, $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime}}", noon); + + // A DateTimeOffset always names an instant, whatever its offset is. + await Refused( + "DateTimeOffset into {t:DateTime}", + client, + $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime}}", + new DateTimeOffset(noon)); + + Console.WriteLine(); + Console.WriteLine(" Two fixes. Declare the timezone in the placeholder, which is what you want whenever"); + Console.WriteLine(" the value really is an instant:"); + + object declared = await Count(client, $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime('UTC')}}", noon); + Console.WriteLine($" {{t:DateTime('UTC')}} with Kind=Utc -> {declared} rows"); + + object offsetDeclared = await Count(client, $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime('UTC')}}", new DateTimeOffset(noon).ToOffset(TimeSpan.FromHours(5))); + Console.WriteLine($" {{t:DateTime('UTC')}} with a +05:00 offset -> {offsetDeclared} rows (the same instant, moved into UTC)"); + + Console.WriteLine(); + Console.WriteLine(" Or pass Kind=Unspecified, which says \"this wall-clock time, in whatever timezone the"); + Console.WriteLine(" server reads it in\" — no instant is claimed, so nothing can be lost:"); + + var wallClock = new DateTime(2026, 6, 1, 12, 0, 0, DateTimeKind.Unspecified); + object unspecified = await Count(client, $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime}}", wallClock); + Console.WriteLine($" {{t:DateTime}} with Kind=Unspecified -> {unspecified} rows"); + Console.WriteLine(); + Console.WriteLine(" DateTime64 is the same rule: {t:DateTime64(3, 'UTC')} declares one, {t:DateTime64(3)}"); + Console.WriteLine(" does not."); + } + + private static async Task NamesThatCollideWithSettings(ClickHouseTcpClient client) + { + Console.WriteLine("\n6. Trap three: a parameter named after a server setting\n"); + Console.WriteLine(" Parameters ride in the Query packet's settings list. A server that reads the name as a"); + Console.WriteLine(" setting applies it as that setting instead of binding it, and the query then fails while"); + Console.WriteLine(" the server is reading the setting's value. The names to avoid are the ordinary setting"); + Console.WriteLine(" names: limit and offset above all, and max_threads, readonly and log_comment too.\n"); + + string sql = $"SELECT id FROM {TableName} ORDER BY id LIMIT {{limit:UInt64}}"; + var collided = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "limit", 2UL } }, + }; + + Console.WriteLine($" Server {(await client.GetServerInfoAsync()).Version}, parameter named 'limit':"); + + // A client of its own for the query that is meant to fail. The server rejects this one while it is still + // reading the settings list, and then closes the socket, so the connection it was on is dead even though + // the client saw an ordinary server error. Disposing this client throws that connection away with it; + // running the query on the shared client would leave a dead connection in its pool. + await using (ClickHouseTcpClient throwaway = ExampleConfig.CreateTcpClient()) + { + try + { + var ids = new List(); + await foreach (object[] row in throwaway.QueryAsync(sql, collided)) + { + ids.Add(row[0]); + } + + Console.WriteLine($" bound correctly — LIMIT {{limit:UInt64}} returned {ids.Count} row(s): {string.Join(", ", ids)}"); + Console.WriteLine(" This server is new enough to tell a parameter from a setting."); + } + catch (ClickHouseTcpException ex) + { + Console.WriteLine($" {Describe(ex)}"); + Console.WriteLine(" The error names neither the parameter nor the setting, so nothing in it points at"); + Console.WriteLine(" the name as the cause. (Code prints as Unknown when ClickHouseErrorCode has no"); + Console.WriteLine(" name for the raw number; the raw number is always there.)"); + Console.WriteLine(" The server also closes the connection after this one, so the next operation on"); + Console.WriteLine(" that connection can fail with a transport error instead — which is why this"); + Console.WriteLine(" example runs the failing query on a client of its own."); + } + } + + Console.WriteLine(); + Console.WriteLine(" The fix is a rename, and it always works:"); + + var renamed = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "row_limit", 2UL } }, + }; + + var kept = new List(); + await foreach (object[] row in client.QueryAsync($"SELECT id FROM {TableName} ORDER BY id LIMIT {{row_limit:UInt64}}", renamed)) + { + kept.Add(row[0]); + } + + Console.WriteLine($" LIMIT {{row_limit:UInt64}} returned {kept.Count} row(s): {string.Join(", ", kept)}"); + Console.WriteLine(); + Console.WriteLine(" This is the server's behaviour and it is version-dependent — 25.8 through 26.6 apply"); + Console.WriteLine(" the name as a setting, newer servers bind it. clickhouse-client --param_limit= fails"); + Console.WriteLine(" the same way, and the driver's HTTP transport is unaffected because it carries the"); + Console.WriteLine(" name separately. So avoid a setting name for a parameter if you support any server in"); + Console.WriteLine(" that range, whatever the server in front of you does today."); + } + + private static void ShowWhatIsAbsent() + { + Console.WriteLine("\n7. What the HTTP client has here and this one does not\n"); + Console.WriteLine(" @name placeholders, rewritten client-side. Write {name:Type}."); + Console.WriteLine(" IParameterTypeResolver. The type comes from the placeholder, or from"); + Console.WriteLine(" ClickHouseTcpParameter.ClickHouseType, or — only for a parameter the query does not"); + Console.WriteLine(" name — from the value's CLR type."); + Console.WriteLine(" IParameterFormatter. There is no hook for how a value is written."); + Console.WriteLine(" DbParameter and DbType. This client is not an ADO.NET provider."); + Console.WriteLine(); + Console.WriteLine(" Null and DBNull both send the null marker, so a Nullable placeholder is the way to"); + Console.WriteLine(" bind an absent value: {city:Nullable(String)}."); + } + + private static async Task Refused(string label, ClickHouseTcpClient client, string sql, object value) + { + try + { + await Count(client, sql, value); + Console.WriteLine($" {label,-38} -> accepted (unexpected)"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" {label}:"); + Console.WriteLine($" {Wrap(ex.Message.Split(" (Parameter")[0])}"); + } + } + + private static ValueTask Count(ClickHouseTcpClient client, string sql, object value) + => client.ExecuteScalarAsync( + sql, + new ClickHouseTcpQueryOptions { Parameters = new ClickHouseTcpParameterCollection { { "t", value } } }); + + // The mapped error code, the number the server actually sent, and the first line of the message. Code is + // Unknown for a code the enum does not name, which is why RawCode is worth printing next to it. + private static string Describe(ClickHouseTcpException exception) + { + string message = exception.Message; + int newline = message.IndexOf('\n'); + if (newline >= 0) + { + message = message[..newline]; + } + + const string prefix = "DB::Exception: "; + if (message.StartsWith(prefix, StringComparison.Ordinal)) + { + message = message[prefix.Length..]; + } + + if (message.Length > 120) + { + message = message[..120] + " ..."; + } + + return exception is ClickHouseTcpServerException server + ? $"{server.Code} (code {server.RawCode}): {message}" + : $"{exception.GetType().Name}: {message}"; + } + + // Reflows a long driver message so the console output stays readable. + private static string Wrap(string message) + { + var lines = new List(); + var line = new System.Text.StringBuilder(); + foreach (string word in message.Split(' ')) + { + if (line.Length + word.Length + 1 > 92) + { + lines.Add(line.ToString()); + line.Clear(); + } + + line.Append(line.Length == 0 ? word : " " + word); + } + + lines.Add(line.ToString()); + return string.Join("\n ", lines); + } +} diff --git a/examples/Tcp/Read/Tcp_008_Poco.cs b/examples/Tcp/Read/Tcp_008_Poco.cs new file mode 100644 index 000000000..6a37e3ab9 --- /dev/null +++ b/examples/Tcp/Read/Tcp_008_Poco.cs @@ -0,0 +1,281 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// Reading a result into a class with QueryAsync<T> and writing one back with +/// InsertRowsAsync<T> — one type, both directions, and the two attributes that adjust the mapping: +/// to rename a property and +/// to take one out of the mapping entirely. +/// +/// +/// This is the tier most application code should use. It converts values to the property's type, so a +/// DateTime column reaches a DateTime property, and each row owns its values, so a row can be +/// returned from the method that read it. +/// +/// +public static class TcpPoco +{ + private const string TableName = "example_tcp_poco"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await CreateTable(client); + await WriteFromPocos(client); + await ReadIntoPocos(client); + await ShowTheMapping(client); + await ShowNotMappedOnInsert(client); + await ShowWhatDoesNotMap(client); + ShowTheRules(); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped '{TableName}'"); + } + } + + private static async Task CreateTable(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + id UInt64, + full_name String, + signal_count UInt32, + recorded_at DateTime('UTC'), + internal_notes String + ) + ENGINE = MergeTree() + ORDER BY id"); + + Console.WriteLine($"Created '{TableName}':"); + Console.WriteLine(" id UInt64, full_name String, signal_count UInt32, recorded_at DateTime('UTC'), internal_notes String"); + } + + private static async Task WriteFromPocos(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. InsertRowsAsync — the columns the INSERT names are read off each object\n"); + + var midnight = new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc); + + var rows = new List + { + new() { Id = 1, DisplayName = "Ada Lovelace", SignalCount = 12, RecordedAt = midnight.AddHours(6), Notes = "not written" }, + new() { Id = 2, DisplayName = "Grace Hopper", SignalCount = 7, RecordedAt = midnight.AddHours(9), Notes = "not written" }, + new() { Id = 3, DisplayName = "Alan Turing", SignalCount = 21, RecordedAt = midnight.AddHours(14), Notes = "not written" }, + }; + + // The statement ends at VALUES and names the columns to fill. Each is matched to a property; a property no + // named column matches is simply not read, which is how Notes stays out of this insert. + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, full_name, signal_count, recorded_at) VALUES", + rows); + + Console.WriteLine($" Inserted {rows.Count} Observation objects into (id, full_name, signal_count, recorded_at)"); + Console.WriteLine(" id <- Id matched on the name"); + Console.WriteLine(" full_name <- DisplayName matched by [ClickHouseTcpColumn(Name = \"full_name\")]"); + Console.WriteLine(" signal_count <- SignalCount matched by ignoring case and underscores"); + Console.WriteLine(" recorded_at <- RecordedAt a DateTime property written as epoch seconds"); + Console.WriteLine(" internal_notes is not in the statement, so nothing was read for it and it took the"); + Console.WriteLine(" column's default. Notes carries [ClickHouseTcpNotMapped] and could not fill it anyway."); + } + + private static async Task ReadIntoPocos(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. QueryAsync — every result column fills the property it maps to\n"); + Console.WriteLine(" ID DisplayName Signals RecordedAt (Kind) Notes"); + Console.WriteLine(" -- -------------- ------- ------------------------- -----"); + + // SELECT * brings internal_notes back too, and it maps to nothing: Notes is [ClickHouseTcpNotMapped], so the + // column is skipped rather than assigned. + await foreach (Observation row in client.QueryAsync($"SELECT * FROM {TableName} ORDER BY id")) + { + Console.WriteLine( + $" {row.Id,2} {row.DisplayName,-14} {row.SignalCount,7} " + + $"{row.RecordedAt.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)} ({row.RecordedAt.Kind}) " + + $"{(row.Notes is null ? "(null)" : row.Notes)}"); + } + + Console.WriteLine(); + Console.WriteLine(" RecordedAt is a real DateTime, converted with the timezone the column declares — the"); + Console.WriteLine(" conversion the object[] tier does not do. Kind is Utc here because the column says UTC."); + Console.WriteLine(" Notes is null: internal_notes was in the result and was skipped."); + } + + private static async Task ShowTheMapping(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. How a column finds its property\n"); + Console.WriteLine(" In order, first match wins:"); + Console.WriteLine(" the exact property name, then case-insensitively, then ignoring underscores."); + Console.WriteLine(" So signal_count reaches SignalCount with no attribute at all. Reach for"); + Console.WriteLine(" [ClickHouseTcpColumn(Name = ...)] only when the names differ by more than that —"); + Console.WriteLine(" full_name and DisplayName here."); + Console.WriteLine(); + Console.WriteLine(" The names matched are the result's, not the table's, so a SELECT alias lines a query up"); + Console.WriteLine(" with a type just as well as an attribute does — and it is the only way to name a"); + Console.WriteLine(" computed column:\n"); + + // The names in the result are the aliases the query chose, not the table's, so an alias is the other way to + // line a result up with a type. + await foreach (Summary row in client.QueryAsync( + $"SELECT count() AS rows, sum(signal_count) AS total_signals, max(recorded_at) AS latest FROM {TableName}")) + { + Console.WriteLine($" SELECT count() AS rows, sum(signal_count) AS total_signals, max(recorded_at) AS latest"); + Console.WriteLine($" Rows {row.Rows}, TotalSignals {row.TotalSignals}, Latest {row.Latest.ToString("u", CultureInfo.InvariantCulture)}"); + Console.WriteLine(" Latest is a DateTimeOffset property, so the offset the column's timezone gives is kept."); + } + } + + private static async Task ShowNotMappedOnInsert(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. [ClickHouseTcpNotMapped] excludes a property in both directions\n"); + Console.WriteLine(" Section 2 showed the read half: internal_notes was skipped. On an insert the exclusion"); + Console.WriteLine(" means the property cannot fill a column, so naming that column is an error rather than"); + Console.WriteLine(" a silent default:\n"); + + try + { + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, full_name, internal_notes) VALUES", + new List { new() { Id = 9, DisplayName = "nobody", Notes = "would have gone here" } }); + } + catch (InvalidOperationException ex) + { + Console.WriteLine($" INSERT INTO ... (id, full_name, internal_notes) throws:"); + Console.WriteLine(Wrap(ex.Message, " ")); + } + + Console.WriteLine(); + Console.WriteLine(" Without the attribute, Notes would match internal_notes by ignoring the underscore, so"); + Console.WriteLine(" the attribute is what makes a property the driver never touches — a cache key, a"); + Console.WriteLine(" computed column, something loaded from elsewhere."); + } + + private static async Task ShowWhatDoesNotMap(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. What a mismatch does\n"); + + // Mapping is resolved against the first block of the result, so a type nothing maps to fails on the first + // row rather than yielding wrong values. + try + { + await foreach (Unrelated _ in client.QueryAsync($"SELECT id, full_name FROM {TableName}")) + { + break; + } + } + catch (InvalidOperationException ex) + { + Console.WriteLine(" A type no result column maps to:"); + Console.WriteLine(Wrap(ex.Message, " ")); + } + + // A property that some column does map to, but whose type the column cannot be read as. + try + { + await foreach (WrongType _ in client.QueryAsync($"SELECT id, full_name FROM {TableName}")) + { + break; + } + } + catch (InvalidOperationException ex) + { + Console.WriteLine("\n A property whose type the column cannot be read as:"); + Console.WriteLine(Wrap(ex.Message, " ")); + } + + Console.WriteLine(); + Console.WriteLine(" Both are checked against the first block, so an empty result yields nothing and"); + Console.WriteLine(" validates nothing. A property that no column reaches is not an error: it keeps its"); + Console.WriteLine(" default, which is what lets one type serve several queries."); + } + + private static void ShowTheRules() + { + Console.WriteLine("\n6. What T has to be\n"); + Console.WriteLine(" A concrete class with a public parameterless constructor."); + Console.WriteLine(" Every property a result column reaches needs a public setter — an init-only or"); + Console.WriteLine(" get-only property cannot be filled, so a record with positional parameters does not"); + Console.WriteLine(" work for reading."); + Console.WriteLine(" Every column an INSERT names needs a public getter of a type that column can be"); + Console.WriteLine(" written from."); + Console.WriteLine(" Rows own their values and stay valid after the enumeration advances. LowCardinality"); + Console.WriteLine(" elements can be shared within a block, so do not mutate an array-valued property"); + Console.WriteLine(" in place."); + Console.WriteLine(" The read and write plans are compiled once per type per client, so a client meant to"); + Console.WriteLine(" be a singleton pays the reflection once."); + } + + // Reflows a long driver message so the console output stays readable. + private static string Wrap(string message, string indent) + { + var lines = new List(); + var line = new System.Text.StringBuilder(); + foreach (string word in message.Split(' ')) + { + if (line.Length + word.Length + 1 > 95) + { + lines.Add(line.ToString()); + line.Clear(); + } + + line.Append(line.Length == 0 ? word : " " + word); + } + + lines.Add(line.ToString()); + return indent + string.Join("\n" + indent, lines); + } + + /// + /// One row of the example's table, used for both the insert and the read. The attributes are the only two the + /// native client has. + /// + private sealed class Observation + { + public ulong Id { get; set; } + + // The column is full_name, which no name-matching rule reaches from DisplayName. + [ClickHouseTcpColumn(Name = "full_name")] + public string DisplayName { get; set; } = string.Empty; + + // signal_count matches this by ignoring case and underscores, so no attribute is needed. + public uint SignalCount { get; set; } + + // A DateTime('UTC') column's epoch-second count, converted on the way in and out. + public DateTime RecordedAt { get; set; } + + // Excluded in both directions. Without this it would match internal_notes. + [ClickHouseTcpNotMapped] + public string? Notes { get; set; } + } + + // A second shape over the same table: the mapping is per query, so one table can feed several types. + private sealed class Summary + { + public ulong Rows { get; set; } + + public ulong TotalSignals { get; set; } + + // DateTimeOffset keeps the offset the column's timezone gives; DateTime would flatten it. + public DateTimeOffset Latest { get; set; } + } + + private sealed class Unrelated + { + public string Something { get; set; } = string.Empty; + } + + private sealed class WrongType + { + public ulong Id { get; set; } + + // full_name is a String column, and a String cannot be read as a Guid. + public Guid FullName { get; set; } + } +} From 1ac06f19fee8b1c86b64c119cce918b92b7223a9 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 12:44:59 +0200 Subject: [PATCH 07/16] Add the native protocol's write examples Two under examples/Tcp/Write/: the columnar insert tier, matching by name with a named subset and the server defaulting the rest; and composite writes, covering both accepted Array(T) shapes and re-inserting a column read straight out of a block for five composite families with nothing rebuilt. These close the write-path documentation debt the plan recorded: the two array shapes, the non-nullable-row rule, and the dense round trip now have a runnable demonstration each. Co-Authored-By: Claude Opus 5 (1M context) --- examples/Program.cs | 13 + examples/README.md | 5 + examples/Tcp/Write/Tcp_009_ColumnarInsert.cs | 382 ++++++++++++++++++ examples/Tcp/Write/Tcp_010_CompositeWrites.cs | 349 ++++++++++++++++ 4 files changed, 749 insertions(+) create mode 100644 examples/Tcp/Write/Tcp_009_ColumnarInsert.cs create mode 100644 examples/Tcp/Write/Tcp_010_CompositeWrites.cs diff --git a/examples/Program.cs b/examples/Program.cs index 01dc0fbef..929f346e4 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -383,6 +383,19 @@ private static async Task RunAllExamples(bool isInteractive) await TcpPoco.Run(); WaitForUser(isInteractive); + // Native Protocol: Writing Data + Console.WriteLine("\n\n" + new string('=', 70)); + Console.WriteLine("NATIVE PROTOCOL: WRITING DATA"); + Console.WriteLine(new string('=', 70) + "\n"); + + Console.WriteLine($"Running: {nameof(TcpColumnarInsert)}"); + await TcpColumnarInsert.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpCompositeWrites)}"); + await TcpCompositeWrites.Run(); + WaitForUser(isInteractive); + Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ALL EXAMPLES COMPLETED SUCCESSFULLY!"); Console.WriteLine(new string('=', 70)); diff --git a/examples/README.md b/examples/README.md index 1905f5211..104aaee5e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -113,6 +113,11 @@ These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/READ - [Tcp_007_Parameters.cs](Tcp/Read/Tcp_007_Parameters.cs) - `ClickHouseTcpParameterCollection` and `ClickHouseTcpQueryOptions.Parameters`, plus the three traps: `{name:Type}` is required, an instant needs a declared timezone, and a parameter named after a server setting - [Tcp_008_Poco.cs](Tcp/Read/Tcp_008_Poco.cs) - `QueryAsync` and `InsertRowsAsync` over one class, the name-matching rules, `[ClickHouseTcpColumn]`, `[ClickHouseTcpNotMapped]`, and what a mapping mismatch reports +### Native Protocol: Writing Data + +- [Tcp_009_ColumnarInsert.cs](Tcp/Write/Tcp_009_ColumnarInsert.cs) - The columnar insert tier: `ClickHouseTcpColumn.Create` per target column plus `InsertAsync`, matching by name so the order is free, a named subset with the server filling the rest, why no ClickHouse type is ever stated, `MaxRowsPerBlock`, and how it differs from `InsertRowsAsync` +- [Tcp_010_CompositeWrites.cs](Tcp/Write/Tcp_010_CompositeWrites.cs) - Writing composites: the jagged and dense `Array(T)` shapes, re-inserting a column read out of a block with nothing rebuilt, the non-nullable-row rule, and `Map`, `Tuple`, `Nullable`, `LowCardinality` + ## How to run ### Prerequisites diff --git a/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs b/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs new file mode 100644 index 000000000..db2d5a64b --- /dev/null +++ b/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs @@ -0,0 +1,382 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// The columnar insert tier: one .Create call per target column, then +/// InsertAsync(sql, columns). Columns are matched to the target by name, so their order is free and a +/// named subset is allowed; the ClickHouse type is never stated, because the server sends the target's schema +/// before any row data. +/// +/// +/// Tcp_006_BlocksAndColumns is the read side of this tier. This is the write side, and the two meet: +/// a column read out of a is a valid insert column. Tcp_010_CompositeWrites covers +/// the composite types and that round trip. +/// +/// +public static class TcpColumnarInsert +{ + // These examples are not the test suite, so fixed names are fine. All four are dropped even if a step throws. + private const string TableName = "example_tcp_columnar_insert"; + private const string DefaultsTable = "example_tcp_columnar_insert_defaults"; + private const string InstantsTable = "example_tcp_columnar_insert_instants"; + private const string BulkTable = "example_tcp_columnar_insert_bulk"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await OneColumnPerTargetColumn(client); + await MatchedByName(client); + await ANamedSubset(client); + await TheServerStatesTheType(client); + await BlockGeometry(client); + await TheRowTierForComparison(client); + RulesWorthKnowing(); + } + finally + { + foreach (string table in new[] { TableName, DefaultsTable, InstantsTable, BulkTable }) + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + + Console.WriteLine("\nDropped every table this example created."); + } + } + + private static async Task OneColumnPerTargetColumn(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + id UInt64, + name String, + score Float64 + ) + ENGINE = MergeTree() + ORDER BY id"); + + Console.WriteLine($"1. One column per target column\n"); + Console.WriteLine($" Created '{TableName}' (id UInt64, name String, score Float64)\n"); + + // The data is already grouped by column, which is how the wire wants it. Nothing is transposed and no + // value is boxed, so this is the shape to reach for when the data is columnar to begin with: a parsed + // file, a computed series, an ETL stage. + // + // Create takes the array over rather than copying it, so treat it as handed away: do not write to ids, + // names or scores until the insert has completed. + var ids = new ulong[] { 1, 2, 3, 4 }; + var names = new[] { "Ada", "Grace", "Alan", "Edsger" }; + var scores = new[] { 99.5, 97.25, 91.0, 94.75 }; + + await client.InsertAsync( + $"INSERT INTO {TableName} (id, name, score) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", ids), + ClickHouseTcpColumn.Create("name", names), + ClickHouseTcpColumn.Create("score", scores), + }); + + Console.WriteLine(" InsertAsync(\"INSERT INTO ... (id, name, score) VALUES\", [three columns])"); + Console.WriteLine(" The statement ends at VALUES. The rows travel after it as native blocks, never as SQL text.\n"); + await Show(client, $"SELECT id, name, score FROM {TableName} ORDER BY id", "id", "name", "score"); + + // The generic argument is the CLR type of one row's value, and the factory reports it back as ElementType. + // TypeName is null: an inserted column has no header of its own, so there is no ClickHouse type to report. + IColumn column = ClickHouseTcpColumn.Create("id", ids); + Console.WriteLine($"\n A built column reports: RowCount {column.RowCount}, ElementType {column.ElementType.Name}, TypeName {column.TypeName ?? "null"}"); + Console.WriteLine(" TypeName is null because the ClickHouse type is the server's to state, which is section 4."); + } + + private static async Task MatchedByName(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. Matched by name, not by position\n"); + + // Both orders differ from the table's and from each other, and the insert still lands correctly: the + // server's schema block names its columns, and each supplied column is looked up by its own name. + await client.InsertAsync( + $"INSERT INTO {TableName} (score, id, name) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("name", new[] { "Barbara", "Frances" }), + ClickHouseTcpColumn.Create("score", new[] { 96.5, 98.0 }), + ClickHouseTcpColumn.Create("id", new ulong[] { 5, 6 }), + }); + + Console.WriteLine(" The statement lists (score, id, name); the columns are supplied as name, score, id."); + Console.WriteLine(" Neither order is the table's, and both rows are still correct:\n"); + await Show(client, $"SELECT id, name, score FROM {TableName} WHERE id > 4 ORDER BY id", "id", "name", "score"); + + Console.WriteLine("\n Every column the statement lists must be supplied, and nothing else. Both mistakes are"); + Console.WriteLine(" caught before a single row is written, and both messages name the columns involved:\n"); + + await ShowRejection( + client, + "score not supplied", + $"INSERT INTO {TableName} (id, name, score) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 7 }), + ClickHouseTcpColumn.Create("name", new[] { "Katherine" }), + }); + + // The lookup is ordinal, so 'ID' is not 'id': the target reports id as missing and ID as unexpected. + await ShowRejection( + client, + "'ID' for 'id'", + $"INSERT INTO {TableName} (id) VALUES", + new IColumn[] { ClickHouseTcpColumn.Create("ID", new ulong[] { 7 }) }); + + Console.WriteLine("\n The second is why names are worth getting exactly right: the comparison is ordinal, as"); + Console.WriteLine(" ClickHouse's own is, so a case difference is a different column and not a near miss."); + } + + private static async Task ANamedSubset(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {DefaultsTable} + ( + id UInt64, + name String, + region String DEFAULT 'unknown', + attempts UInt8 DEFAULT 1 + ) + ENGINE = MergeTree() + ORDER BY id"); + + Console.WriteLine("\n3. A named subset, and the server fills the rest\n"); + Console.WriteLine($" '{DefaultsTable}' has four columns, two of them with a DEFAULT."); + Console.WriteLine(" The statement lists two, so the schema block describes two, so two columns are enough:\n"); + + await client.InsertAsync( + $"INSERT INTO {DefaultsTable} (id, name) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2 }), + ClickHouseTcpColumn.Create("name", new[] { "north", "south" }), + }); + + await Show(client, $"SELECT id, name, region, attempts FROM {DefaultsTable} ORDER BY id", "id", "name", "region", "attempts"); + + Console.WriteLine("\n region and attempts were never sent, and hold the DEFAULT the table declares."); + Console.WriteLine(" It is the statement's column list that decides the subset, not the columns you pass:"); + Console.WriteLine(" omit the list and the server describes every column, so every column must be supplied."); + } + + private static async Task TheServerStatesTheType(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {InstantsTable} + ( + seconds DateTime('UTC'), + millis DateTime64(3, 'UTC'), + micros DateTime64(6, 'UTC') + ) + ENGINE = MergeTree() + ORDER BY seconds"); + + Console.WriteLine("\n4. You never state the ClickHouse type\n"); + Console.WriteLine(" An INSERT over this protocol has two phases. The client sends the statement, the server"); + Console.WriteLine(" answers with a schema block naming and typing the target columns, and only then does the"); + Console.WriteLine(" client serialize. So the target type is known before a byte of data is encoded, and the"); + Console.WriteLine(" caller supplies CLR values only.\n"); + Console.WriteLine(" One DateTime[] into three columns of different precision, with no type stated anywhere:\n"); + + var instants = new[] + { + new DateTime(2026, 6, 1, 10, 0, 0, 125, DateTimeKind.Utc), + new DateTime(2026, 6, 1, 10, 0, 0, 875, DateTimeKind.Utc), + }; + + await client.InsertAsync( + $"INSERT INTO {InstantsTable} (seconds, millis, micros) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("seconds", instants), + ClickHouseTcpColumn.Create("millis", instants), + ClickHouseTcpColumn.Create("micros", instants), + }); + + await Show( + client, + $"SELECT toString(seconds), toString(millis), toString(micros) FROM {InstantsTable} ORDER BY millis", + 26, + "seconds", "millis", "micros"); + + Console.WriteLine("\n Same values, three encodings: whole seconds, milliseconds, microseconds."); + Console.WriteLine(" The 125 and 875 milliseconds are gone from the DateTime column because DateTime holds"); + Console.WriteLine(" seconds, which is the target's decision and not the client's.\n"); + + Console.WriteLine(" What the CLR type must satisfy is the target codec, and a mismatch is rejected before"); + Console.WriteLine(" any row is written:"); + + await ShowRejection( + client, + "long into a DateTime column", + $"INSERT INTO {InstantsTable} (seconds) VALUES", + new IColumn[] { ClickHouseTcpColumn.Create("seconds", new[] { 1780308000L }) }); + + Console.WriteLine("\n And note what is not here: no DESCRIBE, no probe query. The HTTP client's"); + Console.WriteLine(" InsertBinaryAsync has to learn the schema itself, with a SELECT ... WHERE 1=0 per call"); + Console.WriteLine(" unless you pass InsertOptions.ColumnTypes or turn on InsertOptions.UseSchemaCache."); + Console.WriteLine(" Here the schema arrives inside the insert, so there is nothing to cache or skip."); + } + + private static async Task BlockGeometry(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {BulkTable} + ( + id UInt64, + name String, + score Float64 + ) + ENGINE = MergeTree() + ORDER BY id"); + + Console.WriteLine("\n5. ClickHouseTcpInsertOptions.MaxRowsPerBlock\n"); + Console.WriteLine(" One InsertAsync call is one statement, but not necessarily one wire block. The cap"); + Console.WriteLine(" splits the rows into blocks of at most that many, which bounds what the client holds"); + Console.WriteLine(" encoded at once. It defaults to 1,000,000 rows; null writes one block of any height.\n"); + + Console.WriteLine(" The same six rows, once split into three blocks and once written as one:\n"); + Console.WriteLine(" MaxRowsPerBlock Rows stored Active parts"); + Console.WriteLine(" --------------- ----------- ------------"); + + await SixRowsAndCountParts(client, maxRowsPerBlock: 2); + await SixRowsAndCountParts(client, maxRowsPerBlock: null); + + Console.WriteLine(); + Console.WriteLine(" The cap is a client-side concern only. This server recombines the blocks of one insert"); + Console.WriteLine(" before it writes, so the six rows land as one part either way: lowering the cap does not"); + Console.WriteLine(" create parts and raising it does not remove them."); + Console.WriteLine(" Lower it to bound client memory on a very tall insert, and leave it alone otherwise."); + } + + private static async Task SixRowsAndCountParts(ClickHouseTcpClient client, int? maxRowsPerBlock) + { + await client.ExecuteAsync($"TRUNCATE TABLE {BulkTable}"); + + await client.InsertAsync( + $"INSERT INTO {BulkTable} (id, name, score) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3, 4, 5, 6 }), + ClickHouseTcpColumn.Create("name", new[] { "a", "b", "c", "d", "e", "f" }), + ClickHouseTcpColumn.Create("score", new[] { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 }), + }, + new ClickHouseTcpInsertOptions { MaxRowsPerBlock = maxRowsPerBlock }); + + object rows = await client.ExecuteScalarAsync($"SELECT count() FROM {BulkTable}"); + object parts = await client.ExecuteScalarAsync( + $"SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = '{BulkTable}' AND active"); + + Console.WriteLine($" {maxRowsPerBlock?.ToString() ?? "null",-15} {rows,11} {parts,12}"); + } + + private static async Task TheRowTierForComparison(ClickHouseTcpClient client) + { + Console.WriteLine("\n6. The row tier, for comparison\n"); + Console.WriteLine(" InsertRowsAsync takes one object[] per row and the same statement. It is the right"); + Console.WriteLine(" call when the data really is row-shaped, and it differs in three ways:\n"); + Console.WriteLine(" values are matched to the target columns by POSITION, not by name;"); + Console.WriteLine(" every value is boxed, which the caller pays for when it builds the rows;"); + Console.WriteLine(" the client then transposes those rows into one typed column per target.\n"); + + await client.ExecuteAsync($"TRUNCATE TABLE {BulkTable}"); + + var rows = new List + { + new object[] { 1UL, "Ada", 99.5 }, + new object[] { 2UL, "Grace", 97.25 }, + }; + + await client.InsertRowsAsync($"INSERT INTO {BulkTable} (id, name, score) VALUES", rows); + await Show(client, $"SELECT id, name, score FROM {BulkTable} ORDER BY id", "id", "name", "score"); + + // Positional matching is the trap: the values are the right types for the row, just not for the columns + // in the order the statement names them. The message names the position, the column and both CLR types. + try + { + await client.InsertRowsAsync( + $"INSERT INTO {BulkTable} (id, name, score) VALUES", + new List { new object[] { "Alan", 3UL, 91.0 } }); + } + catch (InvalidOperationException ex) + { + Console.WriteLine($"\n Values in the wrong order: {ex.Message}"); + } + + // The shaping cost is measurable and synchronous, so this number is exact rather than a benchmark: it is + // what the two shapes of the same 50,000 rows allocate before either call is made. + const int Rows = 50_000; + + long before = GC.GetAllocatedBytesForCurrentThread(); + var ids = new ulong[Rows]; + var names = new string[Rows]; + var scores = new double[Rows]; + long columnar = GC.GetAllocatedBytesForCurrentThread() - before; + + before = GC.GetAllocatedBytesForCurrentThread(); + var boxed = new object[Rows][]; + for (int i = 0; i < Rows; i++) + { + boxed[i] = new object[] { ids[i], names[i], scores[i] }; + } + + long rowwise = GC.GetAllocatedBytesForCurrentThread() - before; + + Console.WriteLine($"\n Holding the same {Rows:N0} rows of (UInt64, String, Float64):"); + Console.WriteLine($" three typed arrays {columnar,10:N0} bytes"); + Console.WriteLine($" one object[] per row {rowwise,10:N0} bytes ({boxed.Length:N0} arrays, plus a box per number)"); + Console.WriteLine(" The columnar tier's saving is mostly this: the arrays are usually the shape the data is"); + Console.WriteLine(" already in, so neither the boxes nor the per-row arrays are ever created."); + } + + private static void RulesWorthKnowing() + { + Console.WriteLine("\n7. Rules worth knowing\n"); + Console.WriteLine(" Create takes your array over, it does not copy it. Do not write to an array after"); + Console.WriteLine(" handing it to Create, until the insert has completed. The IEnumerable overload"); + Console.WriteLine(" enumerates once into an array, and takes over a T[] passed to it as is."); + Console.WriteLine(" Every column must hold the same number of rows, and each name must be unique."); + Console.WriteLine(" Zero rows is a no-op that still validates: the statement is sent, the schema is"); + Console.WriteLine(" matched, and no data block follows. An empty column list is a no-op too."); + Console.WriteLine(" The columns you build are yours. The insert does not dispose them, and disposing one"); + Console.WriteLine(" before the insert empties it, so keep them alive until the call returns."); + Console.WriteLine(" InsertAsync is a ValueTask: await it once, and do not await it twice."); + } + + // Prints a small result set with a header, so each section can show what the server actually stored. + private static Task Show(ClickHouseTcpClient client, string sql, params string[] headers) + => Show(client, sql, 12, headers); + + private static async Task Show(ClickHouseTcpClient client, string sql, int width, params string[] headers) + { + Console.WriteLine(" " + string.Join(" ", headers.Select(h => h.PadRight(width)))); + Console.WriteLine(" " + string.Join(" ", headers.Select(_ => new string('-', width)))); + await foreach (object[] row in client.QueryAsync(sql)) + { + Console.WriteLine(" " + string.Join(" ", row.Select(v => (v?.ToString() ?? "NULL").PadRight(width)))); + } + } + + // Runs an insert that is expected to be rejected client-side and prints the reason. The client closes the row + // stream cleanly before throwing, so the connection goes back to the pool usable. + private static async Task ShowRejection(ClickHouseTcpClient client, string what, string sql, IReadOnlyList columns) + { + try + { + await client.InsertAsync(sql, columns); + Console.WriteLine($" {what}: accepted, which this example did not expect"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" {what}: {ex.Message.Split(" (Parameter")[0]}"); + } + } +} diff --git a/examples/Tcp/Write/Tcp_010_CompositeWrites.cs b/examples/Tcp/Write/Tcp_010_CompositeWrites.cs new file mode 100644 index 000000000..8d7af4d8f --- /dev/null +++ b/examples/Tcp/Write/Tcp_010_CompositeWrites.cs @@ -0,0 +1,349 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// Writing composite columns on the columnar tier: the two array shapes, and then Map, Tuple, +/// Nullable and LowCardinality. +/// +/// +/// An Array(T) column is accepted in two shapes. Jagged is one T[] per row, which is what +/// .Create builds. Dense is a flat inner column +/// plus per-row offsets, which is the wire's own layout and what a read produces, so a column read out of a +/// re-inserts with nothing rebuilt. Section 3 is that round trip, and it is the reason the +/// tier exists. +/// +/// +/// +/// Tcp_009_ColumnarInsert covers the tier itself: matching by name, the subset rule, and why no ClickHouse +/// type is ever stated. Tcp_006_BlocksAndColumns covers reading the same shapes. +/// +/// +public static class TcpCompositeWrites +{ + // These examples are not the test suite, so fixed names are fine. All three are dropped even if a step throws. + private const string ArraysTable = "example_tcp_composite_writes_arrays"; + private const string DenseTable = "example_tcp_composite_writes_dense"; + private const string OthersTable = "example_tcp_composite_writes_others"; + private const string OthersDenseTable = "example_tcp_composite_writes_others_dense"; + + private const string ArrayColumns = "id, readings, tags, maybe"; + private const string OtherColumns = "id, attrs, point, score, city, nick"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await JaggedArrays(client); + await TheNonNullableRowRule(client); + await DenseArraysAndTheRoundTrip(client); + await TheOtherComposites(client); + await WhatEachTargetAccepts(client); + WhatToRemember(); + } + finally + { + foreach (string table in new[] { ArraysTable, DenseTable, OthersTable, OthersDenseTable }) + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + + Console.WriteLine("\nDropped every table this example created."); + } + } + + private static async Task JaggedArrays(ClickHouseTcpClient client) + { + await client.ExecuteAsync(ArrayDdl(ArraysTable)); + await client.ExecuteAsync(ArrayDdl(DenseTable)); + + Console.WriteLine("1. The jagged shape: one array per row\n"); + Console.WriteLine($" '{ArraysTable}' (id UInt64, readings Array(Float64), tags Array(String),"); + Console.WriteLine(" maybe Array(Nullable(Int32)))\n"); + Console.WriteLine(" Create builds an IColumn, so the CLR type of one row is the array type the"); + Console.WriteLine(" target's element type maps to: double[] for Array(Float64), string[] for Array(String),"); + Console.WriteLine(" int?[] for Array(Nullable(Int32)).\n"); + + // Array.Empty is an empty row, which is a value. It is not a null row: see section 2. + var readings = new[] { new[] { 0.5, 0.75, 1.0 }, new[] { 1.25, 1.5 }, Array.Empty() }; + var tags = new[] { new[] { "north", "roof" }, Array.Empty(), new[] { "south" } }; + var maybe = new[] { new int?[] { 7, null, 9 }, new int?[] { null }, Array.Empty() }; + + await client.InsertAsync( + $"INSERT INTO {ArraysTable} ({ArrayColumns}) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3 }), + ClickHouseTcpColumn.Create("readings", readings), + ClickHouseTcpColumn.Create("tags", tags), + ClickHouseTcpColumn.Create("maybe", maybe), + }); + + await ShowArrays(client, ArraysTable); + + Console.WriteLine("\n Nothing was flattened up front. The codec walks the rows once to build the offsets"); + Console.WriteLine(" the wire needs, then writes each row's elements straight from its own array, so the"); + Console.WriteLine(" only extra buffer is the offsets. Where the element type is itself composite the"); + Console.WriteLine(" elements go through a lazy concatenated view rather than a copy."); + } + + private static async Task TheNonNullableRowRule(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. A row of Array(T) may not be null\n"); + Console.WriteLine(" ClickHouse has no such value: an Array(T) row is a run of elements, possibly of length"); + Console.WriteLine(" zero, and there is no bit on the wire that could say 'absent' instead. So a null row is"); + Console.WriteLine(" refused rather than quietly turned into an empty one:\n"); + + // The offsets pass reaches this row and refuses it. That happens while the block is being encoded, after + // the statement has gone out, so this failure costs the connection: the client cannot leave a half-written + // block on the wire, and drops it instead. Validate your rows before you hand them over. + try + { + await client.InsertAsync( + $"INSERT INTO {ArraysTable} (id, readings) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 4 }), + ClickHouseTcpColumn.Create("readings", new double[]?[] { null }), + }); + } + catch (ArgumentException ex) + { + Console.WriteLine($" {ex.Message.Split(" (Parameter")[0]}"); + } + + Console.WriteLine("\n The two ways out are in the message, and they mean different things:"); + Console.WriteLine(" Array.Empty() is a row that exists and holds nothing."); + Console.WriteLine(" Array(Nullable(T)) is a row whose ELEMENTS may be null, which is the 'maybe' column"); + Console.WriteLine(" above: row 2 is [NULL], one element long, and row 3 is [], zero elements long."); + Console.WriteLine(); + Console.WriteLine(" Unlike the name and type checks in Tcp_009, this one fires while the block is being"); + Console.WriteLine(" encoded, so it is worth checking your rows before the call rather than after it."); + } + + private static async Task DenseArraysAndTheRoundTrip(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. The dense shape, and the round trip it makes free\n"); + Console.WriteLine(" The wire does not carry one array per row. It carries every row's elements end to end"); + Console.WriteLine(" plus one cumulative offset per row, and that is exactly what a read hands back: an"); + Console.WriteLine(" IArrayColumn over the server's own layout. Handed back to an insert, it is written"); + Console.WriteLine(" from that layout with no arrays rebuilt.\n"); + + int blocks = 0; + await foreach (Block block in client.StreamAsync($"SELECT {ArrayColumns} FROM {ArraysTable} ORDER BY id")) + { + blocks++; + + if (block["readings"] is IArrayColumn dense) + { + Console.WriteLine($" readings, as read: {block["readings"].TypeName}, {dense.RowCount} rows"); + Console.WriteLine($" InnerValues = [{string.Join(", ", dense.InnerValues.ToArray())}] (every row's elements, flat)"); + Console.WriteLine($" Offsets = [{string.Join(", ", dense.Offsets.ToArray())}] (cumulative ends, one more entry than rows)"); + Console.WriteLine(" Row i is InnerValues.Slice(Offsets[i], Offsets[i + 1] - Offsets[i]), so row 2's"); + Console.WriteLine($" slice is [{string.Join(", ", dense.InnerValues[dense.Offsets[2]..dense.Offsets[3]].ToArray())}] and it is empty because both offsets are {dense.Offsets[2]}."); + } + + // The block is borrowed, so the re-insert happens inside this iteration. It runs on a second pooled + // connection, because the first is busy streaming this result. + await client.InsertAsync($"INSERT INTO {DenseTable} ({ArrayColumns}) VALUES", block.Columns.ToArray()); + } + + Console.WriteLine($"\n Re-inserted {blocks} block into '{DenseTable}' with no column rebuilt at all:\n"); + await ShowArrays(client, DenseTable); + + Console.WriteLine("\n Every value survived, including the empty rows and the null elements. That is the"); + Console.WriteLine(" whole point of the tier: a copy, a filter, or a backfill can read a block and write it"); + Console.WriteLine(" again without ever materializing a row.\n"); + Console.WriteLine(" Two things to know about it:"); + Console.WriteLine(" A read column carries the name the SELECT gave it, and an insert matches by name, so"); + Console.WriteLine(" rename in the query when the target column is named differently: SELECT readings AS"); + Console.WriteLine(" other_name. There is no way to rename a column object."); + Console.WriteLine(" The dense shape is what you receive, not something you can build. Create only makes"); + Console.WriteLine(" the jagged shape, so a caller that already holds flat values and offsets has to"); + Console.WriteLine(" slice them into per-row arrays first."); + } + + private static async Task TheOtherComposites(ClickHouseTcpClient client) + { + await client.ExecuteAsync(OthersDdl(OthersTable)); + await client.ExecuteAsync(OthersDdl(OthersDenseTable)); + + Console.WriteLine("\n4. Map, Tuple, Nullable and LowCardinality\n"); + + // A Map row is a pair array rather than a dictionary: the wire carries keys and values in order, so a + // pair array can express what a Dictionary cannot. + var attrs = new[] + { + new[] { new KeyValuePair("floor", 3), new KeyValuePair("room", 12) }, + Array.Empty>(), + }; + + await client.InsertAsync( + $"INSERT INTO {OthersTable} ({OtherColumns}) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2 }), + ClickHouseTcpColumn.Create("attrs", attrs), + ClickHouseTcpColumn.Create("point", new[] { (1, "one"), (2, "two") }), + ClickHouseTcpColumn.Create("score", new double?[] { 1.25, null }), + ClickHouseTcpColumn.Create("city", new[] { "Amsterdam", "Amsterdam" }), + ClickHouseTcpColumn.Create("nick", new string?[] { "ada", null }), + }); + + Console.WriteLine(" Map(String, Int64) is KeyValuePair[] per row, not a"); + Console.WriteLine(" Dictionary: the wire carries the keys and the values"); + Console.WriteLine(" as two columns in order, which a pair array matches."); + Console.WriteLine(" Tuple(x Int32, y String) is (int, string) per row. The element names live in"); + Console.WriteLine(" the type string only, so an unnamed ValueTuple is"); + Console.WriteLine(" what a named tuple takes."); + Console.WriteLine(" Nullable(Float64) is double? per row. A reference type is already"); + Console.WriteLine(" nullable, so Nullable(String) takes string."); + Console.WriteLine(" LowCardinality(String) is plain string per row. The client works out the"); + Console.WriteLine(" block's dictionary and its key width; you never"); + Console.WriteLine(" build either."); + Console.WriteLine(" LowCardinality(Nullable(String)) is string per row, null allowed.\n"); + + await ShowOthers(client, OthersTable); + + Console.WriteLine("\n All five take section 3's round trip too. A Map arrives as its key and value columns,"); + Console.WriteLine(" a Tuple as its element columns, a Nullable as a null map plus its inner column, a"); + Console.WriteLine(" LowCardinality as a dictionary plus its keys, and each of those is the layout its codec"); + Console.WriteLine($" writes from. Re-inserted into '{OthersDenseTable}' straight from the read:\n"); + + await foreach (Block block in client.StreamAsync($"SELECT {OtherColumns} FROM {OthersTable} ORDER BY id")) + { + await client.InsertAsync($"INSERT INTO {OthersDenseTable} ({OtherColumns}) VALUES", block.Columns.ToArray()); + } + + await ShowOthers(client, OthersDenseTable); + } + + private static async Task WhatEachTargetAccepts(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. What a composite refuses, and what it says\n"); + + // Both of these are type checks against the target's schema, so they are decided before any row is + // written and the message is the only cost. + await ShowRejection( + client, + "a Dictionary for a Map row", + $"INSERT INTO {OthersTable} (id, attrs) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 3 }), + ClickHouseTcpColumn.Create("attrs", new[] { new Dictionary { ["floor"] = 3 } }), + }); + + await ShowRejection( + client, + "a double for a Nullable(Float64) row", + $"INSERT INTO {OthersTable} (id, score) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 3 }), + ClickHouseTcpColumn.Create("score", new[] { 1.0 }), + }); + + Console.WriteLine(); + Console.WriteLine(" Both messages name the target type, which is the useful half. The CLR half is spelled as"); + Console.WriteLine(" the internal column class, so read its type argument (System.Double here) and compare"); + Console.WriteLine(" that with the list in section 4.\n"); + + // A Map row has the same non-nullable rule as an Array row, and is checked at the same point: while the + // block is being encoded, not before it. + await ShowRejection( + client, + "a null Map row", + $"INSERT INTO {OthersTable} (id, attrs) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 3 }), + ClickHouseTcpColumn.Create("attrs", new KeyValuePair[]?[] { null }), + }); + + Console.WriteLine("\n Same rule as Array(T), same two ways out: an empty pair array for an empty map, or"); + Console.WriteLine(" Map(K, Nullable(V)) to carry null values. And like section 2's, it is a check on the"); + Console.WriteLine(" values rather than on the types, so it fires later than the two above."); + } + + private static void WhatToRemember() + { + Console.WriteLine("\n6. What to remember\n"); + Console.WriteLine(" Pick the CLR type from the target, not from what is convenient: one array per row for"); + Console.WriteLine(" Array(T), one pair array per row for Map(K, V), a ValueTuple for Tuple(...), T? for"); + Console.WriteLine(" Nullable(T), and the plain value for LowCardinality(T)."); + Console.WriteLine(" A row of Array(T) or Map(K, V) is never null. Use an empty array, or make the elements"); + Console.WriteLine(" nullable."); + Console.WriteLine(" A column read out of a block is a valid insert column, and the fastest one: it is"); + Console.WriteLine(" already in the layout the codec writes from. Re-insert it inside the iteration that"); + Console.WriteLine(" yielded it, because the block is borrowed."); + Console.WriteLine(" Match the column's name to the target, in the SELECT if need be."); + } + + private static string OthersDdl(string table) => $@" + CREATE TABLE {table} + ( + id UInt64, + attrs Map(String, Int64), + point Tuple(x Int32, y String), + score Nullable(Float64), + city LowCardinality(String), + nick LowCardinality(Nullable(String)) + ) + ENGINE = MergeTree() + ORDER BY id"; + + private static string ArrayDdl(string table) => $@" + CREATE TABLE {table} + ( + id UInt64, + readings Array(Float64), + tags Array(String), + maybe Array(Nullable(Int32)) + ) + ENGINE = MergeTree() + ORDER BY id"; + + private static async Task ShowArrays(ClickHouseTcpClient client, string table) + { + Console.WriteLine(" id readings tags maybe"); + Console.WriteLine(" -- ---------------- ----------------- ----------------"); + await foreach (object[] row in client.QueryAsync( + $"SELECT id, toString(readings), toString(tags), toString(maybe) FROM {table} ORDER BY id")) + { + Console.WriteLine($" {row[0],2} {row[1],-16} {row[2],-17} {row[3],-16}"); + } + } + + private static async Task ShowOthers(ClickHouseTcpClient client, string table) + { + Console.WriteLine(" id attrs point score city nick"); + Console.WriteLine(" -- ------------------------ ---------- ----- --------- ----"); + await foreach (object[] row in client.QueryAsync( + $@"SELECT id, toString(attrs), toString(point), toString(score), city, toString(nick) + FROM {table} ORDER BY id")) + { + Console.WriteLine($" {row[0],2} {row[1],-24} {row[2],-10} {Text(row[3]),-5} {row[4],-9} {Text(row[5])}"); + } + } + + // toString of a NULL is the empty string, which is indistinguishable from an empty string in a table. + private static string Text(object value) => value is string { Length: 0 } ? "NULL" : value?.ToString() ?? "NULL"; + + // Runs an insert that is expected to be rejected client-side and prints the reason. + private static async Task ShowRejection(ClickHouseTcpClient client, string what, string sql, IReadOnlyList columns) + { + try + { + await client.InsertAsync(sql, columns); + Console.WriteLine($" {what}: accepted, which this example did not expect"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" {what}:"); + Console.WriteLine($" {ex.Message.Split(" (Parameter")[0]}"); + } + } +} From daf87d28a92bb0b4101de1b25e44240bc2a897e7 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 13:57:14 +0200 Subject: [PATCH 08/16] Add the native protocol's data type examples Five under examples/Tcp/Types/: the scalar CLR map including the 256-bit integers and when a Decimal needs ClickHouseTcpDecimal; the temporal types, with the presentation timezone measured under five session_timezone values and the insert side of the same model; composite reads through the typed views and the geo aliases as ValueTuple; Variant, Dynamic and JSON, including the normalization that makes text in differ from text out; and QBit's bit planes, rebuilding a vector at the precision L2DistanceTransposed would use. The QBit and strided-QBit cases this server refuses print its own refusal and the version they need rather than being left out. Co-Authored-By: Claude Opus 5 (1M context) --- examples/Program.cs | 25 + examples/README.md | 8 + examples/Tcp/Types/Tcp_011_ScalarTypes.cs | 481 +++++++++++++++ .../Tcp/Types/Tcp_012_DateTimeAndTimezones.cs | 463 ++++++++++++++ examples/Tcp/Types/Tcp_013_CompositeRead.cs | 574 ++++++++++++++++++ .../Tcp/Types/Tcp_014_VariantDynamicJson.cs | 431 +++++++++++++ .../Tcp/Types/Tcp_015_QBitVectorSearch.cs | 488 +++++++++++++++ 7 files changed, 2470 insertions(+) create mode 100644 examples/Tcp/Types/Tcp_011_ScalarTypes.cs create mode 100644 examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs create mode 100644 examples/Tcp/Types/Tcp_013_CompositeRead.cs create mode 100644 examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs create mode 100644 examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs diff --git a/examples/Program.cs b/examples/Program.cs index 929f346e4..ecff83f43 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -396,6 +396,31 @@ private static async Task RunAllExamples(bool isInteractive) await TcpCompositeWrites.Run(); WaitForUser(isInteractive); + // Native Protocol: Data Types + Console.WriteLine("\n\n" + new string('=', 70)); + Console.WriteLine("NATIVE PROTOCOL: DATA TYPES"); + Console.WriteLine(new string('=', 70) + "\n"); + + Console.WriteLine($"Running: {nameof(TcpScalarTypes)}"); + await TcpScalarTypes.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpDateTimeAndTimezones)}"); + await TcpDateTimeAndTimezones.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpCompositeRead)}"); + await TcpCompositeRead.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpVariantDynamicJson)}"); + await TcpVariantDynamicJson.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpQBitVectorSearch)}"); + await TcpQBitVectorSearch.Run(); + WaitForUser(isInteractive); + Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ALL EXAMPLES COMPLETED SUCCESSFULLY!"); Console.WriteLine(new string('=', 70)); diff --git a/examples/README.md b/examples/README.md index 104aaee5e..ddb1aa865 100644 --- a/examples/README.md +++ b/examples/README.md @@ -118,6 +118,14 @@ These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/READ - [Tcp_009_ColumnarInsert.cs](Tcp/Write/Tcp_009_ColumnarInsert.cs) - The columnar insert tier: `ClickHouseTcpColumn.Create` per target column plus `InsertAsync`, matching by name so the order is free, a named subset with the server filling the rest, why no ClickHouse type is ever stated, `MaxRowsPerBlock`, and how it differs from `InsertRowsAsync` - [Tcp_010_CompositeWrites.cs](Tcp/Write/Tcp_010_CompositeWrites.cs) - Writing composites: the jagged and dense `Array(T)` shapes, re-inserting a column read out of a block with nothing rebuilt, the non-nullable-row rule, and `Map`, `Tuple`, `Nullable`, `LowCardinality` +### Native Protocol: Data Types + +- [Tcp_011_ScalarTypes.cs](Tcp/Types/Tcp_011_ScalarTypes.cs) - The CLR type of every scalar: the integer widths including `Int256`/`UInt256`, `BFloat16`'s lost precision, why the declared precision and not the value decides between `decimal` and `ClickHouseTcpDecimal`, `String` against `FixedString(N)`, and enums as bare ordinals +- [Tcp_012_DateTimeAndTimezones.cs](Tcp/Types/Tcp_012_DateTimeAndTimezones.cs) - `Date`, `Date32`, `DateTime`, `DateTime64(scale)`, `Time`, `Time64(scale)`: the stored count against the presented calendar value, where the presentation timezone comes from, what `DateTime.Kind` does on an insert, and why a parameter naming an instant needs a declared timezone +- [Tcp_013_CompositeRead.cs](Tcp/Types/Tcp_013_CompositeRead.cs) - Reading composites through `IArrayColumn`, `IMapColumn`, `ITupleColumn`, `INestedColumn`, `INullableColumn` and `ILowCardinalityColumn`, how they nest, and the geo aliases — which surface as `ValueTuple` where the HTTP driver builds `System.Tuple` +- [Tcp_014_VariantDynamicJson.cs](Tcp/Types/Tcp_014_VariantDynamicJson.cs) - `IVariantColumn` and `IDynamicColumn`: discriminators, local indices, the two different NULL markers, and typed dispatch without boxing — then `JSON`, which travels as text and comes back normalized, so what you write is not what you read +- [Tcp_015_QBitVectorSearch.cs](Tcp/Types/Tcp_015_QBitVectorSearch.cs) - `QBit(T, N)` and `IQBitColumn`: the transposed bit-plane layout, `GetPlane` and the bitmap byte order, rebuilding a vector from its top planes to match `L2DistanceTransposed`'s precision argument, and the padding a dimension that is not a multiple of 8 costs + ## How to run ### Prerequisites diff --git a/examples/Tcp/Types/Tcp_011_ScalarTypes.cs b/examples/Tcp/Types/Tcp_011_ScalarTypes.cs new file mode 100644 index 000000000..00d7918c6 --- /dev/null +++ b/examples/Tcp/Types/Tcp_011_ScalarTypes.cs @@ -0,0 +1,481 @@ +using System.Globalization; +using System.Net; +using System.Numerics; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// What CLR type each ClickHouse scalar becomes, and the four families where the answer is not the obvious one: +/// the 256-bit integers, the decimals, BFloat16, and the enums. +/// +/// +/// One rule underlies all of it: the client hands back the value the wire carried, in the narrowest CLR +/// type that holds it without loss. So UInt8 is a and not an , +/// FixedString(N) is a [] and not a , and an Enum8 is its +/// ordinal and not its label. The same type is what an insert column must hold, in both directions. +/// +/// +/// +/// Tcp_012 covers the date and time family, Tcp_013 the composites. This one assumes the block tier +/// from Tcp_006. +/// +/// +public static class TcpScalarTypes +{ + private const string TableName = "example_tcp_scalar_types"; + + // Every column of the table above, in one list, so the DDL, the insert and the read agree. + private const string Columns = + "u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, u256, i256, " + + "f32, f64, bf16, d32, d64, d128, d256, flag, text, fixed5, id, ip4, ip6, e8, e16"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await Seed(client); + await TheWholeMap(client); + await WideIntegers(client); + await Decimals(client); + await Floats(client); + await StringsAndBytes(client); + await Enums(client); + await Nothing(client); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped '{TableName}'"); + } + } + + private static async Task Seed(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + u8 UInt8, i8 Int8, u16 UInt16, i16 Int16, u32 UInt32, i32 Int32, u64 UInt64, i64 Int64, + u128 UInt128, i128 Int128, u256 UInt256, i256 Int256, + f32 Float32, f64 Float64, bf16 BFloat16, + d32 Decimal32(2), d64 Decimal64(4), d128 Decimal128(20), d256 Decimal256(40), + flag Bool, text String, fixed5 FixedString(5), + id UUID, ip4 IPv4, ip6 IPv6, + e8 Enum8('red' = 1, 'green' = 2), e16 Enum16('small' = 100, 'big' = 3000) + ) + ENGINE = MergeTree() + ORDER BY u64"); + + // One row, written column by column. Each array's element type is the CLR type that column accepts, so + // this list is the write-side answer to the same question the read side answers below. + await client.InsertAsync( + $"INSERT INTO {TableName} ({Columns}) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("u8", new byte[] { 200 }), + ClickHouseTcpColumn.Create("i8", new sbyte[] { -100 }), + ClickHouseTcpColumn.Create("u16", new ushort[] { 60000 }), + ClickHouseTcpColumn.Create("i16", new short[] { -30000 }), + ClickHouseTcpColumn.Create("u32", new uint[] { 4000000000 }), + ClickHouseTcpColumn.Create("i32", new[] { -2000000000 }), + ClickHouseTcpColumn.Create("u64", new ulong[] { ulong.MaxValue }), + ClickHouseTcpColumn.Create("i64", new[] { long.MinValue }), + ClickHouseTcpColumn.Create("u128", new[] { UInt128.MaxValue }), + ClickHouseTcpColumn.Create("i128", new[] { Int128.MinValue }), + + // The only two numeric types the driver defines itself: .NET stops at 128 bits. + ClickHouseTcpColumn.Create("u256", new[] { UInt256.FromBigInteger(BigInteger.Pow(2, 255)) }), + ClickHouseTcpColumn.Create("i256", new[] { Int256.FromBigInteger(-BigInteger.Pow(2, 255)) }), + + ClickHouseTcpColumn.Create("f32", new[] { 1.5f }), + ClickHouseTcpColumn.Create("f64", new[] { -2.25 }), + + // BFloat16 has no CLR type of its own, so it is written from and read as a float. + ClickHouseTcpColumn.Create("bf16", new[] { 0.1f }), + + // Precision decides the CLR type: 2 and 4 digits fit a decimal, 20 and 40 do not. + ClickHouseTcpColumn.Create("d32", new[] { 1.25m }), + ClickHouseTcpColumn.Create("d64", new[] { 1.2345m }), + ClickHouseTcpColumn.Create("d128", new[] { new ClickHouseTcpDecimal(BigInteger.Parse("123456789012345678901234567890", CultureInfo.InvariantCulture), 20) }), + ClickHouseTcpColumn.Create("d256", new[] { new ClickHouseTcpDecimal(BigInteger.Pow(10, 45) + 7, 40) }), + + ClickHouseTcpColumn.Create("flag", new[] { true }), + ClickHouseTcpColumn.Create("text", new[] { "hello" }), + + // FixedString is bytes, and exactly N of them. + ClickHouseTcpColumn.Create("fixed5", new[] { new byte[] { 0x61, 0x00, 0x62, 0xFF, 0x10 } }), + + ClickHouseTcpColumn.Create("id", new[] { Guid.Parse("61f0c404-5cb3-11e7-907b-a6006ad3dba0") }), + ClickHouseTcpColumn.Create("ip4", new[] { IPAddress.Parse("192.168.0.1") }), + ClickHouseTcpColumn.Create("ip6", new[] { IPAddress.Parse("2001:db8::1") }), + + // An enum is written as its ordinal, never as its label. + ClickHouseTcpColumn.Create("e8", new sbyte[] { 2 }), + ClickHouseTcpColumn.Create("e16", new short[] { 3000 }), + }); + + Console.WriteLine($"Seeded '{TableName}' with one row of every scalar type, written column by column."); + Console.WriteLine("Each Create above states the CLR type that column accepts; the table below is the"); + Console.WriteLine("same answer read back."); + } + + private static async Task TheWholeMap(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. The whole scalar map\n"); + Console.WriteLine(" ClickHouse type IColumn is The value read back"); + Console.WriteLine(" ------------------------------------ -------------------- -------------------"); + + await foreach (Block block in client.StreamAsync($"SELECT {Columns} FROM {TableName}")) + { + foreach (IColumn column in block.Columns) + { + object value = column.GetValue(0); + Console.WriteLine($" {column.TypeName,-36} {Describe(column.ElementType),-20} {Render(value)}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" Nothing in that table is a conversion. ElementType is the type the wire's bytes are,"); + Console.WriteLine(" so a read costs a copy at most, and the same type is what an insert column must hold."); + } + + private static async Task WideIntegers(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. The wide integers: two from .NET, two from this driver\n"); + Console.WriteLine(" Int128 and UInt128 are BCL types, so UInt128.MaxValue and Int128.MinValue work as you"); + Console.WriteLine(" expect. Int256 and UInt256 have no BCL counterpart, so the driver defines them:\n"); + + await foreach (Block block in client.StreamAsync($"SELECT u128, i128, u256, i256 FROM {TableName}")) + { + Console.WriteLine($" UInt128 = {block.Column("u128")[0]}"); + Console.WriteLine($" Int128 = {block.Column("i128")[0]}"); + + UInt256 wide = block.Column("u256")[0]; + Int256 signed = block.Column("i256")[0]; + Console.WriteLine($" UInt256 = {wide}"); + Console.WriteLine($" Int256 = {signed} IsNegative {signed.IsNegative}"); + + Console.WriteLine(); + Console.WriteLine(" They are 32-byte value types, four ulong limbs least significant first, and they"); + Console.WriteLine(" carry exactly what a wire value needs — no arithmetic:"); + Console.WriteLine($" Int256.Size {Int256.Size} bytes"); + Console.WriteLine($" Int256.Zero {Int256.Zero}"); + Console.WriteLine($" new Int256(0, 1, 0, 0) {new Int256(0, 1, 0, 0)} (limb 1 is 2^64)"); + Console.WriteLine($" signed.ToBigInteger() == -2^255 {signed.ToBigInteger() == -BigInteger.Pow(2, 255)}"); + + // Round-tripping through BigInteger is how arithmetic is done: the struct has comparison operators + // but no +, -, * or /. + Int256 doubled = Int256.FromBigInteger(Int256.FromBigInteger(21).ToBigInteger() * 2); + Console.WriteLine($" 21 * 2 via BigInteger {doubled} (there is no Int256 operator *)"); + + Span raw = stackalloc byte[Int256.Size]; + signed.WriteLittleEndian(raw); + Console.WriteLine($" WriteLittleEndian {Convert.ToHexString(raw)}"); + Console.WriteLine($" ReadLittleEndian round trip {Int256.ReadLittleEndian(raw) == signed}"); + + break; + } + } + + private static async Task Decimals(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. Decimals: the declared precision decides the CLR type\n"); + Console.WriteLine(" A Decimal(P, S) is a signed integer mantissa of a width P chooses, and the value is"); + Console.WriteLine(" mantissa / 10^S. P up to 18 fits a System.Decimal; wider does not, so it surfaces as"); + Console.WriteLine(" ClickHouseTcpDecimal. That is decided by P alone, never by the value:\n"); + + await foreach (Block block in client.StreamAsync( + @"SELECT CAST('1.25', 'Decimal(18, 2)') AS at_18, CAST('1.25', 'Decimal(19, 2)') AS at_19")) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine($" {column.TypeName,-16} -> {Describe(column.ElementType),-22} value {column.GetValue(0)}"); + } + + Console.WriteLine(" Both hold 1.25. Only the declared precision differs."); + break; + } + + Console.WriteLine(); + Console.WriteLine(" ClickHouseTcpDecimal is the mantissa and the scale, unchanged from the wire:\n"); + + await foreach (Block block in client.StreamAsync($"SELECT d128, d256 FROM {TableName}")) + { + foreach (IColumn column in block.Columns) + { + var value = (ClickHouseTcpDecimal)column.GetValue(0); + bool narrows = value.TryToDecimal(out decimal narrowed); + Console.WriteLine($" {column.Name} {column.TypeName}"); + Console.WriteLine($" Mantissa {value.Mantissa}"); + Console.WriteLine($" Scale {value.Scale}, Sign {value.Sign}, ToString() {value}"); + Console.WriteLine($" TryToDecimal {narrows}{(narrows ? $" -> {narrowed}" : " (out of a System.Decimal's range)")}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" A System.Decimal holds a 96-bit mantissa and a scale of 0 to 28, so TryToDecimal fails"); + Console.WriteLine(" on either count — and ToDecimal() throws where TryToDecimal returns false:"); + + await foreach (Block block in client.StreamAsync( + @"SELECT CAST('1.25', 'Decimal(20, 2)') AS fits, + CAST('1234567890123456789012345678901.5', 'Decimal(38, 1)') AS mantissa_too_wide, + CAST('1.2345678901234567890123456789012345', 'Decimal(38, 34)') AS scale_too_deep")) + { + foreach (IColumn column in block.Columns) + { + var value = (ClickHouseTcpDecimal)column.GetValue(0); + Console.WriteLine($" {column.Name,-18} {column.TypeName,-16} TryToDecimal {value.TryToDecimal(out _),-5} {value}"); + } + + break; + } + + // Two values of different scale can be the same number, and comparison says so. + var oneDotZero = new ClickHouseTcpDecimal((Int128)10, 1); + var oneDotZeroZero = new ClickHouseTcpDecimal((Int128)100, 2); + Console.WriteLine(); + Console.WriteLine(" Equality and ordering compare the value, not the representation:"); + Console.WriteLine($" ClickHouseTcpDecimal(10, 1) == ClickHouseTcpDecimal(100, 2) {oneDotZero == oneDotZeroZero} ('{oneDotZero}' and '{oneDotZeroZero}')"); + Console.WriteLine($" FromDecimal(1.2500m) keeps the trailing zeros: '{ClickHouseTcpDecimal.FromDecimal(1.2500m)}', Scale {ClickHouseTcpDecimal.FromDecimal(1.2500m).Scale}"); + Console.WriteLine(); + Console.WriteLine(" ToString() is always the invariant fixed-point rendering with exactly Scale digits."); + Console.WriteLine(" The type implements IFormattable, but the format and the provider are ignored:"); + Console.WriteLine($" ToString(\"F3\", InvariantCulture) = '{oneDotZero.ToString("F3", CultureInfo.InvariantCulture)}' (not 1.000)"); + } + + private static async Task Floats(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. Floats, and BFloat16's missing mantissa\n"); + Console.WriteLine(" Float32 is a float and Float64 a double. BFloat16 is a float too — it is a float32 with"); + Console.WriteLine(" the low 16 mantissa bits cut off, so widening it is exact and there is nothing narrower"); + Console.WriteLine(" to hand back. What you lose is precision, on the way in:\n"); + + await foreach (Block block in client.StreamAsync($"SELECT f32, f64, bf16 FROM {TableName}")) + { + Console.WriteLine($" Float32 wrote 1.5f read {block.Column("f32")[0]}"); + Console.WriteLine($" Float64 wrote -2.25 read {block.Column("f64")[0]}"); + Console.WriteLine($" BFloat16 wrote 0.1f read {block.Column("bf16")[0]:R}"); + Console.WriteLine(" 7 stored mantissa bits, so 0.1 is not representable and the nearest value comes back."); + break; + } + } + + private static async Task StringsAndBytes(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. String is text, FixedString(N) is bytes\n"); + + await foreach (Block block in client.StreamAsync($"SELECT text, fixed5, id, ip4, ip6 FROM {TableName}")) + { + byte[] fixedBytes = block.Column("fixed5")[0]; + Console.WriteLine($" String -> string \"{block.Column("text")[0]}\""); + Console.WriteLine($" FixedString(5) -> byte[] {Convert.ToHexString(fixedBytes)} ({fixedBytes.Length} bytes)"); + Console.WriteLine(" No decoding and no trimming: an embedded 0x00 and a byte that is not valid UTF-8"); + Console.WriteLine(" both survive, which a string could not carry."); + Console.WriteLine($" UUID -> Guid {block.Column("id")[0]}"); + Console.WriteLine($" IPv4 -> IPAddress {block.Column("ip4")[0]}"); + Console.WriteLine($" IPv6 -> IPAddress {block.Column("ip6")[0]}"); + Console.WriteLine(" One CLR type for both, told apart by AddressFamily."); + break; + } + + await foreach (Block block in client.StreamAsync( + "SELECT toIPv4('10.0.0.1') AS four, toIPv6('10.0.0.1') AS six")) + { + Console.WriteLine(); + Console.WriteLine(" The same address in each column, and the family is what differs:"); + foreach (IColumn column in block.Columns) + { + var address = (IPAddress)column.GetValue(0)!; + Console.WriteLine($" {column.TypeName,-5} {address,-18} AddressFamily {address.AddressFamily}"); + } + + Console.WriteLine(" An IPv4 address in an IPv6 column is the mapped form, ::ffff:a.b.c.d."); + break; + } + + Console.WriteLine(); + Console.WriteLine(" A String is decoded as UTF-8, so a String column carrying arbitrary bytes is lossy:"); + + await foreach (Block block in client.StreamAsync( + "SELECT CAST(unhex('C3A9') AS String) AS valid, CAST(unhex('FFFE') AS String) AS invalid")) + { + foreach (IColumn column in block.Columns) + { + string text = (string)column.GetValue(0); + Console.WriteLine($" {column.Name,-8} = \"{text}\" -> re-encoded {Convert.ToHexString(System.Text.Encoding.UTF8.GetBytes(text))}"); + } + + Console.WriteLine(" 0xFFFE came back as two replacement characters. Use FixedString(N) for bytes."); + break; + } + + Console.WriteLine(); + Console.WriteLine(" An insert supplies exactly N bytes. Shorter is refused rather than padded:"); + + try + { + await client.InsertAsync( + $"INSERT INTO {TableName} (u64, fixed5) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("u64", new ulong[] { 2 }), + ClickHouseTcpColumn.Create("fixed5", new[] { new byte[] { 0x61, 0x62 } }), + }); + } + catch (ArgumentException ex) + { + Console.WriteLine($" {Wrap(ex.Message.Split(" (Parameter")[0])}"); + } + } + + private static async Task Enums(ClickHouseTcpClient client) + { + Console.WriteLine("\n6. An enum is its ordinal; the labels live in the type string\n"); + + await foreach (Block block in client.StreamAsync($"SELECT e8, e16 FROM {TableName}")) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine($" {column.Name,-4} {column.TypeName}"); + Console.WriteLine($" reads as {Describe(column.ElementType)} = {column.GetValue(0)}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" Enum8 is an Int8 ordinal and Enum16 an Int16 one, and no tier maps either back to its"); + Console.WriteLine(" label. The definition is in IColumn.TypeName, so the map is recoverable — but you parse"); + Console.WriteLine(" it, or you ask the server:"); + + object label = await client.ExecuteScalarAsync($"SELECT toString(e8) FROM {TableName} LIMIT 1"); + Console.WriteLine($" SELECT toString(e8) -> '{label}' (the server's own reverse lookup)"); + + Console.WriteLine(); + Console.WriteLine(" The POCO tier refuses a string property over an enum column rather than guessing:"); + + try + { + await foreach (EnumRow _ in client.QueryAsync($"SELECT e8 AS Colour FROM {TableName}")) + { + break; + } + } + catch (InvalidOperationException ex) + { + Console.WriteLine($" {Wrap(ex.Message)}"); + } + + Console.WriteLine(); + Console.WriteLine(" And an insert takes the ordinal, so map your own enum to its numeric value:"); + Console.WriteLine(" ClickHouseTcpColumn.Create(\"e8\", new sbyte[] { (sbyte)Colour.Green })"); + } + + private static async Task Nothing(ClickHouseTcpClient client) + { + Console.WriteLine("\n7. Nothing: the type of a value that has no type\n"); + Console.WriteLine(" The server gives an untyped NULL and an untyped empty array the Nothing type. It cannot"); + Console.WriteLine(" be a column of a table, so you only ever meet it in an expression's result:\n"); + + await foreach (Block block in client.StreamAsync("SELECT NULL AS nothing_at_all, [] AS empty_array")) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine($" {column.Name,-14} {column.TypeName,-18} reads as {Describe(column.ElementType),-10} value {Render(column.GetValue(0))}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" object is the element type because there is no value to have a type. The server refuses"); + Console.WriteLine(" to store one at all:"); + + try + { + await client.ExecuteAsync($"CREATE TABLE {TableName}_nothing (c Nothing) ENGINE = MergeTree ORDER BY tuple()"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}_nothing"); + Console.WriteLine(" accepted, which this example did not expect"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" {FirstLine(ex.Message)}"); + } + + Console.WriteLine(); + Console.WriteLine(" So a query whose column may be Nothing wants a CAST: SELECT CAST(NULL, 'Nullable(Int32)')."); + } + + // A POCO whose property type is deliberately wrong for the column, to show what the mapping reports. + private sealed class EnumRow + { + public string Colour { get; set; } = string.Empty; + } + + // The C# spelling of a CLR type, which is how a reader will write it. + private static string Describe(Type type) => type switch + { + _ when type == typeof(byte) => "byte", + _ when type == typeof(sbyte) => "sbyte", + _ when type == typeof(ushort) => "ushort", + _ when type == typeof(short) => "short", + _ when type == typeof(uint) => "uint", + _ when type == typeof(int) => "int", + _ when type == typeof(ulong) => "ulong", + _ when type == typeof(long) => "long", + _ when type == typeof(float) => "float", + _ when type == typeof(double) => "double", + _ when type == typeof(decimal) => "decimal", + _ when type == typeof(bool) => "bool", + _ when type == typeof(string) => "string", + _ when type == typeof(byte[]) => "byte[]", + _ when type == typeof(object) => "object", + _ when type == typeof(object[]) => "object[]", + _ => type.Name, + }; + + private static string Render(object? value) => value switch + { + null => "NULL", + byte[] bytes => "0x" + Convert.ToHexString(bytes), + string text => $"\"{text}\"", + bool flag => flag ? "true" : "false", + object[] { Length: 0 } => "[]", + float single => single.ToString("R", CultureInfo.InvariantCulture), + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? "NULL", + }; + + private static string FirstLine(string message) + { + int newline = message.IndexOf('\n'); + string line = newline < 0 ? message : message[..newline]; + return line.StartsWith("DB::Exception: ", StringComparison.Ordinal) ? line["DB::Exception: ".Length..] : line; + } + + // Reflows a long driver message so the console output stays readable. + private static string Wrap(string message) + { + var lines = new List(); + var line = new System.Text.StringBuilder(); + foreach (string word in message.Split(' ')) + { + if (line.Length + word.Length + 1 > 88) + { + lines.Add(line.ToString()); + line.Clear(); + } + + line.Append(line.Length == 0 ? word : " " + word); + } + + lines.Add(line.ToString()); + return string.Join("\n ", lines); + } +} diff --git a/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs b/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs new file mode 100644 index 000000000..ceb0771e2 --- /dev/null +++ b/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs @@ -0,0 +1,463 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// The six date and time types — Date, Date32, DateTime, DateTime64(scale), +/// Time, Time64(scale) — and the timezone model that decides what a value means. +/// +/// +/// Three facts carry the whole subject: +/// +/// +/// +/// A DateTime or DateTime64 stores an instant: a count of seconds (or of +/// 10^-scale seconds) since the Unix epoch, in UTC. A timezone in the type string changes no stored +/// byte. It decides only how that count is presented, and how a wall-clock value is turned into it. +/// +/// +/// When the type string names no timezone, the presentation timezone comes from the session_timezone +/// query setting, falling back to the server's own timezone. Section 3 measures it. +/// +/// +/// A DateTime whose Kind is Utc or Local, and any DateTimeOffset, names an +/// instant. On an insert that is lossless, because the target column's timezone is known. As a query +/// parameter it is refused, because a parameter travels as text with no timezone attached. Section 6. +/// +/// +/// +/// +/// Tcp_006 covers the block-tier mechanics of IDateTimeColumn and ITimeColumn; +/// Tcp_007 demonstrates the parameter refusal. This example is about what the values mean. +/// +/// +public static class TcpDateTimeAndTimezones +{ + private const string TableName = "example_tcp_datetime_timezones"; + private const string KindTable = "example_tcp_datetime_kinds"; + + // 2026-06-01 12:00:00 UTC. Europe/Amsterdam is +02:00 that day, Asia/Tokyo +09:00. + private const long NoonUtcSeconds = 1780315200; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await Seed(client); + await SixTypes(client); + await WhatTheWireCarries(client); + await WhereThePresentationTimezoneComesFrom(client); + await Scale(client); + await KindOnTheWritePath(client); + await KindOnTheParameterPath(client); + await TimeIsNotATimeOfDay(client); + WhatToRemember(); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {KindTable}"); + Console.WriteLine("\nDropped every table this example created."); + } + } + + private static async Task Seed(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + d Date, + d32 Date32, + dt DateTime, + dt_tz DateTime('Europe/Amsterdam'), + dt64 DateTime64(3), + dt64_tz DateTime64(9, 'Asia/Tokyo'), + t Time, + t64 Time64(3) + ) + ENGINE = MergeTree() + ORDER BY d"); + + await client.InsertAsync( + $"INSERT INTO {TableName} (d, d32, dt, dt_tz, dt64, dt64_tz, t, t64) VALUES", + new IColumn[] + { + // A Date is a day number, so it takes a DateOnly and nothing else — not a DateTime. + ClickHouseTcpColumn.Create("d", new[] { new DateOnly(2026, 6, 1) }), + ClickHouseTcpColumn.Create("d32", new[] { new DateOnly(1920, 3, 4) }), + + // Kind=Utc names the instant, which every one of these four columns then stores exactly. + ClickHouseTcpColumn.Create("dt", new[] { DateTime.UnixEpoch.AddSeconds(NoonUtcSeconds) }), + ClickHouseTcpColumn.Create("dt_tz", new[] { DateTime.UnixEpoch.AddSeconds(NoonUtcSeconds) }), + ClickHouseTcpColumn.Create("dt64", new[] { DateTime.UnixEpoch.AddSeconds(NoonUtcSeconds).AddMilliseconds(123) }), + + // A DateTime cannot hold nanoseconds, so the raw count goes in directly. Every one of these + // columns also accepts the integer the wire carries. + ClickHouseTcpColumn.Create("dt64_tz", new[] { (NoonUtcSeconds * 1_000_000_000L) + 123456789L }), + + // A Time is a count from midnight, so it takes a TimeSpan, which can also be negative or + // longer than a day. A TimeOnly cannot express either and is not accepted. + ClickHouseTcpColumn.Create("t", new[] { new TimeSpan(12, 34, 56) }), + ClickHouseTcpColumn.Create("t64", new[] { new TimeSpan(0, 12, 34, 56, 789) }), + }); + + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + Console.WriteLine($"Server {server.Version}, handshake timezone '{server.Timezone}'."); + Console.WriteLine($"Seeded '{TableName}' with one row of each of the six types."); + } + + private static async Task SixTypes(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. The six types, what they store, and what reads them\n"); + Console.WriteLine(" ClickHouse type IColumn The raw count Extra interface"); + Console.WriteLine(" ---------------------------- ---------- ------------------- ---------------"); + + await foreach (Block block in client.StreamAsync($"SELECT * FROM {TableName}")) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine( + $" {column.TypeName,-28} {Describe(column.ElementType),-10} {Raw(column.GetValue(0)),-19} {Extra(column)}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" Date and Date32 are the two that already read as a calendar type: a day number needs no"); + Console.WriteLine(" timezone and no scale, so DateOnly loses nothing and there is no interface to add. The"); + Console.WriteLine(" other four read as the integer the wire carried, and the interface converts it."); + } + + private static async Task WhatTheWireCarries(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. What each count counts\n"); + + await foreach (Block block in client.StreamAsync($"SELECT * FROM {TableName}")) + { + var dt = (IDateTimeColumn)block["dt"]; + var dt64 = (IDateTimeColumn)block["dt64_tz"]; + var t = (ITimeColumn)block["t"]; + + Console.WriteLine($" Date {Raw(block.Column("d")[0]),-19} days since 1970-01-01, unsigned 16-bit"); + Console.WriteLine($" Date32 {Raw(block.Column("d32")[0]),-19} days since 1970-01-01, signed 32-bit, so it reaches before the epoch"); + Console.WriteLine($" DateTime {block.Column("dt")[0],-19} seconds since the epoch, UTC"); + Console.WriteLine($" DateTime64(9) {block.Column("dt64_tz")[0],-19} nanoseconds since the epoch, UTC"); + Console.WriteLine($" Time {block.Column("t")[0],-19} seconds from midnight, signed"); + Console.WriteLine($" Time64(3) {block.Column("t64")[0],-19} milliseconds from midnight, signed"); + + Console.WriteLine(); + Console.WriteLine(" For the two DateTime families that count is a UTC instant. The timezone the column"); + Console.WriteLine(" declares changes no stored byte, only the reading:"); + Console.WriteLine($" dt {block["dt"].TypeName,-30} count {block.Column("dt")[0]} -> {Format(dt.GetDateTimeOffset(0))}"); + Console.WriteLine($" dt_tz {block["dt_tz"].TypeName,-30} count {block.Column("dt_tz")[0]} -> {Format(((IDateTimeColumn)block["dt_tz"]).GetDateTimeOffset(0))}"); + Console.WriteLine(" Same count, different offset. One instant, two presentations."); + + Console.WriteLine(); + Console.WriteLine(" A Time carries no timezone at all, because it is not an instant:"); + Console.WriteLine($" t {block["t"].TypeName,-30} Scale {t.Scale}, GetTimeSpan(0) {t.GetTimeSpan(0)}"); + Console.WriteLine($" dt64_tz {block["dt64_tz"].TypeName,-30} Scale {dt64.Scale}, TimeZone {dt64.TimeZone.Id}"); + + break; + } + } + + private static async Task WhereThePresentationTimezoneComesFrom(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. Where the presentation timezone comes from\n"); + Console.WriteLine(" Measured, not assumed. The same query runs once per session_timezone over one fixed"); + Console.WriteLine(" instant, with a bare DateTime and a DateTime('Europe/Amsterdam') side by side:\n"); + Console.WriteLine(" session_timezone DateTime count bare presented as declared presented as"); + Console.WriteLine(" -------------------- -------------- ------------------------------ ------------------------------"); + + string sql = $@"SELECT toDateTime({NoonUtcSeconds}) AS bare, + toDateTime({NoonUtcSeconds}, 'Europe/Amsterdam') AS declared"; + + foreach (string zone in new[] { string.Empty, "UTC", "Europe/Amsterdam", "Asia/Tokyo", "America/Los_Angeles" }) + { + ClickHouseTcpQueryOptions? options = zone.Length == 0 + ? null + : new ClickHouseTcpQueryOptions { Settings = new Dictionary { ["session_timezone"] = zone } }; + + await foreach (Block block in client.StreamAsync(sql, options)) + { + var bare = (IDateTimeColumn)block["bare"]; + var declared = (IDateTimeColumn)block["declared"]; + Console.WriteLine( + $" {(zone.Length == 0 ? "(not set)" : zone),-20} {block.Column("bare")[0],-14} {Format(bare.GetDateTimeOffset(0)),-30} {Format(declared.GetDateTimeOffset(0))}"); + break; + } + } + + Console.WriteLine(); + Console.WriteLine(" What that shows:"); + Console.WriteLine(" The stored count never moves. It is the same instant in every row."); + Console.WriteLine(" A bare DateTime is presented in the session timezone, and IDateTimeColumn.TimeZone"); + Console.WriteLine(" reports that zone."); + Console.WriteLine(" A DateTime('Europe/Amsterdam') ignores the setting entirely. The type string wins."); + Console.WriteLine(" With the setting unset, the presentation zone is the server's own — the one the"); + Console.WriteLine(" handshake reported, printed at the top of this example."); + Console.WriteLine(); + Console.WriteLine(" So the presentation timezone is: the type string's, or else session_timezone, or else the"); + Console.WriteLine(" server's. The server also sends a TimezoneUpdate packet on the wire; it is not what the"); + Console.WriteLine(" client resolves a bare column against."); + Console.WriteLine(); + Console.WriteLine(" The practical consequence: declare the timezone on any column you care about. A bare"); + Console.WriteLine(" DateTime read by two callers with different session settings gives two different"); + Console.WriteLine(" DateTimeOffsets — correctly, since they are the same instant, but a DateTime with"); + Console.WriteLine(" Kind=Unspecified taken from one of them is not comparable with the other's."); + } + + private static async Task Scale(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. Scale: DateTime64(0..9), and where .NET stops\n"); + Console.WriteLine(" The scale is how many decimal digits of a second the count carries. A .NET tick is"); + Console.WriteLine(" 100 ns, which is scale 7, so scales 8 and 9 hold digits DateTimeOffset cannot:\n"); + Console.WriteLine(" Type Raw count GetDateTimeOffset(0)"); + Console.WriteLine(" -------------------- --------------------- ------------------------------"); + + await foreach (Block block in client.StreamAsync( + @"SELECT toDateTime64('2026-06-01 12:00:00.123456789', 0, 'UTC') AS s0, + toDateTime64('2026-06-01 12:00:00.123456789', 3, 'UTC') AS s3, + toDateTime64('2026-06-01 12:00:00.123456789', 6, 'UTC') AS s6, + toDateTime64('2026-06-01 12:00:00.123456789', 7, 'UTC') AS s7, + toDateTime64('2026-06-01 12:00:00.123456789', 9, 'UTC') AS s9")) + { + foreach (IColumn column in block.Columns) + { + var instants = (IDateTimeColumn)column; + Console.WriteLine($" {column.TypeName,-20} {column.GetValue(0),-21} {instants.GetDateTimeOffset(0):yyyy-MM-dd HH:mm:ss.fffffff}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" Scale 9's last two digits (89) are gone from the DateTimeOffset and still present in the"); + Console.WriteLine(" count. Read IColumn.Values when you need them; the truncation is toward zero."); + Console.WriteLine(); + Console.WriteLine(" Time64 has the same scale range and the same limit: ITimeColumn.GetTimeSpan truncates to"); + Console.WriteLine(" ticks, and IColumn keeps the count."); + } + + private static async Task KindOnTheWritePath(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. DateTime.Kind on the way in\n"); + Console.WriteLine(" A .NET DateTime is a number plus a Kind, and the Kind is what says whether the number is"); + Console.WriteLine(" an instant or a wall-clock reading. An insert honours it, because the target column's"); + Console.WriteLine(" timezone comes from the schema the server sent, so the conversion is never a guess.\n"); + Console.WriteLine($" The host's local zone is {TimeZoneInfo.Local.Id}. Same 12:00 in each case:\n"); + + var noon = new DateTime(2026, 6, 1, 12, 0, 0); + var values = new (string What, object Value)[] + { + ("Kind=Utc", DateTime.SpecifyKind(noon, DateTimeKind.Utc)), + ("Kind=Unspecified", DateTime.SpecifyKind(noon, DateTimeKind.Unspecified)), + ("Kind=Local", DateTime.SpecifyKind(noon, DateTimeKind.Local)), + ("DateTimeOffset +05:00", new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.FromHours(5))), + }; + + foreach (string columnType in new[] { "DateTime('UTC')", "DateTime('Europe/Amsterdam')" }) + { + Console.WriteLine($" Target column {columnType}:"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {KindTable}"); + await client.ExecuteAsync($"CREATE TABLE {KindTable} (t {columnType}) ENGINE = MergeTree ORDER BY tuple()"); + + foreach ((string what, object value) in values) + { + IColumn column = value is DateTimeOffset offset + ? ClickHouseTcpColumn.Create("t", new[] { offset }) + : ClickHouseTcpColumn.Create("t", new[] { (DateTime)value }); + + await client.InsertAsync($"INSERT INTO {KindTable} (t) VALUES", new[] { column }); + + await foreach (Block block in client.StreamAsync($"SELECT t FROM {KindTable}")) + { + var stored = (IDateTimeColumn)block["t"]; + Console.WriteLine($" {what,-22} -> count {block.Column("t")[0]}, presented {Format(stored.GetDateTimeOffset(0))}"); + break; + } + + await client.ExecuteAsync($"TRUNCATE TABLE {KindTable}"); + } + } + + Console.WriteLine(); + Console.WriteLine(" Reading those two blocks together:"); + Console.WriteLine(" Kind=Utc and DateTimeOffset name an instant, so the count is the same whichever column"); + Console.WriteLine(" they go into. Lossless."); + Console.WriteLine(" Kind=Unspecified is a wall clock, read in the COLUMN's timezone — the count differs"); + Console.WriteLine(" between the two targets by the offset. Lossless, and it means what you want when the"); + Console.WriteLine(" value came from a source that had no timezone."); + Console.WriteLine(" Kind=Local is a wall clock read in the HOST's timezone, not the column's. Correct, and"); + Console.WriteLine(" it makes the stored value depend on where your process runs. Prefer Utc or"); + Console.WriteLine(" Unspecified in anything that is deployed more than once."); + Console.WriteLine(); + Console.WriteLine(" A read produces Kind=Utc for a zero-offset column and Kind=Unspecified otherwise, so"); + Console.WriteLine(" an insert of a read value is lossless only if the two columns share a timezone. Take"); + Console.WriteLine(" DateTimeOffset from IDateTimeColumn.GetDateTimeOffset instead, which is unambiguous."); + } + + private static async Task KindOnTheParameterPath(ClickHouseTcpClient client) + { + Console.WriteLine("\n6. The same value as a query parameter: an instant needs a declared timezone\n"); + Console.WriteLine(" A parameter does not travel as a count. It travels as text in the Query packet's"); + Console.WriteLine(" settings list, and the text carries no timezone, so the server reads it in whatever"); + Console.WriteLine(" session_timezone is in force — which section 3 just showed is not something the client"); + Console.WriteLine(" controls. An instant would therefore move silently, so it is refused:\n"); + + var noonUtc = DateTime.SpecifyKind(new DateTime(2026, 6, 1, 12, 0, 0), DateTimeKind.Utc); + + try + { + await client.ExecuteScalarAsync( + "SELECT {t:DateTime}", + new ClickHouseTcpQueryOptions { Parameters = new ClickHouseTcpParameterCollection { { "t", noonUtc } } }); + Console.WriteLine(" accepted, which this example did not expect"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" {{t:DateTime}} with Kind=Utc:"); + Console.WriteLine($" {Wrap(ex.Message.Split(" (Parameter")[0])}"); + } + + Console.WriteLine(); + Console.WriteLine(" Declaring the timezone in the placeholder makes it lossless, because the client can"); + Console.WriteLine(" then move the instant into that zone before writing the text:"); + + foreach ((string placeholder, object value, string note) in new (string, object, string)[] + { + ("{t:DateTime('UTC')}", noonUtc, "Kind=Utc, declared UTC"), + ("{t:DateTime('Asia/Tokyo')}", noonUtc, "Kind=Utc, declared Tokyo — same instant, +09:00 wall clock"), + ("{t:DateTime('UTC')}", new DateTimeOffset(2026, 6, 1, 17, 0, 0, TimeSpan.FromHours(5)), "DateTimeOffset +05:00 — the same instant again"), + ("{t:DateTime}", DateTime.SpecifyKind(new DateTime(2026, 6, 1, 12, 0, 0), DateTimeKind.Unspecified), "Kind=Unspecified — a wall clock, so no timezone is needed"), + }) + { + object? epoch = await client.ExecuteScalarAsync( + $"SELECT toUnixTimestamp(toDateTime({placeholder}, 'UTC'))", + new ClickHouseTcpQueryOptions { Parameters = new ClickHouseTcpParameterCollection { { "t", value } } }); + Console.WriteLine($" {placeholder,-27} -> {epoch} {note}"); + } + + Console.WriteLine(); + Console.WriteLine(" The last row is the one to notice: with Kind=Unspecified the count is whatever the"); + Console.WriteLine(" session timezone makes of 12:00, so it agrees with the others only because this session"); + Console.WriteLine(" is UTC. That is exactly the ambiguity the refusal above protects an instant from."); + Console.WriteLine(); + Console.WriteLine(" Same rule for DateTime64: {t:DateTime64(3, 'UTC')} declares one, {t:DateTime64(3)} does"); + Console.WriteLine(" not. Date, Date32, Time and Time64 have no timezone to declare, so none of this applies"); + Console.WriteLine(" to them."); + } + + private static async Task TimeIsNotATimeOfDay(ClickHouseTcpClient client) + { + Console.WriteLine("\n7. Time is a duration from midnight, not a time of day\n"); + Console.WriteLine(" The count is signed and is not reduced modulo a day, so a Time holds values no clock"); + Console.WriteLine(" face has. That is why the CLR type is TimeSpan and not TimeOnly:\n"); + Console.WriteLine(" Literal Raw count GetTimeSpan(0)"); + Console.WriteLine(" ---------------- ---------- --------------"); + + await foreach (Block block in client.StreamAsync( + @"SELECT CAST('12:34:56', 'Time') AS ordinary, + CAST('-01:30:00', 'Time') AS negative, + CAST('999:00:00', 'Time') AS past_a_day, + CAST('12:34:56.789', 'Time64(3)') AS with_millis")) + { + foreach (IColumn column in block.Columns) + { + var times = (ITimeColumn)column; + Console.WriteLine($" {column.Name,-16} {column.GetValue(0),-10} {times.GetTimeSpan(0)}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" A TimeOnly is refused on an insert for the same reason — it can express neither:"); + + await client.ExecuteAsync($"DROP TABLE IF EXISTS {KindTable}"); + await client.ExecuteAsync($"CREATE TABLE {KindTable} (t Time) ENGINE = MergeTree ORDER BY tuple()"); + + try + { + await client.InsertAsync( + $"INSERT INTO {KindTable} (t) VALUES", + new[] { ClickHouseTcpColumn.Create("t", new[] { new TimeOnly(12, 34, 56) }) }); + Console.WriteLine(" accepted, which this example did not expect"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" {Wrap(ex.Message.Split(" (Parameter")[0])}"); + } + + Console.WriteLine(); + Console.WriteLine(" Pass a TimeSpan, or the raw count as an int (Time) or a long (Time64)."); + } + + private static void WhatToRemember() + { + Console.WriteLine("\n8. What to remember\n"); + Console.WriteLine(" Declare the timezone on a DateTime or DateTime64 column you care about. Without one the"); + Console.WriteLine(" reading depends on session_timezone, which is set per query and not by you."); + Console.WriteLine(" Read instants through IDateTimeColumn.GetDateTimeOffset, not through a DateTime. An"); + Console.WriteLine(" offset is never ambiguous; a DateTime's Kind is Unspecified for any non-UTC column."); + Console.WriteLine(" Write Kind=Utc or a DateTimeOffset for an instant, Kind=Unspecified for a wall clock."); + Console.WriteLine(" Avoid Kind=Local unless the host's zone really is part of the value's meaning."); + Console.WriteLine(" A parameter that names an instant needs {t:DateTime('Zone')}. An insert does not, and"); + Console.WriteLine(" that difference is not a bug: only one of the two carries the column's timezone."); + Console.WriteLine(" Date and Date32 are DateOnly, Time and Time64 are TimeSpan, and none of the four has a"); + Console.WriteLine(" timezone at all."); + Console.WriteLine(" Keep the raw count when the scale is 8 or 9, or when the precision matters more than the"); + Console.WriteLine(" calendar type: it is what the wire carried and it truncates nothing."); + } + + private static string Describe(Type type) => type switch + { + _ when type == typeof(uint) => "uint", + _ when type == typeof(int) => "int", + _ when type == typeof(long) => "long", + _ => type.Name, + }; + + private static string Extra(IColumn column) => column switch + { + IDateTimeColumn instants => $"IDateTimeColumn (TimeZone {instants.TimeZone.Id}, Scale {instants.Scale})", + ITimeColumn times => $"ITimeColumn (Scale {times.Scale}, no timezone)", + _ => "- (already a calendar type)", + }; + + private static string Format(DateTimeOffset value) + => value.ToString("yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture); + + // The boxed wire value, rendered culture-invariantly so the output does not depend on the host. + private static string Raw(object? value) => value switch + { + DateOnly day => day.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + null => "NULL", + _ => value.ToString() ?? "NULL", + }; + + // Reflows a long driver message so the console output stays readable. + private static string Wrap(string message) + { + var lines = new List(); + var line = new System.Text.StringBuilder(); + foreach (string word in message.Split(' ')) + { + if (line.Length + word.Length + 1 > 90) + { + lines.Add(line.ToString()); + line.Clear(); + } + + line.Append(line.Length == 0 ? word : " " + word); + } + + lines.Add(line.ToString()); + return string.Join("\n ", lines); + } +} diff --git a/examples/Tcp/Types/Tcp_013_CompositeRead.cs b/examples/Tcp/Types/Tcp_013_CompositeRead.cs new file mode 100644 index 000000000..9c2ace2d6 --- /dev/null +++ b/examples/Tcp/Types/Tcp_013_CompositeRead.cs @@ -0,0 +1,574 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// Reading the composite types through their typed views: , +/// , , , +/// and — how they nest, and what the geo +/// aliases resolve to. +/// +/// +/// Two things decide how you write the pattern match. First, the view's type argument is the wire's +/// element type, not the row's: a Nullable(Int32) reads as int? but its view is +/// INullableColumn<int>, and a LowCardinality(Nullable(String)) is +/// ILowCardinalityColumn<string>. Second, a composite's child is a column in its own right, so +/// reaching into a nested composite is another pattern match rather than an index into a materialized value. +/// +/// +/// +/// Tcp_006 covers the block tier itself and IArrayColumn in particular; Tcp_010 covers +/// writing these shapes. This example is about the types. +/// +/// +public static class TcpCompositeRead +{ + private const string TableName = "example_tcp_composite_read"; + private const string NestedTable = "example_tcp_composite_read_nested"; + + private const string Columns = + "id, readings, attrs, point, named_point, score, city, nick, matrix, tagged, buckets"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await Seed(client); + await WhichViewEachCompositeOffers(client); + await MapsAndArrays(client); + await Tuples(client); + await Nulls(client); + await LowCardinalities(client); + await Nesting(client); + await NestedColumns(client); + await GeoAliases(client); + await Geometry(client); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {NestedTable}"); + Console.WriteLine("\nDropped every table this example created."); + } + } + + private static async Task Seed(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {TableName} + ( + id UInt64, + readings Array(Float64), + attrs Map(String, Int64), + point Tuple(Float64, Float64), + named_point Tuple(x Int32, y String), + score Nullable(Float64), + city LowCardinality(String), + nick LowCardinality(Nullable(String)), + matrix Array(Array(Int32)), + tagged Array(Tuple(Int32, String)), + buckets Map(String, Array(Int32)) + ) + ENGINE = MergeTree() + ORDER BY id"); + + await client.InsertAsync( + $"INSERT INTO {TableName} ({Columns}) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3 }), + ClickHouseTcpColumn.Create("readings", new[] { new[] { 0.5, 0.75 }, Array.Empty(), new[] { 1.5 } }), + ClickHouseTcpColumn.Create("attrs", new[] + { + new[] { new KeyValuePair("floor", 3), new KeyValuePair("room", 12) }, + Array.Empty>(), + + // The wire carries keys in order, duplicates and all, which is why a row is a pair array + // rather than a Dictionary. + new[] { new KeyValuePair("floor", 1), new KeyValuePair("floor", 2) }, + }), + ClickHouseTcpColumn.Create("point", new[] { (1.0, 2.0), (3.0, 4.0), (5.0, 6.0) }), + ClickHouseTcpColumn.Create("named_point", new[] { (10, "ten"), (20, "twenty"), (30, "thirty") }), + ClickHouseTcpColumn.Create("score", new double?[] { 1.25, null, 3.5 }), + ClickHouseTcpColumn.Create("city", new[] { "Amsterdam", "Amsterdam", "Reykjavik" }), + ClickHouseTcpColumn.Create("nick", new string?[] { "ada", null, "ada" }), + ClickHouseTcpColumn.Create("matrix", new[] { new[] { new[] { 1, 2 }, new[] { 3 } }, Array.Empty(), new[] { new[] { 4 } } }), + ClickHouseTcpColumn.Create("tagged", new[] + { + new[] { (1, "a"), (2, "b") }, + Array.Empty<(int, string)>(), + new[] { (3, "c") }, + }), + ClickHouseTcpColumn.Create("buckets", new[] + { + new[] { new KeyValuePair("evens", new[] { 2, 4 }) }, + Array.Empty>(), + new[] { new KeyValuePair("odds", new[] { 1, 3, 5 }) }, + }), + }); + + // Nested has to be created with flatten_nested = 0 to stay one column rather than becoming one + // Array(T) per field, and the client cannot build one from CLR values, so this one is seeded in SQL. + var oneColumnNested = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["flatten_nested"] = "0" }, + }; + + await client.ExecuteAsync( + $@"CREATE TABLE {NestedTable} (id UInt64, items Nested(sku String, qty UInt32)) + ENGINE = MergeTree() ORDER BY id", + oneColumnNested); + + await client.ExecuteAsync($"INSERT INTO {NestedTable} VALUES (1, [('bolt', 2), ('nut', 3)]), (2, []), (3, [('washer', 7)])"); + + Console.WriteLine($"Seeded '{TableName}' with 3 rows of every composite, and '{NestedTable}' with a Nested column."); + } + + private static async Task WhichViewEachCompositeOffers(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. What each composite reads as, and which view it offers\n"); + Console.WriteLine(" ClickHouse type One row is Pattern-matches to"); + Console.WriteLine(" -------------------------------- ------------------------------ ------------------------------"); + + await foreach (Block block in client.StreamAsync($"SELECT {Columns} FROM {TableName} ORDER BY id")) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine($" {column.TypeName,-32} {Describe(column.ElementType),-30} {View(column)}"); + } + + break; + } + + await foreach (Block block in client.StreamAsync($"SELECT items FROM {NestedTable} ORDER BY id")) + { + IColumn column = block["items"]; + Console.WriteLine($" {column.TypeName,-32} {Describe(column.ElementType),-30} {View(column)}"); + break; + } + + Console.WriteLine(); + Console.WriteLine(" The type argument of a view is the wire's element type, which is not always the row's:"); + Console.WriteLine(" Nullable(Float64) reads double?, view INullableColumn"); + Console.WriteLine(" LowCardinality(Nullable(String)) reads string, view ILowCardinalityColumn"); + Console.WriteLine(" ITupleColumn, INestedColumn, IVariantColumn, IDynamicColumn and IQBitColumn are not"); + Console.WriteLine(" generic at all, so those five need no type argument to match on."); + } + + private static async Task MapsAndArrays(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. Map(K, V): two flat columns plus offsets\n"); + Console.WriteLine(" A Map is byte-identical to Array(Tuple(K, V)), so its view is an Array's shape with the"); + Console.WriteLine(" run split in two. Row i's entries are Offsets[i] to Offsets[i + 1] in both columns:\n"); + + await foreach (Block block in client.StreamAsync($"SELECT id, attrs FROM {TableName} ORDER BY id")) + { + if (block["attrs"] is IMapColumn attrs) + { + ReadOnlySpan offsets = attrs.Offsets; + IColumn keys = attrs.KeyColumn; + IColumn values = attrs.ValueColumn; + + Console.WriteLine($" Offsets = [{string.Join(", ", offsets.ToArray())}] ({offsets.Length} entries for {attrs.RowCount} rows)"); + Console.WriteLine($" KeyColumn = [{string.Join(", ", keys.Values.ToArray())}] RowCount {keys.RowCount}, the total entry count"); + Console.WriteLine($" ValueColumn = [{string.Join(", ", values.Values.ToArray())}]"); + Console.WriteLine(); + + for (int row = 0; row < attrs.RowCount; row++) + { + var pairs = new List(); + for (int entry = offsets[row]; entry < offsets[row + 1]; entry++) + { + pairs.Add($"{keys[entry]}={values[entry]}"); + } + + Console.WriteLine($" row {row}: {(pairs.Count == 0 ? "(empty)" : string.Join(", ", pairs))}"); + } + + Console.WriteLine(); + Console.WriteLine(" Row 2 has 'floor' twice. The two columns keep it, entry order and all, which is"); + Console.WriteLine(" what a Dictionary could not do — and the reason the materialized row is a"); + Console.WriteLine($" KeyValuePair[]: attrs[2] = [{string.Join(", ", attrs[2].Select(p => $"{p.Key}={p.Value}"))}]"); + Console.WriteLine(); + Console.WriteLine(" Taking only the keys, or only the values, therefore costs nothing:"); + Console.WriteLine($" distinct keys across every row = {string.Join(", ", keys.Values.ToArray().Distinct())}"); + } + + break; + } + } + + private static async Task Tuples(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. Tuple(...): one child column per element, and the names are metadata\n"); + + await foreach (Block block in client.StreamAsync($"SELECT point, named_point FROM {TableName} ORDER BY id")) + { + foreach (IColumn column in block.Columns) + { + var tuple = (ITupleColumn)column; + Console.WriteLine($" {column.TypeName}"); + Console.WriteLine($" Children [{string.Join(", ", tuple.Children.Select(child => $"{child.TypeName} as {Describe(child.ElementType)}"))}]"); + Console.WriteLine($" FieldNames {(tuple.FieldNames is null ? "null — the tuple carries no names at all" : "[" + string.Join(", ", tuple.FieldNames.Select(name => name ?? "(unnamed)")) + "]")}"); + Console.WriteLine($" row 0 {Render(column.GetValue(0))}"); + } + + Console.WriteLine(); + Console.WriteLine(" FieldNames is null for an unnamed tuple, so check it before enumerating; a partly"); + Console.WriteLine(" named tuple gives a list with a null entry per unnamed element."); + Console.WriteLine(); + Console.WriteLine(" The names never reach the value. A named Tuple materializes as a plain ValueTuple, so"); + Console.WriteLine(" read one element without building the pair by going through Children:"); + + var named = (ITupleColumn)block["named_point"]; + IColumn xs = (IColumn)named.Children[0]; + Console.WriteLine($" Children[0].Values = [{string.Join(", ", xs.Values.ToArray())}] (the x of every row, no ValueTuple built)"); + + break; + } + } + + private static async Task Nulls(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. Nullable(T): a null map plus a full-height inner column\n"); + + await foreach (Block block in client.StreamAsync($"SELECT score FROM {TableName} ORDER BY id")) + { + // The type argument is double, not double?: it is the inner column's element type. + if (block["score"] is INullableColumn score) + { + ReadOnlySpan nulls = score.NullMap; + IColumn inner = score.Inner; + + Console.WriteLine($" {block["score"].TypeName}, read as {Describe(block["score"].ElementType)}, view INullableColumn"); + Console.WriteLine($" NullMap = [{string.Join(", ", nulls.ToArray())}] one byte per row, 1 means NULL"); + Console.WriteLine($" Inner.Values = [{string.Join(", ", inner.Values.ToArray())}] full height, with a placeholder where the row is NULL"); + Console.WriteLine(); + Console.WriteLine(" The two are indexed by the same row number, so a null-aware read is one branch:"); + + for (int row = 0; row < score.RowCount; row++) + { + Console.WriteLine($" row {row}: {(nulls[row] != 0 ? "NULL" : inner[row].ToString(CultureInfo.InvariantCulture))}"); + } + + Console.WriteLine(); + Console.WriteLine(" Do not read Inner without the null map. The value at a NULL position is the inner"); + Console.WriteLine(" codec's placeholder, not data — here it is 0, which is a perfectly plausible score."); + } + + break; + } + } + + private static async Task LowCardinalities(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. LowCardinality(T): a dictionary plus one key per row\n"); + Console.WriteLine(" This is the view that changes what an algorithm costs. The materialized surface resolves"); + Console.WriteLine(" every row to its entry, so a million rows over a five-entry dictionary materializes a"); + Console.WriteLine(" million values; grouping on the keys instead touches each distinct value once.\n"); + + await foreach (Block block in client.StreamAsync($"SELECT city, nick FROM {TableName} ORDER BY id")) + { + foreach (string name in new[] { "city", "nick" }) + { + if (block[name] is ILowCardinalityColumn lc) + { + // The reserved slots hold the inner codec's placeholder, which for a String is the empty + // string — indistinguishable from data unless they are labelled. + string[] slots = lc.Dictionary.Values.ToArray() + .Select((value, slot) => slot < lc.ReservedSlotCount + ? (slot == 0 && lc.ReservedSlotCount == 2 ? "" : "") + : $"'{value}'") + .ToArray(); + + Console.WriteLine($" {block[name].TypeName}"); + Console.WriteLine($" Dictionary [{string.Join(", ", slots)}] RowCount {lc.Dictionary.RowCount}"); + Console.WriteLine($" Keys [{string.Join(", ", lc.Keys.ToArray())}] one per row, an index into it"); + Console.WriteLine($" ReservedSlotCount {lc.ReservedSlotCount} (so data starts at slot {lc.ReservedSlotCount})"); + + for (int row = 0; row < lc.RowCount; row++) + { + bool isNull = lc.ReservedSlotCount == 2 && lc.Keys[row] == 0; + Console.WriteLine($" row {row}: key {lc.Keys[row]} -> {(isNull ? "NULL" : $"'{lc.Dictionary[lc.Keys[row]]}'")}"); + } + } + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" ReservedSlotCount is the whole reason to have that property: the leading dictionary slots"); + Console.WriteLine(" are not data. It is 1 for a non-nullable inner (slot 0 is the inner default) and 2 for a"); + Console.WriteLine(" nullable one (slot 0 is the NULL marker, slot 1 the default). So a key of 0 means NULL for"); + Console.WriteLine(" one shape and an ordinary default for the other, and reading the property is how you tell"); + Console.WriteLine(" them apart without parsing TypeName."); + Console.WriteLine(); + Console.WriteLine(" The dictionary is per block, not per column or per table, so the same value can have a"); + Console.WriteLine(" different key in the next block of the same result."); + } + + private static async Task Nesting(ClickHouseTcpClient client) + { + Console.WriteLine("\n6. Nesting: a child is a column, so you match again\n"); + + await foreach (Block block in client.StreamAsync($"SELECT matrix, tagged, buckets FROM {TableName} ORDER BY id")) + { + Console.WriteLine($" {block["matrix"].TypeName}: an array whose Inner is another array"); + if (block["matrix"] is IArrayColumn matrix && matrix.Inner is IArrayColumn rows) + { + Console.WriteLine($" outer Offsets [{string.Join(", ", matrix.Offsets.ToArray())}]"); + Console.WriteLine($" inner Offsets [{string.Join(", ", rows.Offsets.ToArray())}]"); + Console.WriteLine($" inner InnerValues [{string.Join(", ", rows.InnerValues.ToArray())}] every element of every sub-array, flat"); + Console.WriteLine(" Two offset levels over one flat run, so a sum over the whole column needs no"); + Console.WriteLine($" array at all: total {Sum(rows.InnerValues)}"); + } + + Console.WriteLine(); + Console.WriteLine($" {block["tagged"].TypeName}: an array whose Inner is a tuple"); + if (block["tagged"] is IArrayColumn<(int, string)> tagged && tagged.Inner is ITupleColumn pairs) + { + Console.WriteLine($" Offsets [{string.Join(", ", tagged.Offsets.ToArray())}]"); + Console.WriteLine($" Inner is ITupleColumn with children [{string.Join(", ", pairs.Children.Select(c => c.TypeName))}]"); + Console.WriteLine($" Inner.Children[1].Values = [{string.Join(", ", ((IColumn)pairs.Children[1]).Values.ToArray())}] every tag, no tuple built"); + } + + Console.WriteLine(); + Console.WriteLine($" {block["buckets"].TypeName}: a map whose ValueColumn is an array"); + if (block["buckets"] is IMapColumn buckets && buckets.ValueColumn is IArrayColumn lists) + { + Console.WriteLine($" Offsets [{string.Join(", ", buckets.Offsets.ToArray())}]"); + Console.WriteLine($" KeyColumn.Values [{string.Join(", ", buckets.KeyColumn.Values.ToArray())}]"); + Console.WriteLine($" ValueColumn is IArrayColumn, Offsets [{string.Join(", ", lists.Offsets.ToArray())}], InnerValues [{string.Join(", ", lists.InnerValues.ToArray())}]"); + } + + Console.WriteLine(); + Console.WriteLine(" Composites nest as deep as the server lets them, with no materialization at any level."); + Console.WriteLine(" The one thing to know is that each match needs the child's element type spelled out,"); + Console.WriteLine(" which IColumn.ElementType on the parent tells you: Array(Array(Int32)) reports int[][],"); + Console.WriteLine(" so the outer view is IArrayColumn and the inner one IArrayColumn."); + + break; + } + } + + private static async Task NestedColumns(ClickHouseTcpClient client) + { + Console.WriteLine("\n7. Nested(...): named fields over shared offsets\n"); + Console.WriteLine(" A Nested column is byte-identical to Array(Tuple(...)) and differs only in keeping the"); + Console.WriteLine(" field names. Its view is by name rather than by position, and is not generic:\n"); + + await foreach (Block block in client.StreamAsync($"SELECT items FROM {NestedTable} ORDER BY id")) + { + if (block["items"] is INestedColumn items) + { + Console.WriteLine($" {block["items"].TypeName}"); + Console.WriteLine($" FieldCount {items.FieldCount}"); + Console.WriteLine($" FieldNames [{string.Join(", ", items.FieldNames)}]"); + Console.WriteLine($" Offsets [{string.Join(", ", items.Offsets.ToArray())}] shared by every field"); + + var skus = (IColumn)items.GetField("sku"); + var quantities = (IColumn)items.GetField("qty"); + Console.WriteLine($" GetField(\"sku\").Values [{string.Join(", ", skus.Values.ToArray())}]"); + Console.WriteLine($" GetField(\"qty\").Values [{string.Join(", ", quantities.Values.ToArray())}]"); + Console.WriteLine(" GetField(int) takes the same field by position."); + Console.WriteLine(); + + ReadOnlySpan offsets = items.Offsets; + for (int row = 0; row < items.RowCount; row++) + { + var entries = new List(); + for (int entry = offsets[row]; entry < offsets[row + 1]; entry++) + { + entries.Add($"{skus[entry]} x{quantities[entry]}"); + } + + Console.WriteLine($" row {row}: {(entries.Count == 0 ? "(empty)" : string.Join(", ", entries))}"); + } + + Console.WriteLine(); + Console.WriteLine($" The materialized row is an object[][] — one object[] per entry, boxed, so the"); + Console.WriteLine($" field columns are the way to read it: items.GetValue(0) = {Render(block["items"].GetValue(0))}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" A Nested column only exists as one column when the table was created with"); + Console.WriteLine(" flatten_nested = 0. On the default setting the server turns Nested(a T, b U) into an"); + Console.WriteLine(" Array(T) named a and an Array(U) named b, and this view never appears."); + } + + private static async Task GeoAliases(ClickHouseTcpClient client) + { + Console.WriteLine("\n8. The geo aliases resolve to structures you have already seen\n"); + Console.WriteLine(" Each is a name for a shape built out of Tuple and Array, and the wire header carries the"); + Console.WriteLine(" alias rather than the structure — so TypeName is the alias, and the view is the"); + Console.WriteLine(" structure's:\n"); + Console.WriteLine(" TypeName One row is Pattern-matches to"); + Console.WriteLine(" --------------- ---------------------------------- ----------------------------------"); + + const string geoSql = @" + SELECT CAST((1.0, 2.0), 'Point') AS p, + CAST([(0.0, 0.0), (1.0, 0.0), (1.0, 1.0)], 'Ring') AS r, + CAST([(0.0, 0.0), (1.0, 1.0)], 'LineString') AS ls, + CAST([[(0.0, 0.0), (1.0, 0.0), (1.0, 1.0)]], 'Polygon') AS pg, + CAST([[(0.0, 0.0), (1.0, 1.0)]], 'MultiLineString') AS mls, + CAST([[[(0.0, 0.0), (1.0, 0.0), (1.0, 1.0)]]], 'MultiPolygon') AS mp"; + + await foreach (Block block in client.StreamAsync(geoSql)) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine($" {column.TypeName,-15} {Describe(column.ElementType),-34} {View(column)}"); + } + + Console.WriteLine(); + Console.WriteLine(" Point is a Tuple(Float64, Float64) and the rest are arrays over it, so a Ring's"); + Console.WriteLine(" coordinates are reachable as two flat columns without any tuple being built:"); + + if (block["r"] is IArrayColumn<(double, double)> ring && ring.Inner is ITupleColumn coordinates) + { + var longitudes = (IColumn)coordinates.Children[0]; + var latitudes = (IColumn)coordinates.Children[1]; + Console.WriteLine($" Offsets [{string.Join(", ", ring.Offsets.ToArray())}]"); + Console.WriteLine($" Children[0] [{string.Join(", ", longitudes.Values.ToArray())}]"); + Console.WriteLine($" Children[1] [{string.Join(", ", latitudes.Values.ToArray())}]"); + Console.WriteLine($" row 0 {Render(block["r"].GetValue(0))}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" The point to carry over from the HTTP driver: a coordinate pair here is a ValueTuple,"); + Console.WriteLine(" where that one builds a System.Tuple. So (double, double) and not Tuple,"); + Console.WriteLine(" and .Item1 / .Item2 on a struct rather than on a class."); + Console.WriteLine(); + Console.WriteLine(" Ring and LineString are distinct types to the server and the same structure to this"); + Console.WriteLine(" client, as are Polygon and MultiLineString. Only the name tells them apart."); + } + + private static async Task Geometry(ClickHouseTcpClient client) + { + Console.WriteLine("\n9. Geometry is the one alias that is not a nested array\n"); + Console.WriteLine(" It names a Variant over the six above, so one column holds rows of different shapes. The"); + Console.WriteLine(" header carries only 'Geometry', so the client expands the alternatives itself, in the"); + Console.WriteLine(" server's own name-sorted discriminator order:\n"); + + const string sql = @" + SELECT g FROM (SELECT arrayJoin([ + CAST(CAST((1.0, 2.0), 'Point'), 'Geometry'), + CAST(CAST([(1.0, 2.0), (3.0, 4.0)], 'LineString'), 'Geometry'), + CAST(CAST([[[(0.0, 0.0), (1.0, 0.0), (1.0, 1.0)]]], 'MultiPolygon'), 'Geometry')]) AS g)"; + + await foreach (Block block in client.StreamAsync(sql)) + { + if (block["g"] is IVariantColumn geometry) + { + Console.WriteLine($" {block["g"].TypeName}, read as {Describe(block["g"].ElementType)}, view IVariantColumn"); + Console.WriteLine($" TypeCount {geometry.TypeCount}"); + Console.WriteLine($" Discriminators [{string.Join(", ", geometry.Discriminators.ToArray())}]"); + Console.WriteLine($" LocalIndices [{string.Join(", ", geometry.LocalIndices.ToArray())}]"); + Console.WriteLine(); + Console.WriteLine(" Alternative order: 0 LineString, 1 MultiLineString, 2 MultiPolygon, 3 Point,"); + Console.WriteLine(" 4 Polygon, 5 Ring. GetTypeColumn names the shape of each row:"); + + for (int row = 0; row < geometry.RowCount; row++) + { + IColumn child = geometry.GetTypeColumn(geometry.Discriminators[row]); + Console.WriteLine($" row {row}: discriminator {geometry.Discriminators[row]} -> {child.TypeName,-14} value {Render(block["g"].GetValue(row))}"); + } + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" Tcp_014 covers IVariantColumn properly, including the NULL discriminator and how to"); + Console.WriteLine(" dispatch on it without boxing."); + } + + private static double Sum(ReadOnlySpan values) + { + double total = 0; + foreach (int value in values) + { + total += value; + } + + return total; + } + + // Which of the block tier's typed views a column offers, found by pattern-matching rather than by reading + // TypeName. The generic ones each need their element type spelled out, which is what makes this list long. + private static string View(IColumn column) => column switch + { + IVariantColumn => "IVariantColumn", + INestedColumn => "INestedColumn", + ITupleColumn => "ITupleColumn", + IMapColumn => "IMapColumn", + IMapColumn => "IMapColumn", + INullableColumn => "INullableColumn", + ILowCardinalityColumn => "ILowCardinalityColumn", + IArrayColumn => "IArrayColumn", + IArrayColumn => "IArrayColumn", + IArrayColumn<(int, string)> => "IArrayColumn<(int, string)>", + IArrayColumn<(double, double)> => "IArrayColumn<(double, double)>", + IArrayColumn<(double, double)[]> => "IArrayColumn<(double, double)[]>", + IArrayColumn<(double, double)[][]> => "IArrayColumn<(double, double)[][]>", + _ => "- (no composite view)", + }; + + private static string Describe(Type type) + { + if (type.IsArray) + { + return Describe(type.GetElementType()!) + "[]"; + } + + if (type.IsGenericType) + { + Type definition = type.GetGenericTypeDefinition(); + string[] arguments = type.GetGenericArguments().Select(Describe).ToArray(); + if (definition == typeof(Nullable<>)) + { + return arguments[0] + "?"; + } + + if (definition.FullName?.StartsWith("System.ValueTuple`", StringComparison.Ordinal) == true) + { + return "(" + string.Join(", ", arguments) + ")"; + } + + string name = definition.Name[..definition.Name.IndexOf('`')]; + return $"{name}<{string.Join(", ", arguments)}>"; + } + + return type switch + { + _ when type == typeof(byte) => "byte", + _ when type == typeof(int) => "int", + _ when type == typeof(uint) => "uint", + _ when type == typeof(long) => "long", + _ when type == typeof(ulong) => "ulong", + _ when type == typeof(double) => "double", + _ when type == typeof(string) => "string", + _ when type == typeof(object) => "object", + _ => type.Name, + }; + } + + private static string Render(object? value) => value switch + { + null => "NULL", + string text => $"'{text}'", + System.Collections.IEnumerable items => "[" + string.Join(", ", items.Cast().Select(Render)) + "]", + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? "NULL", + }; +} diff --git a/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs b/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs new file mode 100644 index 000000000..6ae9ae57e --- /dev/null +++ b/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs @@ -0,0 +1,431 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// The three types whose value is not decided by the type string: Variant(T1, ..., Tn), Dynamic and +/// JSON. +/// +/// +/// Variant and Dynamic are discriminated unions. Both read as IColumn<object>, so every +/// row read that way is boxed, and both expose a columnar view instead — a per-row discriminator plus one typed +/// child column per alternative. They differ in where the alternative list comes from: a Variant declares +/// it in the type string, a Dynamic discovers it per block and reports it as +/// . They also differ in how NULL is marked, which is the one detail that +/// will bite you. +/// +/// +/// +/// JSON is a different problem. This client reads and writes it only in the String serialization +/// (version 1), so a value is its compact JSON text. That works in both directions, but the server parses +/// what you write into real paths and re-renders on the way out, so the text you get back is not the text you +/// sent. Section 5 shows exactly what changes. +/// +/// +public static class TcpVariantDynamicJson +{ + private const string VariantTable = "example_tcp_variant"; + private const string DynamicTable = "example_tcp_dynamic"; + private const string JsonTable = "example_tcp_json"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + try + { + await Seed(client); + await Variants(client); + await Dynamics(client); + await TheTwoCompared(client); + await JsonIsText(client); + await JsonNormalization(client); + } + finally + { + foreach (string table in new[] { VariantTable, DynamicTable, JsonTable }) + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + + Console.WriteLine("\nDropped every table this example created."); + } + } + + private static async Task Seed(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {VariantTable} (id UInt64, v Variant(String, UInt64, Array(Int32))) + ENGINE = MergeTree() ORDER BY id"); + + await client.ExecuteAsync($@" + CREATE TABLE {DynamicTable} (id UInt64, d Dynamic) + ENGINE = MergeTree() ORDER BY id"); + + await client.ExecuteAsync($@" + CREATE TABLE {JsonTable} (id UInt64, doc JSON) + ENGINE = MergeTree() ORDER BY id"); + + // Both union types are written from an IColumn: one row per value, of whichever CLR type the + // chosen alternative takes, and null for a NULL row. + await client.InsertAsync( + $"INSERT INTO {VariantTable} (id, v) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3, 4, 5 }), + ClickHouseTcpColumn.Create("v", new object?[] { 42UL, "hi", null, new[] { 1, 2 }, 7UL }), + }); + + await client.InsertAsync( + $"INSERT INTO {DynamicTable} (id, d) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3, 4 }), + ClickHouseTcpColumn.Create("d", new object?[] { 42UL, "hi", null, 1.5 }), + }); + + Console.WriteLine($"Seeded '{VariantTable}' (5 rows), '{DynamicTable}' (4 rows) and '{JsonTable}'."); + } + + private static async Task Variants(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. Variant: the alternatives are declared, and the server sorts them\n"); + + await foreach (Block block in client.StreamAsync($"SELECT v FROM {VariantTable} ORDER BY id")) + { + IColumn column = block["v"]; + Console.WriteLine($" Declared as Variant(String, UInt64, Array(Int32))"); + Console.WriteLine($" Header says {column.TypeName}"); + Console.WriteLine(" The server canonicalizes the alternatives into name-sorted order, and that order is"); + Console.WriteLine(" the discriminator order. So read it from TypeName, never from what you declared.\n"); + + if (column is IVariantColumn variant) + { + Console.WriteLine($" TypeCount {variant.TypeCount}"); + Console.WriteLine($" Discriminators [{string.Join(", ", variant.Discriminators.ToArray())}] one byte per row"); + Console.WriteLine($" LocalIndices [{string.Join(", ", variant.LocalIndices.ToArray())}] -1 for a NULL row"); + Console.WriteLine($" IVariantColumn.NullDiscriminator = {IVariantColumn.NullDiscriminator} a fixed sentinel, not TypeCount"); + Console.WriteLine(); + Console.WriteLine(" One child column per alternative, holding only the rows that chose it:"); + + for (int discriminator = 0; discriminator < variant.TypeCount; discriminator++) + { + IColumn child = variant.GetTypeColumn(discriminator); + Console.WriteLine($" {discriminator} {child.TypeName,-16} {child.RowCount} row(s)"); + } + + Console.WriteLine(); + Console.WriteLine(" Row i's value is GetTypeColumn(Discriminators[i])[LocalIndices[i]], so dispatch"); + Console.WriteLine(" once per alternative and read the child typed rather than boxed:"); + Console.WriteLine(); + + // The typed children are bound once, outside the row loop. Nothing here boxes. + var strings = (IColumn)variant.GetTypeColumn(1); + var numbers = (IColumn)variant.GetTypeColumn(2); + var lists = (IColumn)variant.GetTypeColumn(0); + ReadOnlySpan discriminators = variant.Discriminators; + ReadOnlySpan local = variant.LocalIndices; + + for (int row = 0; row < column.RowCount; row++) + { + byte discriminator = discriminators[row]; + string reading = discriminator == IVariantColumn.NullDiscriminator + ? "NULL" + : discriminator switch + { + 0 => $"Array(Int32) [{string.Join(", ", lists[local[row]])}]", + 1 => $"String '{strings[local[row]]}'", + 2 => $"UInt64 {numbers[local[row]]}", + _ => "?", + }; + + Console.WriteLine($" row {row}: discriminator {discriminator,3}, local {local[row],2} -> {reading}"); + } + + Console.WriteLine(); + Console.WriteLine(" Passing NullDiscriminator to GetTypeColumn throws — it selects no column — so"); + Console.WriteLine(" guard for it before the call, as the loop above does."); + + try + { + _ = variant.GetTypeColumn(IVariantColumn.NullDiscriminator); + } + catch (IndexOutOfRangeException) + { + Console.WriteLine($" GetTypeColumn({IVariantColumn.NullDiscriminator}) -> IndexOutOfRangeException"); + } + } + + Console.WriteLine(); + Console.WriteLine($" The materialized surface is IColumn: ElementType is {column.ElementType.Name}, so"); + Console.WriteLine(" GetValue boxes every row — including the ones whose alternative is a value type:"); + for (int row = 0; row < column.RowCount; row++) + { + object? value = column.GetValue(row); + Console.WriteLine($" GetValue({row}) -> {(value is null ? "null" : $"{Describe(value.GetType())} {Render(value)}")}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" The server can tell you the same thing in SQL, which is worth knowing for a query you"); + Console.WriteLine(" are debugging:"); + + await foreach (object[] row in client.QueryAsync( + $"SELECT id, variantType(v) FROM {VariantTable} ORDER BY id")) + { + Console.WriteLine($" row {row[0]}: variantType(v) ordinal {row[1]}"); + } + + Console.WriteLine(" variantType returns an Enum8 whose type string spells the whole mapping — and which"); + Console.WriteLine(" this client reads as the bare ordinal, as Tcp_011 section 6 explains."); + } + + private static async Task Dynamics(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. Dynamic: the alternatives are discovered, and they are named\n"); + + await foreach (Block block in client.StreamAsync($"SELECT d FROM {DynamicTable} ORDER BY id")) + { + IColumn column = block["d"]; + Console.WriteLine($" {column.TypeName} — the type string says nothing about what is in it.\n"); + + if (column is IDynamicColumn dynamicColumn) + { + Console.WriteLine($" TypeCount {dynamicColumn.TypeCount}"); + Console.WriteLine($" TypeNames [{string.Join(", ", dynamicColumn.TypeNames)}] read off the wire, in discriminator order"); + Console.WriteLine($" Discriminators [{string.Join(", ", dynamicColumn.Discriminators.ToArray())}] ints here, not bytes"); + Console.WriteLine($" LocalIndices [{string.Join(", ", dynamicColumn.LocalIndices.ToArray())}]"); + Console.WriteLine(); + Console.WriteLine($" NULL is marked with TypeCount ({dynamicColumn.TypeCount}), one past the last type — there is no"); + Console.WriteLine(" fixed sentinel, because the type list is per block rather than declared."); + Console.WriteLine(); + + ReadOnlySpan discriminators = dynamicColumn.Discriminators; + ReadOnlySpan local = dynamicColumn.LocalIndices; + + for (int row = 0; row < column.RowCount; row++) + { + int discriminator = discriminators[row]; + if (discriminator == dynamicColumn.TypeCount) + { + Console.WriteLine($" row {row}: discriminator {discriminator} -> NULL"); + continue; + } + + IColumn child = dynamicColumn.GetTypeColumn(discriminator); + Console.WriteLine( + $" row {row}: discriminator {discriminator} -> {child.TypeName,-8} ({Describe(child.ElementType)}) value {Render(child.GetValue(local[row]))}"); + } + + Console.WriteLine(); + Console.WriteLine(" TypeNames is what makes typed reading possible: the name tells you what to cast a"); + Console.WriteLine(" child to, so a caller can bind IColumn per alternative without inspecting a"); + Console.WriteLine(" single value."); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" Because the type set is per block, the same value can land on a different discriminator"); + Console.WriteLine(" in the next block of the same result. Read TypeNames inside the loop, not once."); + Console.WriteLine(); + Console.WriteLine(" The client infers the ClickHouse type of each written value from its CLR type, so what"); + Console.WriteLine(" went in as a ulong came back as UInt64 and a double as Float64. A value whose CLR type"); + Console.WriteLine(" has no ClickHouse counterpart cannot be written into a Dynamic at all."); + } + + private static async Task TheTwoCompared(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. Variant against Dynamic, in one table\n"); + Console.WriteLine(" Variant(...) Dynamic"); + Console.WriteLine(" --------------- ---------------------------- ------------------------------"); + Console.WriteLine(" alternatives declared in the type string discovered per block"); + Console.WriteLine(" the list parse it out of TypeName IDynamicColumn.TypeNames"); + Console.WriteLine(" Discriminators ReadOnlySpan ReadOnlySpan"); + Console.WriteLine($" NULL is marked {IVariantColumn.NullDiscriminator} (NullDiscriminator) TypeCount"); + Console.WriteLine(" NULL LocalIndex -1 -1"); + Console.WriteLine(" a row not in the rejected by the server widens the type set"); + Console.WriteLine(" alternative list"); + Console.WriteLine(); + Console.WriteLine(" The asymmetry worth remembering: a Variant tells you nothing about its alternatives"); + Console.WriteLine(" through the interface, and a Geometry column (Tcp_013 section 9) does not even carry"); + Console.WriteLine(" them in its type string. So for a Variant, hard-code the order you declared and check it"); + Console.WriteLine(" against TypeName; for a Dynamic, read TypeNames."); + + Console.WriteLine(); + Console.WriteLine(" dynamicType() is the Dynamic counterpart of variantType(), and unlike it returns the"); + Console.WriteLine(" name rather than an ordinal:"); + + await foreach (object[] row in client.QueryAsync( + $"SELECT id, dynamicType(d) FROM {DynamicTable} ORDER BY id")) + { + Console.WriteLine($" row {row[0]}: dynamicType(d) = '{row[1]}'"); + } + + Console.WriteLine(" 'None' is the NULL row. It is not one of the TypeNames."); + } + + private static async Task JsonIsText(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. JSON: one serialization, and it is text\n"); + Console.WriteLine(" ClickHouse can send a JSON column in several encodings. The per-path binary ones split"); + Console.WriteLine(" the column into one sub-column per JSON path; this client decodes none of them. It reads"); + Console.WriteLine(" and writes only the String serialization, version 1, where a value is its JSON text:\n"); + + await foreach (Block block in client.StreamAsync( + @"SELECT CAST('{""a"": 1}', 'JSON') AS plain, + CAST('{""a"": 1, ""z"": ""s""}', 'JSON(a UInt32)') AS typed_path, + CAST(NULL, 'Nullable(JSON)') AS maybe, + CAST(['{""a"":1}', '{""b"":2}'], 'Array(JSON)') AS several")) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine($" {column.Name,-11} {column.TypeName,-16} reads as {Describe(column.ElementType),-9} {Render(column.GetValue(0))}"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" Every spelling of the type is the same String column, so JSON(a UInt32) and"); + Console.WriteLine(" JSON(max_dynamic_paths=8) need no special handling — the arguments ride in TypeName only."); + Console.WriteLine(" Under a composite the version marker comes first, then the composite's own framing."); + Console.WriteLine(); + Console.WriteLine(" Reading needs the query setting output_format_native_write_json_as_string = 1. The client"); + Console.WriteLine(" sets it on every operation, so this is only your problem if you override it:"); + + try + { + var withoutTheSetting = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["output_format_native_write_json_as_string"] = "0" }, + }; + + await foreach (Block _ in client.StreamAsync(@"SELECT CAST('{""a"":1}', 'JSON') AS j", withoutTheSetting)) + { + break; + } + + Console.WriteLine(" accepted, which this example did not expect"); + } + catch (ClickHouseTcpProtocolException ex) + { + Console.WriteLine($" {Wrap(ex.Message)}"); + } + + Console.WriteLine(); + Console.WriteLine(" Writing needs no setting at all: the version marker the client writes tells the server"); + Console.WriteLine(" which encoding it is reading, so version 1 makes it parse the text server-side — into a"); + Console.WriteLine(" JSON(a UInt32) column's typed paths as readily as into an untyped one."); + } + + private static async Task JsonNormalization(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. Text in is not text out\n"); + Console.WriteLine(" A JSON value is parsed into paths and re-rendered, never stored verbatim. So a round trip"); + Console.WriteLine(" through a JSON column is lossy in a way a String column would not be. Written and read"); + Console.WriteLine(" back, unchanged in between:\n"); + + var documents = new (string Text, string What)[] + { + ("{\"b\": 1, \"a\": 2}", "keys are sorted, ordinally"), + ("{ \"x\" : 1 , \"y\": 2 }", "whitespace is dropped"), + ("{\"a\": 1.500, \"b\": 1e3, \"c\": -0.0}", "numbers are re-rendered canonically"), + ("{\"n\": null, \"empty\": {}}", "a JSON null and an empty object contribute no path"), + ("{\"when\": \"2026-06-01T12:00:00Z\"}", "a string the server reads as a DateTime is re-formatted"), + ("{\"B\": 1, \"a\": 2}", "ordinal sorting puts every capital before every lower case"), + }; + + await client.InsertAsync( + $"INSERT INTO {JsonTable} (id, doc) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", Enumerable.Range(0, documents.Length).Select(i => (ulong)i).ToArray()), + ClickHouseTcpColumn.Create("doc", documents.Select(document => document.Text).ToArray()), + }); + + int index = 0; + await foreach (object[] row in client.QueryAsync($"SELECT id, doc FROM {JsonTable} ORDER BY id")) + { + (string text, string what) = documents[index++]; + Console.WriteLine($" {what}"); + Console.WriteLine($" in {text}"); + Console.WriteLine($" out {row[1]}"); + } + + Console.WriteLine(); + Console.WriteLine(" The DateTime row is the one that catches people. \"2026-06-01T12:00:00Z\" was inferred to"); + Console.WriteLine(" be a DateTime path, and a DateTime renders in ClickHouse's own format, so the T and the Z"); + Console.WriteLine(" are gone. The value is not corrupted — but it is no longer the string you wrote, and a"); + Console.WriteLine(" consumer parsing it as ISO 8601 will fail."); + Console.WriteLine(); + Console.WriteLine(" You can see what the server decided each path was:"); + + await foreach (object[] row in client.QueryAsync( + $"SELECT id, toString(JSONAllPathsWithTypes(doc)) FROM {JsonTable} ORDER BY id")) + { + Console.WriteLine($" row {row[0]}: {row[1]}"); + } + + Console.WriteLine(); + Console.WriteLine(" What to do about it:"); + Console.WriteLine(" Do not compare the text you wrote with the text you read. Compare the paths, or the"); + Console.WriteLine(" values at a path, which is what the server can be asked for."); + Console.WriteLine(" Store a timestamp as a real DateTime64 column, not inside a JSON string."); + Console.WriteLine(" Use a String column when you need the bytes back exactly — a JSON column is a set of"); + Console.WriteLine(" typed paths that happens to be spelled as text on this transport."); + } + + private static string Describe(Type type) + { + if (type.IsArray) + { + return Describe(type.GetElementType()!) + "[]"; + } + + return type switch + { + _ when type == typeof(int) => "int", + _ when type == typeof(uint) => "uint", + _ when type == typeof(long) => "long", + _ when type == typeof(ulong) => "ulong", + _ when type == typeof(double) => "double", + _ when type == typeof(string) => "string", + _ when type == typeof(object) => "object", + _ => type.Name, + }; + } + + private static string Render(object? value) => value switch + { + null => "NULL", + string text => $"\"{text}\"", + System.Collections.IEnumerable items => "[" + string.Join(", ", items.Cast().Select(Render)) + "]", + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? "NULL", + }; + + // Reflows a long driver message so the console output stays readable. + private static string Wrap(string message) + { + var lines = new List(); + var line = new System.Text.StringBuilder(); + foreach (string word in message.Split(' ')) + { + if (line.Length + word.Length + 1 > 88) + { + lines.Add(line.ToString()); + line.Clear(); + } + + line.Append(line.Length == 0 ? word : " " + word); + } + + lines.Add(line.ToString()); + return string.Join("\n ", lines); + } +} diff --git a/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs b/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs new file mode 100644 index 000000000..b1c04c4e5 --- /dev/null +++ b/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs @@ -0,0 +1,488 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// QBit(T, N) and : the one type whose whole point is its storage layout. +/// +/// +/// A QBit row is an N-element vector, but the elements of a row are not stored together. The column +/// holds one bit plane per bit position of the element type, and a plane carries that one bit of every +/// element of every row. So the most significant bits of a whole column sit contiguously, which is what lets a +/// distance be computed at reduced precision by reading only the top few planes — the server's +/// L2DistanceTransposed(vector, query, precision) does exactly that, and +/// is the same access from the client. +/// +/// +/// +/// The default IColumn<T> view undoes the transposition and hands back a float[] (or +/// double[]) per row, which is convenient and throws away the only reason to use the type. This example is +/// about the planes. examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs covers the server-side +/// search, which the HTTP transport can do just as well. +/// +/// +public static class TcpQBitVectorSearch +{ + private const string TableName = "example_tcp_qbit"; + private const string WideTable = "example_tcp_qbit_wide"; + + // Int8 elements and the strided QBit(T, N, stride) form both need a newer server. + private static readonly Version StridedAndInt8From = new(26, 7); + + private static readonly (string Word, float[] Vector)[] Corpus = + { + ("apple", new[] { 0.9f, 0.1f, 0.8f, 0.2f, 0.7f }), + ("banana", new[] { 0.85f, 0.15f, 0.75f, 0.25f, 0.65f }), + ("orange", new[] { 0.88f, 0.12f, 0.78f, 0.22f, 0.68f }), + ("dog", new[] { 0.1f, 0.9f, 0.2f, 0.8f, 0.3f }), + ("horse", new[] { 0.15f, 0.85f, 0.25f, 0.75f, 0.35f }), + ("cat", new[] { 0.12f, 0.88f, 0.22f, 0.78f, 0.32f }), + }; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + + try + { + await Seed(client); + await TheGeometry(client); + await ReadingAPlane(client); + await ByteOrderWithinABitmap(client); + await ReducedPrecision(client); + await ElementTypes(client, server); + await Strided(client, server); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {WideTable}"); + Console.WriteLine("\nDropped every table this example created."); + } + } + + private static async Task Seed(ClickHouseTcpClient client) + { + await client.ExecuteAsync($@" + CREATE TABLE {TableName} (word String, vec QBit(Float32, 5)) + ENGINE = MergeTree() ORDER BY word"); + + // A QBit column is written from one float[] per row. The client transposes it into planes. + await client.InsertAsync( + $"INSERT INTO {TableName} (word, vec) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("word", Corpus.Select(entry => entry.Word).ToArray()), + ClickHouseTcpColumn.Create("vec", Corpus.Select(entry => entry.Vector).ToArray()), + }); + + Console.WriteLine($"Seeded '{TableName}' with {Corpus.Length} words as QBit(Float32, 5) embeddings."); + } + + private static async Task TheGeometry(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. What the column reports about its layout\n"); + + await foreach (Block block in client.StreamAsync($"SELECT vec FROM {TableName} ORDER BY word")) + { + IColumn column = block["vec"]; + Console.WriteLine($" {column.TypeName}, reads as {Describe(column.ElementType)}, {column.RowCount} rows\n"); + + if (column is IQBitColumn qbit) + { + Console.WriteLine($" Dimension {qbit.Dimension} the N of QBit(T, N) — elements per vector"); + Console.WriteLine($" BitWidth {qbit.BitWidth} the stored element's bit width, so the number of planes"); + Console.WriteLine($" Stride {qbit.Stride} elements one group of planes covers"); + Console.WriteLine($" GroupCount {qbit.GroupCount} Dimension / Stride"); + Console.WriteLine($" BytesPerRow {qbit.BytesPerRow} ceil(Stride / 8) — one row's bitmap within one plane"); + Console.WriteLine(); + Console.WriteLine(" The body is plane-major and every row is the same width, so its size is exact:"); + + int body = qbit.BitWidth * qbit.RowCount * qbit.BytesPerRow; + int flat = qbit.Dimension * (qbit.BitWidth / 8) * qbit.RowCount; + Console.WriteLine($" BitWidth * RowCount * BytesPerRow = {qbit.BitWidth} * {qbit.RowCount} * {qbit.BytesPerRow} = {body} bytes"); + Console.WriteLine($" the same values as {qbit.Dimension} Float32 per row = {flat} bytes"); + Console.WriteLine(); + Console.WriteLine($" The extra is padding. A plane's row is a whole number of bytes, so {qbit.BytesPerRow * 8} bit slots"); + Console.WriteLine($" carry {qbit.Stride} elements and {(qbit.BytesPerRow * 8) - qbit.Stride} slots go unused in every one of the {qbit.BitWidth} planes. A Stride"); + Console.WriteLine($" that is a multiple of 8 wastes nothing; {qbit.Stride} is not, so this column is the wider one."); + Console.WriteLine(); + Console.WriteLine(" BitWidth is the width of the STORED element, not of the CLR one: a"); + Console.WriteLine(" QBit(BFloat16, N) has 16 planes and still reads as float[]. Section 5."); + } + + break; + } + } + + private static async Task ReadingAPlane(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. A plane is one bit of every element of every row\n"); + + await foreach (Block block in client.StreamAsync($"SELECT word, vec FROM {TableName} ORDER BY word")) + { + if (block["vec"] is IQBitColumn qbit) + { + var words = (IColumn)block["word"]; + Console.WriteLine($" GetPlane(bit) returns RowCount * BytesPerRow = {qbit.RowCount} * {qbit.BytesPerRow} = {qbit.RowCount * qbit.BytesPerRow} bytes."); + Console.WriteLine($" Row r's bitmap is the slice [r * BytesPerRow, (r + 1) * BytesPerRow)."); + Console.WriteLine(); + Console.WriteLine($" bit is the SIGNIFICANCE within the stored element, so {qbit.BitWidth - 1} is the sign bit and 0 the"); + Console.WriteLine(" least significant mantissa bit. (The wire stores the planes the other way round,"); + Console.WriteLine(" most significant first; the accessor hides that.)"); + Console.WriteLine(); + Console.WriteLine($" The top {qbit.BitWidth - 24} planes of this corpus, most significant first:"); + Console.WriteLine(); + Console.WriteLine(" plane bitmaps, one byte per row"); + Console.WriteLine(" ------ -------------------------"); + + for (int bit = qbit.BitWidth - 1; bit >= 24; bit--) + { + ReadOnlySpan plane = qbit.GetPlane(bit); + Console.WriteLine($" bit {bit,2} {string.Join(" ", plane.ToArray().Select(value => value.ToString("X2", CultureInfo.InvariantCulture)))}"); + } + + // The highest plane that is not identical across every row. Found from the data rather than + // asserted, because it depends entirely on what the vectors are. + int firstDifference = qbit.BitWidth - 1; + while (firstDifference >= 0 && Uniform(qbit.GetPlane(firstDifference))) + { + firstDifference--; + } + + Console.WriteLine(); + Console.WriteLine($" The top {qbit.BitWidth - 1 - firstDifference} planes are identical in every row: bit {qbit.BitWidth - 1} is the sign and every vector"); + Console.WriteLine($" here is positive, and the exponent's high bits agree because every element is in"); + Console.WriteLine($" [0.1, 0.9]. The first plane that separates the corpus is bit {firstDifference}. That is the type's"); + Console.WriteLine(" bargain: precision costs planes, and how many you can drop depends on the data."); + + Console.WriteLine(); + Console.WriteLine($" Within a row's bitmap, element i is bit i % 8 of byte BytesPerRow - 1 - i / 8. With"); + Console.WriteLine($" BytesPerRow = {qbit.BytesPerRow} there is one byte per row, so element i is simply bit i:"); + Console.WriteLine(); + Console.WriteLine($" word bit {firstDifference} bitmap elements with bit {firstDifference} set"); + Console.WriteLine(" ------- ------------- ------------------------"); + + ReadOnlySpan interesting = qbit.GetPlane(firstDifference); + for (int row = 0; row < qbit.RowCount; row++) + { + byte bitmap = interesting[(row * qbit.BytesPerRow) + qbit.BytesPerRow - 1]; + var set = new List(); + for (int element = 0; element < qbit.Dimension; element++) + { + if ((bitmap & (1 << element)) != 0) + { + set.Add(element); + } + } + + Console.WriteLine($" {words[row],-7} {Convert.ToString(bitmap, 2).PadLeft(8, '0')} {(set.Count == 0 ? "none" : string.Join(", ", set))}"); + } + + Console.WriteLine(); + Console.WriteLine(" A bit index outside the planes, or a group outside the groups, is refused:"); + foreach (Action attempt in new Action[] { () => qbit.GetPlane(qbit.BitWidth), () => qbit.GetPlane(0, 1) }) + { + try + { + attempt(); + } + catch (ArgumentOutOfRangeException ex) + { + Console.WriteLine($" {ex.Message.Split(" (Parameter")[0]}"); + } + } + } + + break; + } + } + + private static async Task ByteOrderWithinABitmap(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. Past 8 elements the bytes run backwards\n"); + Console.WriteLine(" The bits within a byte run least significant first, but the bytes run in the reverse of"); + Console.WriteLine(" the element order — element 0 is in the LAST byte. Equivalently, a row's bitmap is the"); + Console.WriteLine(" big-endian encoding of a BytesPerRow-byte integer whose bit i is element i. That is"); + Console.WriteLine(" invisible at 5 elements and not at 12:\n"); + + await client.ExecuteAsync($@" + CREATE TABLE {WideTable} (v QBit(Float32, 12)) + ENGINE = MergeTree() ORDER BY tuple()"); + + // Elements 0 and 8 negative, the rest positive, so the sign plane says exactly where they sit. + float[] signs = Enumerable.Range(0, 12).Select(i => i is 0 or 8 ? -1.0f : 1.0f).ToArray(); + await client.InsertAsync( + $"INSERT INTO {WideTable} (v) VALUES", + new[] { ClickHouseTcpColumn.Create("v", new[] { signs }) }); + + Console.WriteLine($" One row of QBit(Float32, 12): [{string.Join(", ", signs.Select(value => value.ToString(CultureInfo.InvariantCulture)))}]"); + Console.WriteLine(" Only elements 0 and 8 are negative.\n"); + + await foreach (Block block in client.StreamAsync($"SELECT v FROM {WideTable}")) + { + if (block["v"] is IQBitColumn wide) + { + ReadOnlySpan sign = wide.GetPlane(wide.BitWidth - 1); + Console.WriteLine($" BytesPerRow {wide.BytesPerRow} (ceil(12 / 8))"); + Console.WriteLine($" sign plane {string.Join(" ", sign.ToArray().Select(value => Convert.ToString(value, 2).PadLeft(8, '0')))}"); + Console.WriteLine(" ^ byte 0 ^ byte 1"); + Console.WriteLine(" Byte 1 holds elements 0-7 and byte 0 elements 8-11, so the bit set in byte 1 is"); + Console.WriteLine(" element 0 and the bit set in byte 0 is element 8."); + Console.WriteLine(); + Console.WriteLine(" The formula covers both: element i is bit i % 8 of byte BytesPerRow - 1 - i / 8."); + Console.WriteLine($" element 0 -> bit 0 of byte {wide.BytesPerRow - 1 - (0 / 8)}"); + Console.WriteLine($" element 8 -> bit 0 of byte {wide.BytesPerRow - 1 - (8 / 8)}"); + Console.WriteLine(); + Console.WriteLine($" With Stride not a multiple of 8, the {(wide.BytesPerRow * 8) - wide.Dimension} unused bits are the high bits of byte 0."); + } + + break; + } + } + + private static async Task ReducedPrecision(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. Why the planes exist: a distance at reduced precision\n"); + Console.WriteLine(" Reading only the top K planes and treating the rest of each element's bits as zero gives"); + Console.WriteLine(" a truncated float — a quarter of the bytes at K = 8. Rebuilt from its planes, the vector"); + Console.WriteLine(" of 'apple':\n"); + Console.WriteLine(" Planes read Bytes per row Reconstructed vector"); + Console.WriteLine(" ----------- ------------- ----------------------------------------------"); + + await foreach (Block block in client.StreamAsync($"SELECT word, vec FROM {TableName} ORDER BY word")) + { + if (block["vec"] is IQBitColumn qbit) + { + var words = (IColumn)block["word"]; + int apple = 0; + for (int row = 0; row < words.RowCount; row++) + { + if (words[row] == "apple") + { + apple = row; + } + } + + foreach (int keep in new[] { 32, 16, 12, 8 }) + { + float[] rebuilt = Reconstruct(qbit, apple, keep); + Console.WriteLine( + $" top {keep,2} {keep * qbit.BytesPerRow,3} [{string.Join(", ", rebuilt.Select(value => value.ToString("0.####", CultureInfo.InvariantCulture)))}]"); + } + + // The materialized view, for comparison: it reads every plane, which is what "top 32" did. + float[] materialized = ((IColumn)block["vec"])[apple]; + Console.WriteLine(); + Console.WriteLine($" The top row is exact, and equals what the IColumn view hands back:"); + Console.WriteLine($" [{string.Join(", ", materialized.Select(value => value.ToString("0.####", CultureInfo.InvariantCulture)))}]"); + } + + break; + } + + Console.WriteLine(); + Console.WriteLine(" The server does the same arithmetic in L2DistanceTransposed's third argument, which is a"); + Console.WriteLine(" count of planes. Ranking 'apple' against the corpus at three precisions:\n"); + Console.WriteLine(" word precision 32 precision 12 precision 8"); + Console.WriteLine(" ------- ------------ ------------ -----------"); + + const string query = "[0.9, 0.1, 0.8, 0.2, 0.7]"; + await foreach (object[] row in client.QueryAsync($@" + SELECT word, + L2DistanceTransposed(vec, {query}, 32) AS d32, + L2DistanceTransposed(vec, {query}, 12) AS d12, + L2DistanceTransposed(vec, {query}, 8) AS d8 + FROM {TableName} + ORDER BY d32")) + { + Console.WriteLine($" {row[0],-7} {Number(row[1]),-12} {Number(row[2]),-12} {Number(row[3])}"); + } + + Console.WriteLine(); + Console.WriteLine(" Read the columns, not just the numbers. At precision 12 the ranking is unchanged and the"); + Console.WriteLine(" distances are already wrong in the second digit. At precision 8 apple and orange tie and"); + Console.WriteLine(" banana comes out ahead of both — the ranking has broken, while the two clusters are still"); + Console.WriteLine(" cleanly separated. That is the trade the type is for: shortlist cheaply at low precision,"); + Console.WriteLine(" then re-rank the shortlist exactly. How low you can go is a property of your vectors, so"); + Console.WriteLine(" measure it rather than picking a number."); + Console.WriteLine(); + Console.WriteLine(" Where the client's plane access earns its keep is the work the server has no function"); + Console.WriteLine(" for: a custom metric, a quantizer, or an index built over the top planes only. Reading"); + Console.WriteLine(" GetPlane(bit) for the few bits you want touches only those bytes, whereas the"); + Console.WriteLine(" IColumn view materializes every element of every row."); + } + + private static async Task ElementTypes(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) + { + Console.WriteLine("\n5. The element types, and the CLR type each reads as\n"); + Console.WriteLine(" Type BitWidth Reads as Note"); + Console.WriteLine(" ------------------ -------- --------- --------------------------------------"); + + foreach (string element in new[] { "BFloat16", "Float32", "Float64", "Int8" }) + { + if (element == "Int8" && server.Version < StridedAndInt8From) + { + Console.WriteLine($" QBit(Int8, 5) - - skipped: needs ClickHouse {StridedAndInt8From} or newer,"); + Console.WriteLine($" this server is {server.Version}"); + continue; + } + + string table = $"{TableName}_{element}"; + try + { + await client.ExecuteAsync($"CREATE TABLE {table} (v QBit({element}, 5)) ENGINE = MergeTree() ORDER BY tuple()"); + await client.ExecuteAsync($"INSERT INTO {table} VALUES ([1.0, 2.0, 0.5, -1.0, 0.25])"); + + await foreach (Block block in client.StreamAsync($"SELECT v FROM {table}")) + { + var qbit = (IQBitColumn)block["v"]; + string note = element switch + { + "BFloat16" => "16 planes, widened to float on the way out", + "Float64" => "the only one that reads as double[]", + "Int8" => "since ClickHouse 26.7", + _ => "the common case", + }; + Console.WriteLine($" {block["v"].TypeName,-18} {qbit.BitWidth,-8} {Describe(block["v"].ElementType),-9} {note}"); + Console.WriteLine($" row 0 = [{string.Join(", ", ((System.Collections.IEnumerable)block["v"].GetValue(0)!).Cast().Select(Number))}]"); + break; + } + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" QBit({element}, 5) refused by this server:"); + Console.WriteLine($" {FirstLine(ex.Message)}"); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + } + + Console.WriteLine(); + Console.WriteLine(" A BFloat16 element's plane positions are those of the 16-bit brain-float, not of the"); + Console.WriteLine(" float you read: bit 15 is its sign, and there are only 16 planes to ask for."); + } + + private static async Task Strided(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) + { + Console.WriteLine("\n6. The strided form, and why Stride and GroupCount exist\n"); + Console.WriteLine(" ClickHouse 26.7 added an optional third argument, QBit(T, N, stride), which splits a row"); + Console.WriteLine(" into N / stride independent groups, each carrying its own full set of planes. A plane is"); + Console.WriteLine(" then GroupCount disjoint runs, so GetPlane(bit) cannot name it and GetPlane(bit, group)"); + Console.WriteLine(" is the accessor.\n"); + + if (server.Version < StridedAndInt8From) + { + Console.WriteLine($" Skipped: needs ClickHouse {StridedAndInt8From} or newer, this server is {server.Version}."); + Console.WriteLine(" Confirming the server's own answer rather than assuming it:"); + + try + { + await client.ExecuteAsync($"CREATE TABLE {WideTable}_strided (v QBit(Float32, 8, 4)) ENGINE = MergeTree() ORDER BY tuple()"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {WideTable}_strided"); + Console.WriteLine(" accepted, which this example did not expect"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" {FirstLine(ex.Message)}"); + } + } + else + { + Console.WriteLine(" This server is new enough to declare one. Note that this client does not decode the"); + Console.WriteLine(" strided body yet, so reading such a column reports a NotSupportedException:"); + + try + { + await client.ExecuteAsync($"CREATE TABLE {WideTable}_strided (v QBit(Float32, 8, 4)) ENGINE = MergeTree() ORDER BY tuple()"); + await client.ExecuteAsync($"INSERT INTO {WideTable}_strided VALUES ([1, 2, 3, 4, 5, 6, 7, 8])"); + + await foreach (Block _ in client.StreamAsync($"SELECT v FROM {WideTable}_strided")) + { + break; + } + + Console.WriteLine(" read, which this example did not expect"); + } + catch (Exception ex) when (ex is NotSupportedException or ClickHouseTcpServerException) + { + Console.WriteLine($" {ex.GetType().Name}: {FirstLine(ex.Message)}"); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {WideTable}_strided"); + } + } + + Console.WriteLine(); + Console.WriteLine(" So on any server this client reads, GroupCount is 1 and Stride equals Dimension. Both"); + Console.WriteLine(" properties are there so plane-reading code is written against the general layout —"); + Console.WriteLine(" GetPlane(bit, group), BytesPerRow from Stride — and needs no change when it is not."); + } + + // True when every byte of a plane is the same, so the plane separates no row from any other. + private static bool Uniform(ReadOnlySpan plane) + { + for (int i = 1; i < plane.Length; i++) + { + if (plane[i] != plane[0]) + { + return false; + } + } + + return true; + } + + // Rebuilds one row's vector from its top `keep` planes, leaving the rest of each element's bits zero. This is + // the client-side equivalent of L2DistanceTransposed's precision argument. + private static float[] Reconstruct(IQBitColumn column, int row, int keep) + { + var rebuilt = new float[column.Dimension]; + for (int element = 0; element < column.Dimension; element++) + { + uint bits = 0; + for (int bit = column.BitWidth - 1; bit >= column.BitWidth - keep; bit--) + { + ReadOnlySpan plane = column.GetPlane(bit); + byte bitmap = plane[(row * column.BytesPerRow) + column.BytesPerRow - 1 - (element / 8)]; + if ((bitmap & (1 << (element % 8))) != 0) + { + bits |= 1u << bit; + } + } + + rebuilt[element] = BitConverter.UInt32BitsToSingle(bits); + } + + return rebuilt; + } + + private static string Describe(Type type) => type switch + { + _ when type == typeof(float[]) => "float[]", + _ when type == typeof(double[]) => "double[]", + _ when type == typeof(sbyte[]) => "sbyte[]", + _ => type.Name, + }; + + private static string Number(object? value) + => value is IFormattable formattable ? formattable.ToString("0.######", CultureInfo.InvariantCulture) : "-"; + + private static string FirstLine(string message) + { + int newline = message.IndexOf('\n'); + string line = newline < 0 ? message : message[..newline]; + if (line.StartsWith("DB::Exception: ", StringComparison.Ordinal)) + { + line = line["DB::Exception: ".Length..]; + } + + int scope = line.IndexOf(": In scope", StringComparison.Ordinal); + return scope < 0 ? line : line[..scope]; + } +} From 4421144d543ee14e9faa0328b7ffd8281625a02e Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 14:37:37 +0200 Subject: [PATCH 09/16] Add the native protocol's connection and session examples Four under examples/Tcp/Connection/: sessions, showing what one pinned connection buys and that disposal closes rather than pools it; the pool's seven knobs with the cap, the PoolTimeout expiry, the sweep and Lifo against Fifo all measured; TLS configuration and the mistakes the constructor refuses; and the timeouts, separating ReadTimeout's idle deadline from a total time limit. Pool state is not reportable, so Tcp_017 reads the pool's own log lines and polls system.processes from a second client for the concurrency figure: from the client side a query waiting for a slot and a query running look alike. Tcp_018 needs a TLS endpoint for its last section and CI has none, so that part is opt-in on CLICKHOUSE_TCP_TLS_CONNECTION_STRING and says why it skipped. The rest needs no server: the port derivation, and the refusals. Co-Authored-By: Claude Opus 5 (1M context) --- examples/Program.cs | 21 + examples/README.md | 7 + examples/Tcp/Connection/Tcp_016_Sessions.cs | 207 ++++++++++ examples/Tcp/Connection/Tcp_017_PoolTuning.cs | 373 ++++++++++++++++++ examples/Tcp/Connection/Tcp_018_Tls.cs | 259 ++++++++++++ examples/Tcp/Connection/Tcp_019_Timeouts.cs | 323 +++++++++++++++ 6 files changed, 1190 insertions(+) create mode 100644 examples/Tcp/Connection/Tcp_016_Sessions.cs create mode 100644 examples/Tcp/Connection/Tcp_017_PoolTuning.cs create mode 100644 examples/Tcp/Connection/Tcp_018_Tls.cs create mode 100644 examples/Tcp/Connection/Tcp_019_Timeouts.cs diff --git a/examples/Program.cs b/examples/Program.cs index ecff83f43..d309ffb07 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -421,6 +421,27 @@ private static async Task RunAllExamples(bool isInteractive) await TcpQBitVectorSearch.Run(); WaitForUser(isInteractive); + // Native Protocol: Connections and Sessions + Console.WriteLine("\n\n" + new string('=', 70)); + Console.WriteLine("NATIVE PROTOCOL: CONNECTIONS AND SESSIONS"); + Console.WriteLine(new string('=', 70) + "\n"); + + Console.WriteLine($"Running: {nameof(TcpSessions)}"); + await TcpSessions.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpPoolTuning)}"); + await TcpPoolTuning.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpTls)}"); + await TcpTls.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpTimeouts)}"); + await TcpTimeouts.Run(); + WaitForUser(isInteractive); + Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ALL EXAMPLES COMPLETED SUCCESSFULLY!"); Console.WriteLine(new string('=', 70)); diff --git a/examples/README.md b/examples/README.md index ddb1aa865..1877cfc11 100644 --- a/examples/README.md +++ b/examples/README.md @@ -126,6 +126,13 @@ These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/READ - [Tcp_014_VariantDynamicJson.cs](Tcp/Types/Tcp_014_VariantDynamicJson.cs) - `IVariantColumn` and `IDynamicColumn`: discriminators, local indices, the two different NULL markers, and typed dispatch without boxing — then `JSON`, which travels as text and comes back normalized, so what you write is not what you read - [Tcp_015_QBitVectorSearch.cs](Tcp/Types/Tcp_015_QBitVectorSearch.cs) - `QBit(T, N)` and `IQBitColumn`: the transposed bit-plane layout, `GetPlane` and the bitmap byte order, rebuilding a vector from its top planes to match `L2DistanceTransposed`'s precision argument, and the padding a dimension that is not a multiple of 8 costs +### Native Protocol: Connections and Sessions + +- [Tcp_016_Sessions.cs](Tcp/Connection/Tcp_016_Sessions.cs) - `OpenSessionAsync`: one pinned connection, so a temporary table and a `SET` survive from one operation to the next, `IsOpen`, one operation at a time, disposal closing rather than pooling the connection, and `SET ROLE` as the native answer to HTTP's per-query `Roles` +- [Tcp_017_PoolTuning.cs](Tcp/Connection/Tcp_017_PoolTuning.cs) - `MinPoolSize`, `MaxPoolSize`, `PoolTimeout`, `IdleTimeout`, `MaxConnectionLifetime`, `SweepInterval` and `PoolReusePolicy`, measured: concurrency actually capped, `PoolTimeout` expiring, the sweep retiring and topping up, `Lifo` against `Fifo`, and what a `ClickHouseTcpDataSource` shares +- [Tcp_018_Tls.cs](Tcp/Connection/Tcp_018_Tls.cs) - `UseTls`, `TlsServerName`, `TlsCaCertificatePath` (which replaces the host trust store rather than adding to it), `TlsAllowInvalidCertificates`, `ConfigureTls`, the default port moving to 9440, and the TLS mistakes the constructor refuses before anything connects +- [Tcp_019_Timeouts.cs](Tcp/Connection/Tcp_019_Timeouts.cs) - `DialTimeout`, `ReadTimeout` as an idle deadline rather than a time limit, `PoolTimeout`, `StatementMaxLength` in the log line, `MaxSendBufferBytes`, and where a `CancellationToken` takes over + ## How to run ### Prerequisites diff --git a/examples/Tcp/Connection/Tcp_016_Sessions.cs b/examples/Tcp/Connection/Tcp_016_Sessions.cs new file mode 100644 index 000000000..048667fe7 --- /dev/null +++ b/examples/Tcp/Connection/Tcp_016_Sessions.cs @@ -0,0 +1,207 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// OpenSessionAsync: one connection out of the pool, pinned until the session is disposed, so state a +/// single connection holds — a temporary table, what a SET changed, which roles are active — survives from +/// one operation to the next. +/// +/// +/// Three rules come with that. One operation runs at a time, because the protocol carries one query per +/// connection. Disposal closes the connection instead of returning it to the pool, so no unrelated caller +/// can inherit the session's state. And a session holds one of the pool's slots for its whole lifetime, so keep it +/// short — Tcp_017_PoolTuning shows what happens when sessions outnumber the pool. +/// +/// +/// +/// A session is also the native answer to HTTP's per-query Roles, which this transport does not have: +/// SET ROLE inside a session applies to every operation that follows it and to nothing else. +/// +/// +public static class TcpSessions +{ + private const string RoleTable = "example_tcp_sessions_orders"; + private const string RoleName = "example_tcp_sessions_reader"; + private const string RoleUser = "example_tcp_sessions_user"; + + // The password of the user this example creates to demonstrate SET ROLE. Not a connection string: the + // endpoint still comes from ExampleConfig. + private const string RoleUserPassword = "example_tcp_sessions_pw"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + await StateThatSurvives(client); + await OneOperationAtATime(client); + await DisposalClosesTheConnection(); + await SetRoleInASession(client); + WhatToRemember(); + } + + private static async Task StateThatSurvives(ClickHouseTcpClient client) + { + Console.WriteLine("1. One pinned connection, so connection-local state survives\n"); + + await using IClickHouseTcpSession session = await client.OpenSessionAsync(); + Console.WriteLine($" Opened a session. IsOpen = {session.IsOpen}"); + + // A temporary table belongs to the connection that created it, which is why it needs a session at all. + // It also needs no cleanup: the server drops it when the connection closes, and disposing the session + // closes the connection. + await session.ExecuteAsync("CREATE TEMPORARY TABLE example_tcp_sessions_scratch (id UInt64, note String) ENGINE = Memory"); + await session.InsertRowsAsync( + "INSERT INTO example_tcp_sessions_scratch (id, note) VALUES", + new[] + { + new object[] { 1UL, "first" }, + new object[] { 2UL, "second" }, + }); + + object rows = await session.ExecuteScalarAsync("SELECT count() FROM example_tcp_sessions_scratch"); + Console.WriteLine($" Created a TEMPORARY TABLE, inserted 2 rows, read back {rows} — three operations, one connection"); + + const string visible = "SELECT count() FROM system.tables WHERE is_temporary AND name = 'example_tcp_sessions_scratch'"; + Console.WriteLine($" system.tables sees it inside the session: {await session.ExecuteScalarAsync(visible)}"); + + // The same client, but this operation takes whatever connection the pool hands out, so it is a different + // session on the server and the table is not there. + Console.WriteLine($" ... and not from the client's pool: {await client.ExecuteScalarAsync(visible)}"); + + // A SET is connection state too, so it lasts exactly as long as the session. + await session.ExecuteAsync("SET max_threads = 7"); + Console.WriteLine("\n After SET max_threads = 7 in the session:"); + Console.WriteLine($" getSetting('max_threads') in the session = {await session.ExecuteScalarAsync("SELECT getSetting('max_threads')")}"); + Console.WriteLine($" getSetting('max_threads') on the client = {await client.ExecuteScalarAsync("SELECT getSetting('max_threads')")}"); + Console.WriteLine(" A client-level setting reaches every operation instead; see Tcp_002's set_ keys."); + } + + private static async Task OneOperationAtATime(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. One operation at a time\n"); + + await using IClickHouseTcpSession session = await client.OpenSessionAsync(); + + // Not awaited yet: an async method body starts on the calling thread, so by the time this returns the + // session has already claimed its connection for this query. + ValueTask running = session.ExecuteScalarAsync("SELECT sleep(0.2), 'slow'"); + + Console.WriteLine($" A query is in flight. IsOpen = {session.IsOpen} (busy is not closed)"); + + try + { + _ = await session.ExecuteScalarAsync("SELECT 'me too'"); + } + catch (InvalidOperationException ex) + { + Console.WriteLine($" A second operation on the same session throws {ex.GetType().Name}:"); + Console.WriteLine($" {ex.Message}"); + } + + _ = await running; + Console.WriteLine($"\n The first one finished, so the session is free again: {await session.ExecuteScalarAsync("SELECT 'next'")}"); + Console.WriteLine(" To run operations at once, run them on the client: each takes its own connection."); + } + + private static async Task DisposalClosesTheConnection() + { + Console.WriteLine("\n3. Disposal closes the connection, it does not pool it\n"); + + // MaxPoolSize = 1 makes the experiment decisive. If disposal returned the connection to the pool, the + // second session would be handed the same one and would find the first session's temporary table. + await using var client = new ClickHouseTcpClient(ExampleConfig.TcpBuilder().ToOptions() with { MaxPoolSize = 1 }); + + const string visible = "SELECT count() FROM system.tables WHERE is_temporary AND name = 'example_tcp_sessions_handover'"; + + IClickHouseTcpSession first = await client.OpenSessionAsync(); + await using (first) + { + await first.ExecuteAsync("CREATE TEMPORARY TABLE example_tcp_sessions_handover (id UInt64) ENGINE = Memory"); + Console.WriteLine($" First session created a temporary table and sees it: {await first.ExecuteScalarAsync(visible)}"); + } + + Console.WriteLine($" Disposed it. IsOpen = {first.IsOpen}"); + + await using IClickHouseTcpSession second = await client.OpenSessionAsync(); + Console.WriteLine($" Second session, from a pool of exactly one connection, sees: {await second.ExecuteScalarAsync(visible)}"); + Console.WriteLine(" Zero, so the first connection was closed rather than reused. That is the point: a"); + Console.WriteLine(" caller must never inherit another caller's temporary tables and settings."); + Console.WriteLine(" It also means nothing has to be dropped — and that a session costs a reconnect."); + } + + private static async Task SetRoleInASession(ClickHouseTcpClient admin) + { + Console.WriteLine("\n4. SET ROLE, which is what this transport has instead of a per-query role\n"); + Console.WriteLine(" ClickHouseTcpQueryOptions carries no Roles: there is nowhere on the wire to put one per"); + Console.WriteLine(" query. A session is the equivalent, and it is per connection rather than per query.\n"); + + await admin.ExecuteAsync($"CREATE OR REPLACE TABLE {RoleTable} (id UInt64) ENGINE = MergeTree ORDER BY id"); + await admin.ExecuteAsync($"CREATE ROLE OR REPLACE {RoleName}"); + await admin.ExecuteAsync($"GRANT SELECT ON {RoleTable} TO {RoleName}"); + await admin.ExecuteAsync($"CREATE USER OR REPLACE {RoleUser} IDENTIFIED WITH plaintext_password BY '{RoleUserPassword}'"); + await admin.ExecuteAsync($"GRANT {RoleName} TO {RoleUser}"); + Console.WriteLine($" Created user '{RoleUser}', role '{RoleName}' holding SELECT on '{RoleTable}'"); + + try + { + // Same server as everything else here; only the credentials differ. + var asUser = ExampleConfig.TcpBuilder(); + asUser.Username = RoleUser; + asUser.Password = RoleUserPassword; + + await using var client = new ClickHouseTcpClient(asUser.ToOptions()); + await using IClickHouseTcpSession session = await client.OpenSessionAsync(); + + Console.WriteLine($"\n Fresh session, granted roles active by default: {await session.ExecuteScalarAsync("SELECT toString(currentRoles())")}"); + + await session.ExecuteAsync("SET ROLE NONE"); + Console.WriteLine($" After SET ROLE NONE: {await session.ExecuteScalarAsync("SELECT toString(currentRoles())")}"); + + try + { + _ = await session.ExecuteScalarAsync($"SELECT count() FROM {RoleTable}"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" Reading the table now fails with {ex.Code} ({ex.RawCode}), so the grant really came from the role."); + + // A server-side error in a query the server accepted does not end the session: the connection is + // still good, only the query failed. + Console.WriteLine($" IsOpen after that error = {session.IsOpen}, so the session carries on"); + } + + await session.ExecuteAsync($"SET ROLE {RoleName}"); + Console.WriteLine($" After SET ROLE {RoleName}, the read works again: {await session.ExecuteScalarAsync($"SELECT count() FROM {RoleTable}")} rows"); + + // The client's own operations run over other connections, which never saw either SET ROLE. + Console.WriteLine($"\n Meanwhile an operation on the client, over a pooled connection: {await client.ExecuteScalarAsync("SELECT toString(currentRoles())")}"); + Console.WriteLine(" Untouched. A SET ROLE reaches exactly the connection it ran on."); + } + finally + { + await admin.ExecuteAsync($"DROP USER IF EXISTS {RoleUser}"); + await admin.ExecuteAsync($"DROP ROLE IF EXISTS {RoleName}"); + await admin.ExecuteAsync($"DROP TABLE IF EXISTS {RoleTable}"); + Console.WriteLine($"\n Dropped the user, the role and '{RoleTable}'. The temporary tables above needed no cleanup."); + } + } + + private static void WhatToRemember() + { + Console.WriteLine("\n5. Worth knowing before you open one\n"); + Console.WriteLine(" Keep it short. A session holds one of MaxPoolSize connections from OpenSessionAsync"); + Console.WriteLine(" until disposal, so as many sessions as the pool is wide leaves nothing for anything"); + Console.WriteLine(" else, and the next caller waits out PoolTimeout and then fails. See Tcp_017."); + Console.WriteLine(); + Console.WriteLine(" Finish what you stream. A StreamAsync or QueryAsync result holds the session until it"); + Console.WriteLine(" is read to the end or its enumerator is disposed — 'await foreach' does that for you."); + Console.WriteLine(" One left suspended mid-enumeration and never disposed cannot give its slot back at all,"); + Console.WriteLine(" which is the one thing here not demonstrated: showing it means leaking a connection."); + Console.WriteLine(); + Console.WriteLine(" Read IsOpen as a floor, not a promise. False is certain: the session is finished, its"); + Console.WriteLine(" server-side state is gone, and the answer is a new session rather than a retry. True"); + Console.WriteLine(" only means nothing is known to be wrong. A failed transport, a cancellation, or a"); + Console.WriteLine(" half-read stream ends a session; a server error in a query the server accepted does not."); + } +} diff --git a/examples/Tcp/Connection/Tcp_017_PoolTuning.cs b/examples/Tcp/Connection/Tcp_017_PoolTuning.cs new file mode 100644 index 000000000..56d7fe7be --- /dev/null +++ b/examples/Tcp/Connection/Tcp_017_PoolTuning.cs @@ -0,0 +1,373 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Driver.Examples; + +/// +/// Sizing the native client's connection pool: MinPoolSize, MaxPoolSize, PoolTimeout, +/// IdleTimeout, MaxConnectionLifetime, SweepInterval and PoolReusePolicy — what each +/// one does, and what it looks like when it acts. +/// +/// +/// One connection carries one query, so MaxPoolSize is the client's concurrency limit, for the whole +/// process rather than per caller. Everything below is measured: the counts come from the pool's own log and from +/// system.processes on the server, because nothing on the client reports how many connections are open, +/// idle or in use. +/// +/// +/// +/// The lifetime limits default to minutes, which an example cannot wait out, so the sections that show them set +/// them to milliseconds. The code path is the same one a 30-minute limit takes. +/// +/// +public static class TcpPoolTuning +{ + public static async Task Run() + { + TheKnobs(); + await ReuseAndTheOnlyWindowIntoThePool(); + await MaxPoolSizeCapsConcurrency(); + await PoolTimeoutExpires(); + await RetirementAndTheSweep(); + await LifoAgainstFifo(); + await WhatADataSourceShares(); + } + + private static void TheKnobs() + { + ClickHouseTcpClientOptions defaults = new(); + + Console.WriteLine("1. The pool keys and their defaults\n"); + Console.WriteLine($" MinPoolSize {defaults.MinPoolSize,-8} connections kept open when the pool can"); + Console.WriteLine($" MaxPoolSize {defaults.MaxPoolSize,-8} hard cap, and so the concurrency limit"); + Console.WriteLine($" PoolTimeout {defaults.PoolTimeout.TotalSeconds + "s",-8} wait for a slot before TimeoutException"); + Console.WriteLine($" IdleTimeout {defaults.IdleTimeout.TotalMinutes + "m",-8} unused for this long, and it is retired"); + Console.WriteLine($" MaxConnectionLifetime {defaults.MaxConnectionLifetime.TotalMinutes + "m",-8} open for this long, and it is retired"); + Console.WriteLine($" SweepInterval {"derived",-8} how often the pool looks for work to do"); + Console.WriteLine($" PoolReusePolicy {defaults.PoolReusePolicy,-8} which idle connection is handed out next"); + Console.WriteLine(); + Console.WriteLine(" TimeSpan.Zero opts out of IdleTimeout and MaxConnectionLifetime; PoolTimeout has to be"); + Console.WriteLine(" positive. A null SweepInterval derives the period as a quarter of the shorter of the two"); + Console.WriteLine(" limits, held between 1 and 30 seconds — 30 seconds at these defaults. The derived value"); + Console.WriteLine(" is not exposed, so the rule is the only way to know it."); + + // The same keys exist on the connection string, so a deployment can size the pool without a rebuild. + var builder = ExampleConfig.TcpBuilder(); + builder.MinPoolSize = 2; + builder.MaxPoolSize = 8; + builder.PoolTimeout = TimeSpan.FromSeconds(5); + builder.IdleTimeout = TimeSpan.FromSeconds(45); + builder.MaxConnectionLifetime = TimeSpan.FromMinutes(10); + builder.PoolReusePolicy = ClickHouseTcpPoolReusePolicy.Fifo; + + ClickHouseTcpClientOptions tuned = builder.ToOptions(); + Console.WriteLine("\n The same thing through the connection string (MinPoolSize=2;MaxPoolSize=8;...):"); + Console.WriteLine($" Min {tuned.MinPoolSize}, Max {tuned.MaxPoolSize}, PoolTimeout {tuned.PoolTimeout}, IdleTimeout {tuned.IdleTimeout}, Lifetime {tuned.MaxConnectionLifetime}, {tuned.PoolReusePolicy}"); + } + + private static async Task ReuseAndTheOnlyWindowIntoThePool() + { + Console.WriteLine("\n2. What the pool will tell you\n"); + Console.WriteLine(" There are no counters to read, so the pool's log is the window into it. These lines come"); + Console.WriteLine(" from the ClickHouse.Driver.Tcp.Pool and .Connection categories, at Debug and Trace.\n"); + + var capture = new LogCapture(); + await using (var client = new ClickHouseTcpClient(Options() with { LoggerFactory = capture })) + { + _ = await client.ExecuteScalarAsync("SELECT 1"); + _ = await client.ExecuteScalarAsync("SELECT 2"); + } + + Print(capture.Lines.Where(l => !l.StartsWith("Client", StringComparison.Ordinal))); + Console.WriteLine("\n Two queries, one connection: the first opened it, the second reused it, and the drain"); + Console.WriteLine(" at disposal closed it. 'its 2 operation' is that connection's use count."); + } + + private static async Task MaxPoolSizeCapsConcurrency() + { + Console.WriteLine("\n3. MaxPoolSize caps how many operations run at once\n"); + Console.WriteLine(" Four queries, each sleeping 150 ms, started together. A second client watches"); + Console.WriteLine(" system.processes to see how many of them the server is really running.\n"); + + await Measure(maxPoolSize: 2); + await Measure(maxPoolSize: 4); + + Console.WriteLine("\n The queries are not lost when the pool is full, only queued: each waits for a slot for"); + Console.WriteLine(" up to PoolTimeout. So MaxPoolSize is a throughput knob, and PoolTimeout is the deadline"); + Console.WriteLine(" on getting one of its slots."); + } + + private static async Task Measure(int maxPoolSize) + { + string marker = $"example_tcp_pool_cap_{maxPoolSize}"; + var capture = new LogCapture(); + + await using var observer = ExampleConfig.CreateTcpClient(); + await using var client = new ClickHouseTcpClient(Options() with { MaxPoolSize = maxPoolSize, LoggerFactory = capture }); + + var clock = Stopwatch.StartNew(); + Task work = Task.WhenAll(Enumerable.Range(0, 4).Select(_ => Task.Run(async () => + await client.ExecuteScalarAsync($"SELECT sleep(0.15) /* {marker} */")))); + + // The marker is a comment, so it appears in the query text the server reports. The observer's own query + // carries it too, hence the second condition. + string count = $"SELECT count() FROM system.processes WHERE query LIKE '%{marker}%' AND query NOT LIKE '%system.processes%'"; + + int mostSeen = 0; + while (!work.IsCompleted && clock.ElapsedMilliseconds < 5000) + { + mostSeen = Math.Max(mostSeen, Convert.ToInt32(await observer.ExecuteScalarAsync(count))); + await Task.Delay(20); + } + + await work; + long elapsed = clock.ElapsedMilliseconds; + + Console.WriteLine($" MaxPoolSize = {maxPoolSize}"); + Console.WriteLine($" connections opened, from the pool log : {capture.Count("opening one")}"); + Console.WriteLine($" most running at once, from the server : {mostSeen}"); + Console.WriteLine($" wall clock for all four : {elapsed} ms"); + } + + private static async Task PoolTimeoutExpires() + { + Console.WriteLine("\n4. PoolTimeout, when there is nothing left to hand out\n"); + + var capture = new LogCapture(); + await using var client = new ClickHouseTcpClient(Options() with + { + MaxPoolSize = 1, + PoolTimeout = TimeSpan.FromMilliseconds(250), + LoggerFactory = capture, + }); + + // A session pins its connection for its whole lifetime, so one session against a pool of one is an + // exhausted pool — no sleeping query needed. + await using IClickHouseTcpSession session = await client.OpenSessionAsync(); + Console.WriteLine(" MaxPoolSize = 1, PoolTimeout = 250 ms, and a session holds the only connection."); + + var clock = Stopwatch.StartNew(); + try + { + _ = await client.ExecuteScalarAsync("SELECT 1"); + } + catch (TimeoutException ex) + { + Console.WriteLine($"\n A query on the client threw TimeoutException after {clock.ElapsedMilliseconds} ms:"); + Console.WriteLine($" {ex.Message}"); + } + + Console.WriteLine("\n The pool logged it too:"); + Print(capture.Lines.Where(l => l.Contains("PoolTimeout", StringComparison.Ordinal))); + Console.WriteLine(); + Console.WriteLine(" Raising PoolTimeout only makes the caller wait longer for a pool that is too small."); + Console.WriteLine(" The message also names the other cause: a streamed result nobody finished still holds"); + Console.WriteLine(" its connection."); + } + + private static async Task RetirementAndTheSweep() + { + Console.WriteLine("\n5. Retiring connections: MaxConnectionLifetime, IdleTimeout, SweepInterval, MinPoolSize\n"); + + // Age is read at checkout and at return, so a 1 ms limit means no connection is ever reused. + var byAge = new LogCapture(); + await using (var client = new ClickHouseTcpClient(Options() with + { + MaxConnectionLifetime = TimeSpan.FromMilliseconds(1), + LoggerFactory = byAge, + })) + { + _ = await client.ExecuteScalarAsync("SELECT 1"); + _ = await client.ExecuteScalarAsync("SELECT 2"); + } + + Console.WriteLine(" MaxConnectionLifetime = 1 ms, two queries:"); + Print(byAge.Lines.Where(l => l.StartsWith("Pool", StringComparison.Ordinal))); + Console.WriteLine(" No reuse at all: the connection is over age by the time it comes back, so it is closed"); + Console.WriteLine(" on return and the next query opens another. That check is between operations, never"); + Console.WriteLine(" inside one, so no query is ever cut short by it.\n"); + + // Idle retirement is the sweep's work, so it happens without any operation to trigger it. + var byIdle = new LogCapture(); + await using (var client = new ClickHouseTcpClient(Options() with + { + IdleTimeout = TimeSpan.FromMilliseconds(150), + SweepInterval = TimeSpan.FromMilliseconds(100), + LoggerFactory = byIdle, + })) + { + _ = await client.ExecuteScalarAsync("SELECT 1"); + long waited = await WaitFor(byIdle, "Retired"); + Console.WriteLine(" IdleTimeout = 150 ms, SweepInterval = 100 ms, one query then nothing:"); + Print(byIdle.Lines.Where(l => l.Contains("Retired", StringComparison.Ordinal))); + Console.WriteLine($" The sweep retired it {waited} ms after the query, with no operation involved."); + } + + Console.WriteLine(); + + // The same sweep restores the floor, which is why MinPoolSize needs no traffic to take effect. + var byFloor = new LogCapture(); + await using (var client = new ClickHouseTcpClient(Options() with + { + MinPoolSize = 3, + MaxPoolSize = 5, + SweepInterval = TimeSpan.FromMilliseconds(100), + LoggerFactory = byFloor, + })) + { + long waited = await WaitFor(byFloor, "Connected to ClickHouse", occurrences: 3); + Console.WriteLine(" MinPoolSize = 3, SweepInterval = 100 ms, and not one query run:"); + Console.WriteLine($" connections opened by the sweep: {byFloor.Count("Connected to ClickHouse")} after {waited} ms"); + } + + Console.WriteLine(); + Console.WriteLine(" The floor and IdleTimeout multiply: neither limit respects MinPoolSize, so a quiet pool"); + Console.WriteLine(" retires its connections and the sweep opens replacements. A floor of 10 against a"); + Console.WriteLine(" 5-second idle limit is 10 handshakes every 5 seconds from an idle application. Size the"); + Console.WriteLine(" two together. Set IdleTimeout below the shortest idle timeout on the path to the server:"); + Console.WriteLine(" a proxy that drops an idle connection without a FIN leaves one that only looks alive."); + } + + private static async Task LifoAgainstFifo() + { + Console.WriteLine("\n6. PoolReusePolicy: which idle connection comes back out\n"); + Console.WriteLine(" Three queries at once fill a pool of three, then three run one after another. The use"); + Console.WriteLine(" count in the reuse line says whether they landed on one connection or on all three.\n"); + + foreach (ClickHouseTcpPoolReusePolicy policy in new[] { ClickHouseTcpPoolReusePolicy.Lifo, ClickHouseTcpPoolReusePolicy.Fifo }) + { + var capture = new LogCapture(); + await using var client = new ClickHouseTcpClient(Options() with + { + MaxPoolSize = 3, + PoolReusePolicy = policy, + LoggerFactory = capture, + }); + + await Task.WhenAll(Enumerable.Range(0, 3).Select(_ => Task.Run(async () => + await client.ExecuteScalarAsync("SELECT sleep(0.15)")))); + + for (int i = 0; i < 3; i++) + { + _ = await client.ExecuteScalarAsync("SELECT 1"); + } + + // "Reusing a pooled connection, its N operation, ..." — N is that connection's use count, which is + // what tells one policy from the other. + IEnumerable counts = capture.Lines + .Where(l => l.Contains("Reusing", StringComparison.Ordinal)) + .Select(l => l[(l.IndexOf("its ", StringComparison.Ordinal) + 4)..].Split(' ')[0]); + + string shape = policy == ClickHouseTcpPoolReusePolicy.Lifo + ? "one connection, used again and again" + : "each of the three in turn"; + + Console.WriteLine($" {policy,-4} use count of the connection each sequential query got: {string.Join(", ", counts)} ({shape})"); + } + + Console.WriteLine(); + Console.WriteLine(" Lifo keeps returning to the connection that came back last, so traffic concentrates on a"); + Console.WriteLine(" hot few and the rest go idle and close — a pool sized for peak load costs little"); + Console.WriteLine(" off-peak. Fifo spreads the work, so under steady load every connection is used again"); + Console.WriteLine(" inside its idle window and the whole pool stays warm. Both are equally correct: age,"); + Console.WriteLine(" idleness and liveness are checked whichever end the connection comes from."); + } + + private static async Task WhatADataSourceShares() + { + Console.WriteLine("\n7. What a ClickHouseTcpDataSource shares\n"); + + await using var dataSource = new ClickHouseTcpDataSource(Options() with { MaxPoolSize = 8 }); + + Console.WriteLine($" One data source owns one client, and that client owns one pool: {dataSource.Options.MaxPoolSize} connections"); + Console.WriteLine(" for every consumer that is injected with it (Tcp_003 registers one). So MaxPoolSize is"); + Console.WriteLine(" the whole application's concurrency budget, not each service's."); + Console.WriteLine(); + Console.WriteLine(" Two data sources, or two clients built with 'new', are two pools that share nothing but"); + Console.WriteLine(" the server. That is what a keyed registration per endpoint buys, and it is also the"); + Console.WriteLine(" accident behind a client built per request: every one pays a handshake and none of them"); + Console.WriteLine(" reuses anything."); + Console.WriteLine(); + Console.WriteLine(" Sizing, in short: MaxPoolSize at or a little above the number of operations you want in"); + Console.WriteLine(" flight, remembering each inserting connection can buffer MaxSendBufferBytes (Tcp_019);"); + Console.WriteLine(" MinPoolSize only where a cold first query matters; and one slot per session you hold."); + } + + private static ClickHouseTcpClientOptions Options() => ExampleConfig.TcpBuilder().ToOptions(); + + private static void Print(IEnumerable lines) + { + foreach (string line in lines) + { + Console.WriteLine($" {line}"); + } + } + + /// Waits for the pool to log something, so the example never sleeps longer than it must. + private static async Task WaitFor(LogCapture capture, string contains, int occurrences = 1) + { + var clock = Stopwatch.StartNew(); + while (clock.ElapsedMilliseconds < 3000 && capture.Count(contains) < occurrences) + { + await Task.Delay(25); + } + + return clock.ElapsedMilliseconds; + } + + /// + /// An that keeps the lines instead of printing them, so the example can show only + /// the ones under discussion. A real application passes the container's factory; see Tcp_003. + /// + private sealed class LogCapture : ILoggerFactory + { + private readonly List lines = []; + + public IReadOnlyList Lines + { + get + { + lock (lines) + { + return lines.ToArray(); + } + } + } + + public int Count(string contains) + => Lines.Count(l => l.Contains(contains, StringComparison.Ordinal)); + + public ILogger CreateLogger(string categoryName) => new Sink(categoryName, lines); + + public void AddProvider(ILoggerProvider provider) + { + } + + public void Dispose() + { + } + + private sealed class Sink(string category, List lines) : ILogger + { + // The client asks before formatting, so answering true is what makes Trace-level lines appear. + public bool IsEnabled(LogLevel logLevel) => true; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + lock (lines) + { + lines.Add($"{category[(category.LastIndexOf('.') + 1)..]}: {formatter(state, exception)}"); + } + } + } + } +} diff --git a/examples/Tcp/Connection/Tcp_018_Tls.cs b/examples/Tcp/Connection/Tcp_018_Tls.cs new file mode 100644 index 000000000..782eaa1ec --- /dev/null +++ b/examples/Tcp/Connection/Tcp_018_Tls.cs @@ -0,0 +1,259 @@ +using System.Net.Security; +using System.Security.Authentication; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// Encrypting the native transport: UseTls, TlsServerName, TlsCaCertificatePath, +/// TlsAllowInvalidCertificates and the ConfigureTls hook — plus the port, which follows +/// UseTls to 9440 unless one is given. +/// +/// +/// The native handshake sends the password as plaintext, so on any untrusted network TLS is the only thing +/// protecting the credentials. Nothing above the transport changes: the protocol sends the same bytes either way. +/// +/// +/// +/// The examples' server is plaintext, so the connecting part of this example is opt-in: set +/// CLICKHOUSE_TCP_TLS_CONNECTION_STRING to a secure endpoint and it runs. Everything else here needs no +/// server at all, because the client checks its TLS configuration when it is constructed — including reading the +/// certificate authority file. +/// +/// +public static class TcpTls +{ + private const string TlsConnectionStringVariable = "CLICKHOUSE_TCP_TLS_CONNECTION_STRING"; + + public static async Task Run() + { + WhatTlsIsFor(); + ThePortFollowsUseTls(); + BothWaysToSetIt(); + CheckedAtConstruction(); + PinningAnAuthorityReplacesTheTrustStore(); + TheEscapeHatch(); + await ConnectIfAnEndpointWasGiven(); + } + + private static void WhatTlsIsFor() + { + Console.WriteLine("1. Why TLS, on this transport\n"); + Console.WriteLine(" The native protocol's first packet carries the username and password in the clear, and"); + Console.WriteLine(" then every block of data. UseTls encrypts the socket underneath all of it. The protocol"); + Console.WriteLine(" bytes are identical either way, so nothing above the transport changes."); + Console.WriteLine(); + Console.WriteLine(" TLS is not negotiated in band. The server has to be listening for secure native"); + Console.WriteLine(" connections (tcp_port_secure, conventionally 9440), and a TLS client pointed at a"); + Console.WriteLine(" plaintext port fails its handshake rather than falling back to plaintext."); + } + + private static void ThePortFollowsUseTls() + { + Console.WriteLine("\n2. The port comes from UseTls when you do not give one\n"); + + // Port is int?, and null is not "0" but "derive it". ToString shows the port a connection would dial. + ClickHouseTcpClientOptions plain = ExampleConfig.TcpBuilder().ToOptions() with { Port = null }; + ClickHouseTcpClientOptions secure = plain with { UseTls = true }; + ClickHouseTcpClientOptions explicitPort = secure with { Port = 19440 }; + + Console.WriteLine($" UseTls = false, Port unset : {plain}"); + Console.WriteLine($" UseTls = true, Port unset : {secure}"); + Console.WriteLine($" UseTls = true, Port 19440 : {explicitPort}"); + Console.WriteLine(); + Console.WriteLine(" So switching a deployment to TLS is one key, as long as the server uses the conventional"); + Console.WriteLine(" port. An explicit Port is always used as given."); + } + + private static void BothWaysToSetIt() + { + Console.WriteLine("\n3. The same four keys, in a connection string and on the options record\n"); + + var builder = ExampleConfig.TcpBuilder(); + builder.Port = null; + builder.UseTls = true; + builder.TlsServerName = "clickhouse.internal"; + + // Naming an authority file here touches nothing: it is read when a client is constructed, and this + // example never constructs one from these options. + builder.TlsCaCertificatePath = "/etc/ssl/ca.pem"; + + ClickHouseTcpClientOptions fromBuilder = builder.ToOptions(); + + Console.WriteLine(" UseTls=true;TlsServerName=clickhouse.internal;TlsCaCertificatePath=/etc/ssl/ca.pem"); + Console.WriteLine(" TlsAllowInvalidCertificates=false"); + Console.WriteLine(); + Console.WriteLine($" builder.ToOptions() : {fromBuilder}"); + Console.WriteLine($" TlsServerName {fromBuilder.TlsServerName}"); + Console.WriteLine($" TlsCaCertificatePath {fromBuilder.TlsCaCertificatePath ?? "(null: the host trust store)"}"); + Console.WriteLine($" TlsAllowInvalidCertificates {fromBuilder.TlsAllowInvalidCertificates}"); + Console.WriteLine(" Port left unset, so the rendered options above show the 9440 it resolved to"); + Console.WriteLine(); + Console.WriteLine(" TlsServerName is the name presented as SNI and matched against the certificate; it"); + Console.WriteLine(" defaults to Host, so set it only when Host is an address or an alias the certificate"); + Console.WriteLine(" does not name. ConfigureTls is the one TLS setting with no connection-string key: it is"); + Console.WriteLine(" a delegate, so it can only be set in code."); + } + + private static void CheckedAtConstruction() + { + Console.WriteLine("\n4. What is refused before anything connects\n"); + Console.WriteLine(" Every line below comes from constructing a client, with no server involved.\n"); + + ClickHouseTcpClientOptions plaintext = ExampleConfig.TcpBuilder().ToOptions(); + + // A TLS setting on a client that does not use TLS is refused rather than ignored. Ignoring it is how a + // connection meant to be encrypted ends up in the clear with a configured authority as the only evidence. + Refused("TlsServerName, UseTls left false", plaintext with { TlsServerName = "clickhouse.internal" }); + Refused("TlsAllowInvalidCertificates, UseTls left false", plaintext with { TlsAllowInvalidCertificates = true }); + Refused("TlsCaCertificatePath, UseTls left false", plaintext with { TlsCaCertificatePath = "/etc/ssl/ca.pem" }); + Refused("ConfigureTls, UseTls left false", plaintext with { ConfigureTls = _ => { } }); + + ClickHouseTcpClientOptions tls = plaintext with { UseTls = true }; + + // Contradictory rather than merely redundant: with validation off, the authority would be read and never + // consulted. + Refused("TlsAllowInvalidCertificates and TlsCaCertificatePath together", tls with + { + TlsAllowInvalidCertificates = true, + TlsCaCertificatePath = "/etc/ssl/ca.pem", + }); + + Refused("A blank TlsCaCertificatePath", tls with { TlsCaCertificatePath = " " }); + + // The authority file is read once, when the client is constructed, so a wrong path or an unparseable file + // fails here instead of on the first connection — or worse, on the first reconnect at 3am. + string missing = Path.Combine(Path.GetTempPath(), "example-tcp-tls-no-such-ca.pem"); + Refused("A TlsCaCertificatePath that does not exist", tls with { TlsCaCertificatePath = missing }); + + string notACertificate = Path.Combine(Path.GetTempPath(), "example-tcp-tls-not-a-ca.pem"); + try + { + File.WriteAllText(notACertificate, "these are not the certificates you are looking for\n"); + Refused("A TlsCaCertificatePath that is not a PEM certificate", tls with { TlsCaCertificatePath = notACertificate }); + } + finally + { + File.Delete(notACertificate); + } + + // The connection-string parser is strict about these two keys for the same reason: a value it cannot read + // must not quietly become the plaintext default. + try + { + _ = ClickHouseTcpClientOptions.FromConnectionString("Host=clickhouse.example;UseTls=perhaps"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" UseTls=perhaps in a connection string -> {ex.GetType().Name}"); + Console.WriteLine($" {ex.Message}"); + } + } + + private static void Refused(string what, ClickHouseTcpClientOptions options) + { + try + { + // Constructed only to be refused: nothing here reaches a socket. + using var client = new ClickHouseTcpClient(options); + Console.WriteLine($" {what} -> accepted, which is not what this example expected"); + } + catch (Exception ex) when (ex is ArgumentException or IOException) + { + Console.WriteLine($" {what} -> {ex.GetType().Name}"); + Console.WriteLine($" {ex.Message.Split(" (Parameter")[0]}"); + } + } + + private static void PinningAnAuthorityReplacesTheTrustStore() + { + Console.WriteLine("\n5. TlsCaCertificatePath replaces the host's trust store — it does not add to it\n"); + Console.WriteLine(" Set it and the server must chain to one of the authorities in that file. A certificate"); + Console.WriteLine(" the host would have accepted on its own is then refused. That is the point of naming an"); + Console.WriteLine(" authority: an additive check would still accept a certificate mis-issued by any of the"); + Console.WriteLine(" hundred-odd public authorities the host trusts."); + Console.WriteLine(); + Console.WriteLine(" So a private authority is the case it serves. Pointing it at a public root to 'also"); + Console.WriteLine(" allow' a private one does not work, and pinning it in front of a server whose"); + Console.WriteLine(" certificate is publicly issued breaks that server."); + Console.WriteLine(); + Console.WriteLine(" The file must hold at least one self-issued root, which is what the chain is anchored"); + Console.WriteLine(" to; it may also hold intermediates, which are used only to build a chain to an anchor."); + Console.WriteLine(" Host name matching still happens either way — pinning roots does not replace it."); + Console.WriteLine(); + Console.WriteLine(" TlsAllowInvalidCertificates is the other thing entirely: it stops the client checking"); + Console.WriteLine(" that the peer is the server it asked for, so anyone who can intercept the connection can"); + Console.WriteLine(" present any certificate and read the handshake password. For a private authority, pin"); + Console.WriteLine(" the root and keep the check."); + } + + private static void TheEscapeHatch() + { + Console.WriteLine("\n6. ConfigureTls, for what the four keys do not cover\n"); + + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions() with + { + UseTls = true, + ConfigureTls = tls => + { + tls.EnabledSslProtocols = SslProtocols.Tls13; + + // Where a client certificate would go: tls.ClientCertificates = new X509Certificate2Collection(cert); + }, + }; + + // The client calls this once per connection, just before the handshake. Calling it here, on a fresh + // options object, is only to show what it sets. + var authentication = new SslClientAuthenticationOptions(); + options.ConfigureTls(authentication); + + Console.WriteLine($" The hook set EnabledSslProtocols = {authentication.EnabledSslProtocols}"); + Console.WriteLine(); + Console.WriteLine(" It runs last, after everything the four keys set, which is what makes it an escape"); + Console.WriteLine(" hatch — client certificates, a protocol floor, cipher suites, a validation callback of"); + Console.WriteLine(" your own — and also what lets it weaken the transport in two ways that are easy to miss:"); + Console.WriteLine(); + Console.WriteLine(" replacing RemoteCertificateValidationCallback drops the check the keys configured;"); + Console.WriteLine(" clearing TargetHost stops the server name being matched at all, while chain"); + Console.WriteLine(" validation still appears to run."); + Console.WriteLine(); + Console.WriteLine(" With TlsCaCertificatePath set, a CertificateChainPolicy is already in place and the hook"); + Console.WriteLine(" receives it and may edit it. .NET then ignores CertificateRevocationCheckMode, so"); + Console.WriteLine(" revocation goes through that policy's own RevocationMode."); + } + + private static async Task ConnectIfAnEndpointWasGiven() + { + Console.WriteLine("\n7. Connecting over TLS\n"); + + string? connectionString = Environment.GetEnvironmentVariable(TlsConnectionStringVariable); + if (string.IsNullOrWhiteSpace(connectionString)) + { + Console.WriteLine($" Skipped: {TlsConnectionStringVariable} is not set."); + Console.WriteLine(); + Console.WriteLine(" The server these examples run against speaks plaintext on 9000, and the one CI"); + Console.WriteLine(" starts publishes 8123 and 9000 only, so there is no secure port to dial. Nothing"); + Console.WriteLine(" here fakes one: a certificate invented to make a connection succeed would teach the"); + Console.WriteLine(" wrong thing, and turning validation off to get past it would teach something worse."); + Console.WriteLine(); + Console.WriteLine(" To run this section, point it at a server listening on tcp_port_secure:"); + Console.WriteLine($" export {TlsConnectionStringVariable}=\"Host=my-host;UseTls=true;Username=default;Password=...\""); + Console.WriteLine(" A ClickHouse Cloud service is the easy case: its native endpoint is TLS on 9440 with"); + Console.WriteLine(" a publicly issued certificate, so UseTls=true and no other TLS key is needed."); + return; + } + + Console.WriteLine($" {TlsConnectionStringVariable} is set, so connecting over TLS."); + + await using var client = new ClickHouseTcpClient(connectionString); + ClickHouseTcpClientOptions options = client.Options; + + Console.WriteLine($" {options}"); + Console.WriteLine($" UseTls {options.UseTls}, TlsServerName {options.TlsServerName ?? "(Host)"}, " + + $"CA {options.TlsCaCertificatePath ?? "(host trust store)"}, AllowInvalid {options.TlsAllowInvalidCertificates}"); + + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + object now = await client.ExecuteScalarAsync("SELECT 'encrypted'"); + Console.WriteLine($" Connected to {server}: SELECT returned {now}"); + } +} diff --git a/examples/Tcp/Connection/Tcp_019_Timeouts.cs b/examples/Tcp/Connection/Tcp_019_Timeouts.cs new file mode 100644 index 000000000..dd7076022 --- /dev/null +++ b/examples/Tcp/Connection/Tcp_019_Timeouts.cs @@ -0,0 +1,323 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Driver.Examples; + +/// +/// The native client's deadlines and limits: DialTimeout, ReadTimeout, PoolTimeout, +/// StatementMaxLength and MaxSendBufferBytes. +/// +/// +/// The three deadlines cover three different phases and never overlap: PoolTimeout bounds the wait for a +/// pool slot, DialTimeout bounds the connect and handshake that may follow it, and ReadTimeout bounds +/// how long the server may stay silent while a response is being read. That last one is the one to +/// understand — it measures silence, not duration, so a query that streams for an hour never trips it. +/// +/// +/// +/// None of them bounds a whole operation. That is what a CancellationToken is for, and every method takes +/// one; Tcp_022_Cancellation is about what cancelling does to the connection. +/// +/// +public static class TcpTimeouts +{ + public static async Task Run() + { + WhichDeadlineCoversWhat(); + await DialingTheWrongThing(); + await ReadTimeoutMeasuresSilence(); + PoolTimeoutInOneLine(); + await StatementMaxLengthCapsWhatIsLogged(); + MaxSendBufferBytesAndTheValues(); + } + + private static void WhichDeadlineCoversWhat() + { + ClickHouseTcpClientOptions defaults = new(); + + Console.WriteLine("1. Four bounds, four different phases\n"); + Console.WriteLine($" PoolTimeout {defaults.PoolTimeout.TotalSeconds,5}s waiting for one of MaxPoolSize connections"); + Console.WriteLine($" DialTimeout {defaults.DialTimeout.TotalSeconds,5}s socket connect plus the protocol handshake"); + Console.WriteLine($" ReadTimeout {defaults.ReadTimeout.TotalSeconds,5}s the longest silence allowed while reading a response"); + Console.WriteLine(" CancellationToken the whole operation, and the only one that bounds it"); + Console.WriteLine(); + Console.WriteLine(" A checkout that has to open a connection can therefore take up to PoolTimeout plus"); + Console.WriteLine(" DialTimeout: the two apply to different phases, so they add rather than overlap."); + Console.WriteLine(); + Console.WriteLine(" ReadTimeout = TimeSpan.Zero removes the deadline and leaves the caller's token as the"); + Console.WriteLine(" only bound. PoolTimeout and DialTimeout must be positive — there is no opting out of"); + Console.WriteLine(" those, because a wait with no bound at all is how a request hangs forever."); + } + + private static async Task DialingTheWrongThing() + { + Console.WriteLine("\n2. DialTimeout, and the two dial failures that are not timeouts\n"); + + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); + + // Refused: the port answers at once with a reset, so the deadline is never involved. + var clock = Stopwatch.StartNew(); + Exception? refused = await Failing(options with { Port = 1, DialTimeout = TimeSpan.FromSeconds(2) }); + Console.WriteLine($" Nothing listening on the port, after {clock.ElapsedMilliseconds} ms:"); + Console.WriteLine($" {Describe(refused)}"); + Console.WriteLine($" inner: {refused?.InnerException?.GetType().Name} — a refusal is instant, so DialTimeout never came up"); + + // The HTTP port. Both interfaces are ClickHouse, but they speak different protocols, and the native client + // reads the HTTP server's reply as a protocol packet. + clock.Restart(); + Exception? wrongPort = await Failing(options with { Port = ExampleConfig.HttpPort, DialTimeout = TimeSpan.FromSeconds(2) }); + Console.WriteLine($"\n The HTTP port ({ExampleConfig.HttpPort}) instead of the native one, after {clock.ElapsedMilliseconds} ms:"); + Console.WriteLine($" {Describe(wrongPort)}"); + Console.WriteLine(" Packet type 72 is 'H', the first byte of the HTTP response. Not a timeout either."); + + // What DialTimeout is actually for: a peer that accepts the connection and then says nothing. A firewall, + // or a load balancer with no healthy backend behind it, looks exactly like this local listener. + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + int silentPort = ((IPEndPoint)listener.LocalEndpoint).Port; + Task accepted = AcceptOneAndSayNothing(listener); + + try + { + clock.Restart(); + Exception? silent = await Failing(options with + { + Host = "127.0.0.1", + Port = silentPort, + DialTimeout = TimeSpan.FromMilliseconds(300), + }); + + Console.WriteLine("\n A socket that accepts and never answers, with DialTimeout = 300 ms:"); + Console.WriteLine($" {Describe(silent)}"); + Console.WriteLine($" ... after {clock.ElapsedMilliseconds} ms. The connect succeeded; it is the handshake that never"); + Console.WriteLine(" finished. DialTimeout covers both, which is why an endpoint that answers the socket"); + Console.WriteLine(" and nothing else is bounded at all."); + } + finally + { + listener.Stop(); + (await accepted)?.Dispose(); + } + } + + /// Pings a server that is expected to be unreachable, and reports why it was. + private static async Task Failing(ClickHouseTcpClientOptions options) + { + try + { + await using var client = new ClickHouseTcpClient(options); + await client.PingAsync(); + return null; + } + catch (Exception ex) + { + return ex; + } + } + + private static string Describe(Exception? failure) + => failure is null ? "it answered, which is not what this example expected" : $"{failure.GetType().Name}: {failure.Message}"; + + private static async Task AcceptOneAndSayNothing(TcpListener listener) + { + try + { + return await listener.AcceptTcpClientAsync(); + } + catch (Exception ex) when (ex is SocketException or ObjectDisposedException) + { + // The listener was stopped first, which is the normal ending here. + return null; + } + } + + private static async Task ReadTimeoutMeasuresSilence() + { + Console.WriteLine("\n3. ReadTimeout is an idle deadline, not a time limit on the query\n"); + + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); + + // sleepEachRow with max_block_size = 1 makes the server send one row at a time with a gap between them, + // which is what a slow-but-alive server looks like from here. + await using (var chatty = new ClickHouseTcpClient(options with { ReadTimeout = TimeSpan.FromMilliseconds(250) })) + { + var clock = Stopwatch.StartNew(); + int rows = 0; + await foreach (object[] row in chatty.QueryAsync( + "SELECT number, sleepEachRow(0.05) FROM numbers(8) SETTINGS max_block_size = 1")) + { + rows++; + } + + Console.WriteLine(" ReadTimeout = 250 ms, 8 rows arriving 50 ms apart:"); + Console.WriteLine($" read {rows} rows in {clock.ElapsedMilliseconds} ms — longer than the deadline, and it never fired."); + Console.WriteLine(" Every byte that arrives resets the clock, so total duration is not what it bounds."); + } + + await OneGapTooWide(options); + + // The other half of "silence": a slow consumer is not a silent server. + await using (var pausing = new ClickHouseTcpClient(options with { ReadTimeout = TimeSpan.FromMilliseconds(150) })) + { + var clock = Stopwatch.StartNew(); + int rows = 0; + await foreach (object[] row in pausing.QueryAsync("SELECT number FROM numbers(4) SETTINGS max_block_size = 1")) + { + rows++; + + // Holding each row for longer than the deadline before asking for the next one. + await Task.Delay(200); + } + + Console.WriteLine("\n ReadTimeout = 150 ms, and a consumer that sits on each row for 200 ms:"); + Console.WriteLine($" read {rows} rows in {clock.ElapsedMilliseconds} ms, no timeout. The clock runs only while"); + Console.WriteLine(" the client is waiting on the transport, so your own processing time is never on it."); + } + + // The opt-out, for a stream that is legitimately silent for a long time. + await using (var unbounded = new ClickHouseTcpClient(options with { ReadTimeout = TimeSpan.Zero })) + { + var clock = Stopwatch.StartNew(); + _ = await unbounded.ExecuteScalarAsync("SELECT sleepEachRow(0.4) FROM numbers(1)"); + Console.WriteLine($"\n ReadTimeout = TimeSpan.Zero, a 400 ms silence: completed in {clock.ElapsedMilliseconds} ms."); + Console.WriteLine(" With no deadline the caller's CancellationToken is the only bound left. Prefer a"); + Console.WriteLine(" generous ReadTimeout to none: what it catches is a connection dropped without a"); + Console.WriteLine(" FIN, which nothing else notices and TCP alone takes about fifteen minutes to give up on."); + } + } + + /// + /// One silence wider than the deadline, and what the pool then does with the connection. Its own method so + /// that the logger factory below is disposed — and its lines flushed to the console — before the next + /// section prints. + /// + private static async Task OneGapTooWide(ClickHouseTcpClientOptions options) + { + // The pool's own lines, at Trace, because the reuse line a healthy connection produces is a Trace line + // (Tcp_017 shows what one looks like). Its absence below is the evidence. + using ILoggerFactory poolLog = LoggerFactory.Create(builder => builder + .AddFilter((category, _) => category == "ClickHouse.Driver.Tcp.Pool") + .AddSimpleConsole(console => console.SingleLine = true) + .SetMinimumLevel(LogLevel.Trace)); + + await using var strict = new ClickHouseTcpClient(options with + { + ReadTimeout = TimeSpan.FromMilliseconds(150), + MaxPoolSize = 1, + LoggerFactory = poolLog, + }); + + Console.WriteLine("\n ReadTimeout = 150 ms, the same query with 500 ms between rows, and the pool's own lines:"); + + var clock = Stopwatch.StartNew(); + try + { + await foreach (object[] row in strict.QueryAsync( + "SELECT number, sleepEachRow(0.5) FROM numbers(3) SETTINGS max_block_size = 1")) + { + } + } + catch (TimeoutException ex) + { + Console.WriteLine($" TimeoutException after {clock.ElapsedMilliseconds} ms: {ex.Message}"); + } + + // A second query on the same client, whose pool holds exactly one connection. Had the timed-out + // connection gone back into the pool, this is the one that would have got it. + _ = await strict.ExecuteScalarAsync("SELECT 1"); + + Console.WriteLine(" The pool closed that connection instead of pooling it — 'no longer reusable' — and"); + Console.WriteLine(" the query after it opened another rather than reusing one. A socket that stopped"); + Console.WriteLine(" answering mid-response is of no use to the next caller."); + } + + private static void PoolTimeoutInOneLine() + { + Console.WriteLine("\n4. PoolTimeout\n"); + Console.WriteLine(" The third deadline belongs to the pool, so Tcp_017_PoolTuning demonstrates it: with"); + Console.WriteLine(" MaxPoolSize connections in use, the next operation waits PoolTimeout for a free one and"); + Console.WriteLine(" then throws TimeoutException. Two things hold a connection longer than a caller expects —"); + Console.WriteLine(" a session, for its whole lifetime, and a streamed result nobody finished reading."); + } + + private static async Task StatementMaxLengthCapsWhatIsLogged() + { + Console.WriteLine("\n5. StatementMaxLength, which caps the query text that leaves the client\n"); + Console.WriteLine(" It bounds two channels: the Debug log line below, and the db.query.text span attribute"); + Console.WriteLine(" that IncludeSqlInActivityTags turns on. The default is 5 — a stub, not a statement — so"); + Console.WriteLine(" recording query text is something you ask for.\n"); + + const string sql = "SELECT 'a statement long enough to show the cut'"; + + foreach (int max in new[] { 5, 60 }) + { + // Only the client category, so the pool and connection lines stay out of the way. A real application + // configures this through the container; see Tcp_003. + using ILoggerFactory factory = LoggerFactory.Create(builder => builder + .AddFilter((category, level) => category == "ClickHouse.Driver.Tcp.Client" && level >= LogLevel.Debug) + .AddSimpleConsole(console => console.SingleLine = true) + .SetMinimumLevel(LogLevel.Debug)); + + Console.WriteLine($" StatementMaxLength = {max}, and the client's own log lines that follow:"); + + await using (var client = new ClickHouseTcpClient(ExampleConfig.TcpBuilder().ToOptions() with + { + LoggerFactory = factory, + StatementMaxLength = max, + })) + { + _ = await client.ExecuteScalarAsync(sql); + } + + // The factory is disposed at the end of this iteration, which drains the console logger, so its lines + // land before the next heading prints. + } + + Console.WriteLine(); + Console.WriteLine($" The statement was {sql.Length} characters, so at 5 the log line carries a stub of it and"); + Console.WriteLine(" at 60 the whole thing. Zero or less keeps the text out even where the span attribute is on."); + } + + private static void MaxSendBufferBytesAndTheValues() + { + ClickHouseTcpClientOptions defaults = new(); + + Console.WriteLine("\n6. MaxSendBufferBytes, and what the constructor refuses\n"); + Console.WriteLine($" MaxSendBufferBytes defaults to {defaults.MaxSendBufferBytes / (1024 * 1024)} MiB. It is a soft cap on the client's send"); + Console.WriteLine(" buffer during an insert: while a wire block is written, buffered bytes are flushed to the"); + Console.WriteLine(" socket whenever they exceed it. Soft, because a single column larger than the cap still"); + Console.WriteLine(" buffers in full."); + Console.WriteLine(); + Console.WriteLine(" It is independent of MaxRowsPerBlock (Tcp_009), which decides how large a block is; this"); + Console.WriteLine(" decides how much of one is held in memory on the way out. Peak send-buffer memory is"); + Console.WriteLine($" about MaxSendBufferBytes × MaxPoolSize — {defaults.MaxSendBufferBytes / (1024 * 1024)} MiB × {defaults.MaxPoolSize} at the defaults — when every"); + Console.WriteLine(" connection is inserting at once. Nothing reports how much is buffered, so this one is a"); + Console.WriteLine(" sizing decision rather than something to watch."); + Console.WriteLine(); + Console.WriteLine(" Every value here is checked when the client is constructed, not on first use:\n"); + + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); + + Refused("MaxSendBufferBytes = 0", options with { MaxSendBufferBytes = 0 }); + Refused("ReadTimeout = -1s", options with { ReadTimeout = TimeSpan.FromSeconds(-1) }); + Refused("PoolTimeout = TimeSpan.Zero", options with { PoolTimeout = TimeSpan.Zero }); + Refused("DialTimeout = 30 days", options with { DialTimeout = TimeSpan.FromDays(30) }); + } + + private static void Refused(string what, ClickHouseTcpClientOptions options) + { + try + { + // Never reaches a socket, so a synchronous Dispose is all this needs. + using var client = new ClickHouseTcpClient(options); + Console.WriteLine($" {what} -> accepted, which is not what this example expected"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" {what,-28} -> {ex.GetType().Name}: {ex.Message.Split(" (Parameter")[0]}"); + } + } +} From cd8c04149c5f923dc540120ca76f1c591c7834f6 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 15:45:01 +0200 Subject: [PATCH 10/16] Add the native protocol's advanced examples Six under examples/Tcp/Advanced/: settings precedence and query ids; the progress, profile-info and profile-event callbacks, with the timeline showing progress arriving while the rows do; cancellation, including what the server records and why the connection is closed rather than pooled; the exception hierarchy and which errors are worth retrying; compression measured in bytes on the wire; and the server info a caller gates behaviour on. Tcp_024 measures wire size through a counting socket and prints no wall-clock ranking: over loopback there is no bandwidth to save, so timing there measures the CPU cost and none of the benefit. The bytes say the client's codec does not decide what the server sends - a zstd client reads an LZ4 response, because the query packet carries one flag and the server frames with its own network_compression_method - while it does decide what an insert writes. Tcp_020 and Tcp_023 each hold a connection busy to demonstrate a real transient failure. Both start the holder with AsTask() rather than Task.Run, so the query packet is sent on the caller's thread: queued behind a busy thread pool, the holder could start second and be the query that was refused. Tcp_023 also lets only the retrying side declare the concurrency limit, so which query loses is fixed by construction rather than by ordering. Tcp_020 runs its unparseable-setting-value case on a throwaway client. That error is raised while the server parses the settings list, so it closes the socket and the pool keeps the connection, and the next operation on it fails. Co-Authored-By: Claude Opus 5 (1M context) --- examples/Program.cs | 29 ++ examples/README.md | 9 + .../Advanced/Tcp_020_SettingsAndQueryId.cs | 323 ++++++++++++++ .../Advanced/Tcp_021_ProgressAndStatistics.cs | 247 +++++++++++ examples/Tcp/Advanced/Tcp_022_Cancellation.cs | 303 ++++++++++++++ .../Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs | 396 ++++++++++++++++++ examples/Tcp/Advanced/Tcp_024_Compression.cs | 350 ++++++++++++++++ examples/Tcp/Advanced/Tcp_025_ServerInfo.cs | 187 +++++++++ 8 files changed, 1844 insertions(+) create mode 100644 examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs create mode 100644 examples/Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs create mode 100644 examples/Tcp/Advanced/Tcp_022_Cancellation.cs create mode 100644 examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs create mode 100644 examples/Tcp/Advanced/Tcp_024_Compression.cs create mode 100644 examples/Tcp/Advanced/Tcp_025_ServerInfo.cs diff --git a/examples/Program.cs b/examples/Program.cs index d309ffb07..8bf2497ab 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -442,6 +442,35 @@ private static async Task RunAllExamples(bool isInteractive) await TcpTimeouts.Run(); WaitForUser(isInteractive); + // Native Protocol: Advanced + Console.WriteLine("\n\n" + new string('=', 70)); + Console.WriteLine("NATIVE PROTOCOL: ADVANCED"); + Console.WriteLine(new string('=', 70) + "\n"); + + Console.WriteLine($"Running: {nameof(TcpSettingsAndQueryId)}"); + await TcpSettingsAndQueryId.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpProgressAndStatistics)}"); + await TcpProgressAndStatistics.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpCancellation)}"); + await TcpCancellation.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpErrorsAndRetries)}"); + await TcpErrorsAndRetries.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpCompression)}"); + await TcpCompression.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpServerInfo)}"); + await TcpServerInfo.Run(); + WaitForUser(isInteractive); + Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ALL EXAMPLES COMPLETED SUCCESSFULLY!"); Console.WriteLine(new string('=', 70)); diff --git a/examples/README.md b/examples/README.md index 1877cfc11..772493f49 100644 --- a/examples/README.md +++ b/examples/README.md @@ -133,6 +133,15 @@ These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/READ - [Tcp_018_Tls.cs](Tcp/Connection/Tcp_018_Tls.cs) - `UseTls`, `TlsServerName`, `TlsCaCertificatePath` (which replaces the host trust store rather than adding to it), `TlsAllowInvalidCertificates`, `ConfigureTls`, the default port moving to 9440, and the TLS mistakes the constructor refuses before anything connects - [Tcp_019_Timeouts.cs](Tcp/Connection/Tcp_019_Timeouts.cs) - `DialTimeout`, `ReadTimeout` as an idle deadline rather than a time limit, `PoolTimeout`, `StatementMaxLength` in the log line, `MaxSendBufferBytes`, and where a `CancellationToken` takes over +### Native Protocol: Advanced + +- [Tcp_020_SettingsAndQueryId.cs](Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs) - Client-level `CustomSettings` against per-query `ClickHouseTcpQueryOptions.Settings` and the precedence between them, a misspelled setting name being ignored rather than refused, `QueryId` in `system.query_log` and what reusing one does, and `async_insert` as a setting that changes what an insert means +- [Tcp_021_ProgressAndStatistics.cs](Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs) - `ClickHouseTcpQueryCallbacks`: `OnProgress` interleaved with the rows as the query runs, why every counter is an increment, `OnProfileInfo`'s once-per-query summary, `OnProfileEvents`' increments and gauges, and the callback contract — synchronous, on the draining thread, and never allowed to throw +- [Tcp_022_Cancellation.cs](Tcp/Advanced/Tcp_022_Cancellation.cs) - A `CancellationToken` through `QueryAsync`, `StreamAsync` and `ExecuteAsync`: what the caller catches, the cancellation the server logs, why the connection is closed rather than pooled, why abandoning a result is the same thing, and how it differs from `ReadTimeout` and `max_execution_time` +- [Tcp_023_ErrorsAndRetries.cs](Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs) - `ClickHouseTcpServerException` (`Code`, `RawCode`, `Name`, `ServerStackTrace`), `ClickHouseTcpTransportException`, `ClickHouseTcpProtocolException`, switching on `ClickHouseErrorCode`, what `IsTransient` does and does not promise, a retry that recovers, and why retrying an insert needs `insert_deduplication_token` and a table that can deduplicate +- [Tcp_024_Compression.cs](Tcp/Advanced/Tcp_024_Compression.cs) - `Compression=lz4|zstd|none` and `ClickHouseTcpClientOptions.Compressor`, measured in bytes on the wire: what LZ4 saves by default, why the client's codec does not choose what the server sends (`network_compression_method` does), what it does choose on an insert, and why loopback cannot measure the benefit +- [Tcp_025_ServerInfo.cs](Tcp/Advanced/Tcp_025_ServerInfo.cs) - `GetServerInfoAsync` and every field of `ClickHouseTcpServerInfo`, the build number `Version` does not carry, gating on `ProtocolRevision` (query parameters need 54459) with one gate that passes and one that does not, and gating on the server version with a printed skip + ## How to run ### Prerequisites diff --git a/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs b/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs new file mode 100644 index 000000000..5c7ff11de --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs @@ -0,0 +1,323 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// The two places a ClickHouse setting can be set — for +/// every operation the client runs, for one — and +/// , which is how a query is found again in +/// system.query_log or stopped with KILL QUERY. +/// +/// +/// Tcp_002_ConnectionString sets a client-level setting from a connection string key and reads it back; +/// this example is about what happens when both levels name the same setting, and about the settings that change +/// how an operation behaves rather than only what it reports. async_insert is the worked example. +/// +/// +public static class TcpSettingsAndQueryId +{ + private const string TableName = "example_tcp_async_insert"; + + public static async Task Run() + { + // Two client-level settings, from the set_ keys of the connection string. Tcp_002 covers the + // spelling; what matters here is that they are the client's defaults for every operation. + var builder = ExampleConfig.TcpBuilder(); + builder["set_max_threads"] = 2; + builder["set_max_block_size"] = 4096; + + await using var client = new ClickHouseTcpClient(builder.ToOptions()); + + await TwoLevels(client); + await AMisspelledNameIsIgnored(client); + await QueryIdInTheLog(client); + await ReusingAQueryId(client); + + try + { + await AsyncInsert(client); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped {TableName}."); + } + } + + private static async Task TwoLevels(ClickHouseTcpClient client) + { + Console.WriteLine("1. Client-level settings against per-query settings\n"); + + // getSetting reports the value in force for the query asking, which makes the precedence observable. + const string sql = "SELECT getSetting('max_threads')::String, getSetting('max_block_size')::String"; + + Console.WriteLine($" Options.CustomSettings {string.Join(", ", client.Options.CustomSettings.Select(s => $"{s.Key}={s.Value}"))}"); + Console.WriteLine($" no per-query options {await Pair(client, sql, null)}"); + + // Only max_threads is named twice, and only max_threads changes. + var oneKey = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["max_threads"] = "7" }, + }; + Console.WriteLine($" Settings max_threads=7 {await Pair(client, sql, oneKey)}"); + Console.WriteLine($" the next query {await Pair(client, sql, null)}"); + Console.WriteLine(); + Console.WriteLine(" A per-query value replaces the client-level one for that key alone: max_block_size"); + Console.WriteLine(" kept the client's 4096. And it applies to one operation — nothing is left behind on"); + Console.WriteLine(" the connection, because the settings travel in the query packet rather than as a SET."); + Console.WriteLine(); + Console.WriteLine(" To carry a setting across operations, put it on the client, or run SET inside a"); + Console.WriteLine(" session (Tcp_016), which pins one connection and so can hold session state."); + Console.WriteLine(); + Console.WriteLine(" Settings is IReadOnlyDictionary: every value is text, so a number is"); + Console.WriteLine(" spelled \"7\". HTTP's QueryOptions.CustomSettings takes object instead."); + } + + private static async Task Pair(ClickHouseTcpClient client, string sql, ClickHouseTcpQueryOptions? options) + { + await foreach (object[] row in client.QueryAsync(sql, options)) + { + return $"max_threads={row[0],-3} max_block_size={row[1]}"; + } + + return "(no row)"; + } + + private static async Task AMisspelledNameIsIgnored(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. A name the server does not know is ignored, not refused\n"); + + // The settings list is not validated against the server's setting names, so a typo costs nothing and + // does nothing. There is no client-side check either: the name is whatever string you passed. + object value = await client.ExecuteScalarAsync("SELECT 1", new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["maxx_threads"] = "7" }, + }); + + Console.WriteLine($" Settings[\"maxx_threads\"] = \"7\", then SELECT 1 -> {value}, no error at all."); + + // A value that cannot be parsed as the setting's type does fail, which is the only feedback there is. + // + // On its own throwaway client, deliberately. The server raises this error while it is still reading the + // settings list — before it has accepted the query — and closes the socket, which the pool does not + // notice, so the connection goes back into the pool dead and the *next* operation on this client fails + // with a ClickHouseTcpTransportException. Scoping it to a client that is disposed here disposes the dead + // connection with it. Tcp_007 does the same for the same reason. + await using (var throwaway = new ClickHouseTcpClient(client.Options)) + { + try + { + await throwaway.ExecuteScalarAsync("SELECT 1", new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["max_threads"] = "lots" }, + }); + Console.WriteLine(" max_threads = \"lots\" was accepted, which is not what this example expected"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" Settings[\"max_threads\"] = \"lots\" -> {ex.Code} ({ex.RawCode}): {FirstLine(ex.Message)}"); + } + } + + Console.WriteLine(); + Console.WriteLine(" So a wrong name is silent and a wrong value is loud. Read a setting back with"); + Console.WriteLine(" getSetting('name') when it matters that it arrived."); + Console.WriteLine(); + Console.WriteLine(" That second query ran on a client of its own, because a bad setting value is refused"); + Console.WriteLine(" before the query is accepted and the server closes the connection on its way out. An"); + Console.WriteLine(" ordinary query error — a syntax error, an unknown table — leaves the connection usable,"); + Console.WriteLine(" and Tcp_023 shows that; this one does not."); + } + + private static async Task QueryIdInTheLog(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. QueryId, and finding the query again\n"); + + // Unique per run: a query id is the key of a system.query_log row, and two runs of this example against + // one server must not collide. + string queryId = $"example-tcp-020-{Guid.NewGuid():N}"; + var options = new ClickHouseTcpQueryOptions { QueryId = queryId }; + + object rows = await client.ExecuteScalarAsync("SELECT count() FROM numbers(100000)", options); + Console.WriteLine($" Ran SELECT count() FROM numbers(100000) as query_id = {queryId}"); + Console.WriteLine($" result {rows}"); + + // The QueryFinish record is queued independently of the response reaching the client, so a flush issued + // straight after the query can miss it. Retry the flush and the read rather than sleeping. + string found = await ReadLog( + client, + "SELECT type::String || ' read_rows=' || toString(read_rows) || ' threads=' || Settings['max_threads'] " + + "FROM system.query_log WHERE query_id = {id:String} AND type = 'QueryFinish'", + queryId); + + Console.WriteLine($" system.query_log by query_id: {found}"); + Console.WriteLine(); + Console.WriteLine(" The Settings column holds the settings the query ran with, client-level ones"); + Console.WriteLine(" included, which is the other reason to set a query id: it is the only handle that"); + Console.WriteLine(" ties an application's own request to a server-side row. It is also what"); + Console.WriteLine(" KILL QUERY WHERE query_id = '...' takes."); + } + + private static async Task ReusingAQueryId(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. Reusing one\n"); + + string queryId = $"example-tcp-020-reuse-{Guid.NewGuid():N}"; + var options = new ClickHouseTcpQueryOptions { QueryId = queryId }; + + await client.ExecuteScalarAsync("SELECT 1", options); + await client.ExecuteScalarAsync("SELECT 2", options); + Console.WriteLine(" Two queries, one after the other, under the same id: both accepted. The id is not"); + Console.WriteLine(" unique — the log now holds two rows for it, and telling them apart means reading"); + Console.WriteLine(" event_time_microseconds."); + + // While one is still running, the server refuses the second. Its own client, so that the two queries are + // genuinely concurrent rather than queued behind one connection. + await using var second = new ClickHouseTcpClient(client.Options); + string busyId = $"example-tcp-020-busy-{Guid.NewGuid():N}"; + var busy = new ClickHouseTcpQueryOptions { QueryId = busyId }; + + // Started without Task.Run, so the query packet goes out on this thread rather than whenever the thread + // pool gets to it. That ordering is what decides which of the two the server refuses. + Task slow = client + .ExecuteScalarAsync("SELECT sleepEachRow(0.05) FROM numbers(6)", busy) + .AsTask(); + + // Waits until the server really is running it, rather than guessing with a delay. Until this returns, the + // id is not yet claimed and it is undecided which query would be the duplicate. + await WaitUntilRunning(second, busyId); + + try + { + await second.ExecuteScalarAsync("SELECT 1", busy); + Console.WriteLine(" A concurrent reuse was accepted, which is not what this example expected"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($"\n The same id while the first is still running -> {ex.Code} (RawCode {ex.RawCode})"); + Console.WriteLine($" {FirstLine(ex.Message)}"); + Console.WriteLine(" Code reads Unknown because ClickHouseErrorCode does not name 216; RawCode"); + Console.WriteLine(" always carries the server's number. Tcp_023 is about that pair."); + } + + await slow; + Console.WriteLine("\n Use a fresh id per attempt (a Guid, or your own request id) unless you want the"); + Console.WriteLine(" server to reject a duplicate submission for you — which, with a retry, is the one"); + Console.WriteLine(" case where reusing an id is the point rather than a mistake."); + } + + private static async Task AsyncInsert(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. async_insert: a setting that changes what an insert means\n"); + + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteAsync($"CREATE TABLE {TableName} (id UInt64, note String) ENGINE = MergeTree ORDER BY id"); + + object[][] first = [[1UL, "a"], [2UL, "b"]]; + object[][] second = [[3UL, "c"], [4UL, "d"]]; + + // Settings live on ClickHouseTcpInsertOptions too: it derives from ClickHouseTcpQueryOptions and adds + // MaxRowsPerBlock (Tcp_009). + var waits = new ClickHouseTcpInsertOptions + { + Settings = new Dictionary { ["async_insert"] = "1", ["wait_for_async_insert"] = "1" }, + }; + var doesNotWait = new ClickHouseTcpInsertOptions + { + Settings = new Dictionary { ["async_insert"] = "1", ["wait_for_async_insert"] = "0" }, + }; + + var clock = Stopwatch.StartNew(); + await client.InsertRowsAsync($"INSERT INTO {TableName} (id, note) VALUES", first, waits); + long waited = clock.ElapsedMilliseconds; + object afterWaiting = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + + Console.WriteLine($" async_insert=1, wait_for_async_insert=1: returned after {waited} ms, and the rows are"); + Console.WriteLine($" already queryable — count() = {afterWaiting}. The rows went into a server-side buffer"); + Console.WriteLine(" shared with other clients' inserts, and the call waited for that buffer to be written."); + + clock.Restart(); + await client.InsertRowsAsync($"INSERT INTO {TableName} (id, note) VALUES", second, doesNotWait); + long notWaited = clock.ElapsedMilliseconds; + object immediately = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + + // The only way to make the second batch's visibility deterministic. Without it, the count above is 2 or 4 + // depending on whether the buffer happened to flush, which is exactly the guarantee being given up. + await client.ExecuteAsync("SYSTEM FLUSH ASYNC INSERT QUEUE"); + object afterFlush = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + + Console.WriteLine($"\n async_insert=1, wait_for_async_insert=0: returned after {notWaited} ms."); + Console.WriteLine($" count() straight afterwards = {immediately}. That number is 2 on one run and 4 on the next:"); + Console.WriteLine(" the buffer flushes on its own schedule and the call no longer waits for it."); + Console.WriteLine($" After SYSTEM FLUSH ASYNC INSERT QUEUE: count() = {afterFlush}."); + Console.WriteLine(); + Console.WriteLine(" Two things the pair changes, neither of which is visible in the API:"); + Console.WriteLine(" - a returned InsertRowsAsync no longer means the rows are stored, so a failure"); + Console.WriteLine(" after the return is reported to nobody;"); + Console.WriteLine(" - read-after-write stops holding, so a test that inserts and counts fails."); + Console.WriteLine(); + Console.WriteLine(" It is worth it for many small inserts from many clients, which is what the server-side"); + Console.WriteLine(" buffer is for. For one large insert, MaxRowsPerBlock and a plain insert are better."); + } + + /// + /// Reads one scalar out of system.query_log, retrying the flush and the read. Pick an expression that + /// is never NULL for a row that exists, so that "no row yet" and "row with an empty value" cannot be confused. + /// + private static async Task ReadLog(ClickHouseTcpClient client, string sql, string queryId) + { + var options = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "id", queryId } }, + }; + + for (int attempt = 1; attempt <= 5; attempt++) + { + await client.ExecuteAsync("SYSTEM FLUSH LOGS"); + await foreach (object[] row in client.QueryAsync(sql, options)) + { + return $"{row[0]} (attempt {attempt})"; + } + + await Task.Delay(50); + } + + return "no row appeared in system.query_log after 5 attempts"; + } + + /// Waits until system.processes shows the query, so a race cannot decide the next assertion. + private static async Task WaitUntilRunning(ClickHouseTcpClient client, string queryId) + { + var options = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "id", queryId } }, + }; + + for (int attempt = 0; attempt < 300; attempt++) + { + object running = await client.ExecuteScalarAsync( + "SELECT count() FROM system.processes WHERE query_id = {id:String}", + options); + if (Convert.ToUInt64(running) > 0) + { + return; + } + + await Task.Delay(10); + } + } + + /// The server's message is one long line with its own detail appended; the first line is the fact. + private static string FirstLine(string message) + { + string text = message.Replace("DB::Exception: ", string.Empty); + int newline = text.IndexOf('\n'); + if (newline >= 0) + { + text = text[..newline]; + } + + return text.Length <= 110 ? text : text[..110] + "..."; + } +} diff --git a/examples/Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs b/examples/Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs new file mode 100644 index 000000000..fbb6bef1f --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs @@ -0,0 +1,247 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// : the metadata the server interleaves into a response — +/// while the query is still running, +/// once with the execution summary, and +/// with the server's own performance counters. +/// +/// +/// This is what the native protocol has that HTTP does not. HTTP reports the same numbers in a trailing header, +/// after the response; here they arrive as packets between the data blocks, so a long query can drive a progress +/// bar while it runs. +/// +/// +/// +/// The contract matters more than the numbers. A callback runs synchronously on the thread draining the +/// response, in packet order, so anything slow in one stalls the read. A callback that throws propagates out of +/// the operation and terminates the connection — this example does not demonstrate that, because there is nothing +/// to see: the result is simply gone. Keep them to counters and a log line, and never let one throw. +/// +/// +public static class TcpProgressAndStatistics +{ + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + await ProgressArrivesDuringTheQuery(client); + IncrementsNotTotals(); + await ProfileInfoOnce(client); + await ProfileEvents(client); + WhatElseIsThere(); + } + + private static async Task ProgressArrivesDuringTheQuery(ClickHouseTcpClient client) + { + Console.WriteLine("1. OnProgress arrives while the query runs\n"); + + // The record of what happened, in the order it happened. Appending from the callback is safe without a + // lock precisely because callbacks run on the thread draining the response — the same thread this loop + // body runs on. + var timeline = new List(); + var clock = Stopwatch.StartNew(); + ClickHouseTcpProgress total = default; + int packets = 0; + int rowsSoFar = 0; + int beforeTheLastRow = 0; + + var options = new ClickHouseTcpQueryOptions + { + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnProgress = progress => + { + packets++; + total += progress; + if (rowsSoFar < 8) + { + beforeTheLastRow++; + } + + timeline.Add($"progress +{progress.Rows} rows at {clock.ElapsedMilliseconds,4} ms"); + }, + }, + + // interactive_delay is how often the server reports progress, in microseconds. The default is 100 ms; + // 30 ms makes the interleaving obvious in an example short enough to run in CI. + Settings = new Dictionary + { + ["interactive_delay"] = "30000", + ["max_block_size"] = "1", + }, + }; + + int rows = 0; + await foreach (object[] row in client.QueryAsync( + "SELECT number, sleepEachRow(0.04) FROM numbers(8)", options)) + { + rows++; + rowsSoFar = rows; + timeline.Add($"row {rows} at {clock.ElapsedMilliseconds,4} ms"); + } + + Console.WriteLine(" 8 rows, each taking the server 40 ms, one row per block:\n"); + foreach (string line in timeline) + { + Console.WriteLine($" {line}"); + } + + Console.WriteLine(); + Console.WriteLine($" {packets} progress packets and {rows} rows, interleaved — {beforeTheLastRow} of the packets arrived before the"); + Console.WriteLine(" last row, which is the whole point. On HTTP every one of those numbers arrives after"); + Console.WriteLine($" the response. Summed: {total.Rows} rows, {total.Bytes} bytes, {total.ElapsedNs / 1_000_000} ms of server-side time."); + } + + private static void IncrementsNotTotals() + { + Console.WriteLine("\n2. Every counter is an increment\n"); + + // Two packets, added rather than replaced. Keeping the last one reports the most recent step, not the run. + var first = new ClickHouseTcpProgress(rows: 100, bytes: 800, totalRows: 1000, wroteRows: 0, wroteBytes: 0, elapsedNs: 5_000_000); + var next = new ClickHouseTcpProgress(rows: 250, bytes: 2000, totalRows: 500, wroteRows: 0, wroteBytes: 0, elapsedNs: 7_000_000); + + Console.WriteLine($" packet 1 Rows={first.Rows,4} Bytes={first.Bytes,5} TotalRows={first.TotalRows}"); + Console.WriteLine($" packet 2 Rows={next.Rows,4} Bytes={next.Bytes,5} TotalRows={next.TotalRows}"); + Console.WriteLine($" first + next Rows={(first + next).Rows,4} Bytes={(first + next).Bytes,5} TotalRows={(first + next).TotalRows}"); + Console.WriteLine(); + Console.WriteLine(" TotalRows is an increment too: it is the rise in the server's estimate of the rows"); + Console.WriteLine(" this query has to read, so a progress bar's denominator is the running sum of it and"); + Console.WriteLine(" can grow as the server learns more. Use operator + or ClickHouseTcpProgress.Add."); + Console.WriteLine(); + Console.WriteLine(" WroteRows and WroteBytes are the insert side of the same packet. On 26.6 an insert"); + Console.WriteLine(" through this client produces no progress packets at all, so they read zero — a large"); + Console.WriteLine(" insert has no progress to report yet."); + } + + private static async Task ProfileInfoOnce(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. OnProfileInfo, once, with totals rather than increments\n"); + + ClickHouseTcpProfileInfo info = default; + int calls = 0; + + var options = new ClickHouseTcpQueryOptions + { + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnProfileInfo = summary => + { + info = summary; + calls++; + }, + }, + }; + + // A LIMIT, so that AppliedLimit and RowsBeforeLimit have something to say. + int rows = 0; + await foreach (object[] row in client.QueryAsync( + "SELECT number FROM numbers(1000) ORDER BY number DESC LIMIT 5", options)) + { + rows++; + } + + Console.WriteLine($" SELECT number FROM numbers(1000) ORDER BY number DESC LIMIT 5 ({rows} rows read)\n"); + Console.WriteLine($" called {calls} time"); + Console.WriteLine($" Rows {info.Rows}"); + Console.WriteLine($" Blocks {info.Blocks}"); + Console.WriteLine($" Bytes {info.Bytes}"); + Console.WriteLine($" AppliedLimit {info.AppliedLimit}"); + Console.WriteLine($" RowsBeforeLimit {info.RowsBeforeLimit}"); + Console.WriteLine($" CalculatedRowsBeforeLimit {info.CalculatedRowsBeforeLimit}"); + Console.WriteLine(); + Console.WriteLine(" RowsBeforeLimit is what a paging UI wants for its 'of N' — but only when"); + Console.WriteLine(" CalculatedRowsBeforeLimit is true. The server does not always work it out, and the"); + Console.WriteLine(" field is then zero rather than absent, so the flag is the one to read first."); + Console.WriteLine(); + Console.WriteLine(" Bytes counts the result as the server measured it in memory, not the bytes that"); + Console.WriteLine(" crossed the socket. Tcp_024 measures those."); + } + + private static async Task ProfileEvents(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. OnProfileEvents: the server's own counters, as it goes\n"); + + // Two dictionaries, because the block carries two kinds of row. type 1 is an increment to add up; type 2 + // is a gauge reading that replaces the last one. + var increments = new Dictionary(StringComparer.Ordinal); + var gauges = new Dictionary(StringComparer.Ordinal); + var threadIds = new HashSet(); + int blocks = 0; + + var options = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["interactive_delay"] = "30000", ["max_block_size"] = "1" }, + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnProfileEvents = block => + { + blocks++; + + // The block is borrowed: valid until the callback returns. Names have to be copied out (they + // are already strings); spans must not outlive it. + IColumn name = block.Column("name"); + ReadOnlySpan value = block.Column("value").Values; + ReadOnlySpan type = block.Column("type").Values; + ReadOnlySpan thread = block.Column("thread_id").Values; + + for (int row = 0; row < block.RowCount; row++) + { + threadIds.Add(thread[row]); + if (type[row] == 1) + { + increments.TryGetValue(name[row], out long soFar); + increments[name[row]] = soFar + value[row]; + } + else + { + gauges[name[row]] = value[row]; + } + } + }, + }, + }; + + await foreach (object[] row in client.QueryAsync("SELECT number, sleepEachRow(0.03) FROM numbers(8)", options)) + { + } + + Console.WriteLine($" {blocks} blocks of counters arrived during the query, {increments.Count} distinct increments and"); + Console.WriteLine($" {gauges.Count} gauges. thread_id values seen: {string.Join(", ", threadIds.Order())} — 0 is the query-wide total.\n"); + + foreach (string counter in new[] { "SelectedRows", "SelectedBytes", "SleepFunctionMicroseconds", "NetworkSendBytes", "RealTimeMicroseconds" }) + { + string reading = increments.TryGetValue(counter, out long sum) ? sum.ToString("N0") : "(not reported)"; + Console.WriteLine($" increment {counter,-26} {reading,12}"); + } + + foreach (string gauge in gauges.Keys.Order()) + { + Console.WriteLine($" gauge {gauge,-26} {gauges[gauge],12:N0}"); + } + + Console.WriteLine(); + Console.WriteLine(" Every counter in system.events and system.metrics can appear here, so this is the"); + Console.WriteLine(" whole of what the server knows about its own work on this query. Reading `name`"); + Console.WriteLine(" allocates a string per row and the same counter arrives on every packet, so pick the"); + Console.WriteLine(" handful you care about rather than keeping them all."); + } + + private static void WhatElseIsThere() + { + Console.WriteLine("\n5. The rest of the record\n"); + Console.WriteLine(" OnLog the server's own log lines, when the query sets send_logs_level."); + Console.WriteLine(" priority is a Poco severity, so a lower number is more severe."); + Console.WriteLine(" OnTotals the WITH TOTALS row, in the query's own result shape."); + Console.WriteLine(" OnExtremes two rows, the minimum and the maximum, when the extremes setting is on."); + Console.WriteLine(); + Console.WriteLine(" All three hand over a borrowed Block on the same contract as StreamAsync: copy out"); + Console.WriteLine(" what must outlive the callback, and retain neither the block nor a span over it."); + Console.WriteLine(); + Console.WriteLine(" An unset callback costs nothing beyond the discarded result. The packets are decoded"); + Console.WriteLine(" either way, because skipping one would leave the connection misaligned."); + } +} diff --git a/examples/Tcp/Advanced/Tcp_022_Cancellation.cs b/examples/Tcp/Advanced/Tcp_022_Cancellation.cs new file mode 100644 index 000000000..7a2627dc1 --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_022_Cancellation.cs @@ -0,0 +1,303 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Driver.Examples; + +/// +/// Cancelling a native-protocol operation: what the caller sees, what the server is told, and what it costs the +/// connection pool. +/// +/// +/// Every method takes a , and it is the only bound on a whole operation — the three +/// deadlines in Tcp_019_Timeouts each cover one phase. Cancelling is not free, though: the client tells the +/// server the result is abandoned and then closes the connection, because a socket part-way through a response +/// nobody will read is of no use to the next caller. The client itself stays usable; its pool opens another. +/// +/// +public static class TcpCancellation +{ + public static async Task Run() + { + await CancellingMidResult(); + await WhatTheServerWasTold(); + int abandonedAfter = await ThePoolDiscardsIt(); + WhatThePoolLinesSay(abandonedAfter); + await ExecuteAndStream(); + TheOtherWaysAnOperationEnds(); + } + + private static async Task CancellingMidResult() + { + Console.WriteLine("1. Cancelling part-way through a result\n"); + + await using var client = ExampleConfig.CreateTcpClient(); + using var cancellation = new CancellationTokenSource(); + + int rows = 0; + var clock = Stopwatch.StartNew(); + try + { + // 40 rows at 50 ms each, one row per block, so the loop body really does run between rows. + await foreach (object[] row in client.QueryAsync( + "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 1", + cancellationToken: cancellation.Token)) + { + rows++; + if (rows == 3) + { + cancellation.Cancel(); + } + } + + Console.WriteLine(" The loop finished, which is not what this example expected"); + } + catch (OperationCanceledException ex) + { + Console.WriteLine($" Cancelled after {rows} of 40 rows, {clock.ElapsedMilliseconds} ms in."); + Console.WriteLine($" Caught {ex.GetType().Name}: {ex.Message}"); + Console.WriteLine(); + Console.WriteLine(" The runtime raises TaskCanceledException here, which derives from"); + Console.WriteLine(" OperationCanceledException — catch the base one. It is not a"); + Console.WriteLine(" ClickHouseTcpException: nothing went wrong between the client and the server,"); + Console.WriteLine(" the caller asked to stop. Tcp_023 covers the exceptions that are."); + } + + // The same client, straight afterwards. Cancelling costs a connection, not the client. + object still = await client.ExecuteScalarAsync("SELECT 'the client is still usable'"); + Console.WriteLine($"\n Next operation on the same client: {still}"); + } + + private static async Task WhatTheServerWasTold() + { + Console.WriteLine("\n2. What the server was told\n"); + + await using var client = ExampleConfig.CreateTcpClient(); + string queryId = $"example-tcp-022-{Guid.NewGuid():N}"; + using var cancellation = new CancellationTokenSource(); + + try + { + await foreach (object[] row in client.QueryAsync( + "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 1", + new ClickHouseTcpQueryOptions { QueryId = queryId }, + cancellation.Token)) + { + cancellation.Cancel(); + } + } + catch (OperationCanceledException) + { + } + + // The QueryFinish/ExceptionWhileProcessing record is queued independently of the response reaching the + // client, so the flush and the read are retried rather than delayed. + string logged = await ReadLog( + client, + "SELECT type::String || ' exception_code=' || toString(exception_code) || ' ' || splitByChar('(', exception)[1] " + + "FROM system.query_log WHERE query_id = {id:String} AND type != 'QueryStart'", + queryId); + + Console.WriteLine($" system.query_log for that query_id:\n {logged}"); + Console.WriteLine(); + Console.WriteLine(" 735 is QUERY_WAS_CANCELLED_BY_CLIENT. The client sent a Cancel packet before closing"); + Console.WriteLine(" the connection, so the server stopped the query rather than finishing it into a socket"); + Console.WriteLine(" nobody was reading. That is the difference between cancelling and hanging up: the"); + Console.WriteLine(" work stops, and the reason is in the log."); + } + + /// + /// Six operations on a one-connection pool, with the pool's own log lines: two ordinary ones, a cancelled one, + /// an abandoned one, and an ordinary one after each. Its own method so that the logger factory is disposed — + /// and its lines flushed to the console — before the interpretation prints. + /// + /// How many rows the abandoned enumeration read before breaking out. + private static async Task ThePoolDiscardsIt() + { + Console.WriteLine("\n3. The connection is closed, not pooled\n"); + Console.WriteLine(" MaxPoolSize = 1, and the pool's own log lines. Had a connection gone back into the"); + Console.WriteLine(" pool, the operation after it would be reusing it — the pool holds only one.\n"); + + using ILoggerFactory poolLog = LoggerFactory.Create(builder => builder + .AddFilter((category, _) => category == "ClickHouse.Driver.Tcp.Pool") + .AddSimpleConsole(console => console.SingleLine = true) + .SetMinimumLevel(LogLevel.Trace)); + + await using var client = new ClickHouseTcpClient(ExampleConfig.TcpBuilder().ToOptions() with + { + MaxPoolSize = 1, + LoggerFactory = poolLog, + }); + + // Two ordinary operations first, so that a reuse line is in the output to compare against. + _ = await client.ExecuteScalarAsync("SELECT 1"); + _ = await client.ExecuteScalarAsync("SELECT 2"); + + using var cancellation = new CancellationTokenSource(); + try + { + await foreach (object[] row in client.QueryAsync( + "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 1", + cancellationToken: cancellation.Token)) + { + cancellation.Cancel(); + } + } + catch (OperationCanceledException) + { + } + + _ = await client.ExecuteScalarAsync("SELECT 3"); + + // No token this time: the loop simply stops reading, which is abandonment rather than cancellation. + int rows = 0; + await foreach (object[] row in client.QueryAsync( + "SELECT number FROM numbers(10000000) SETTINGS max_block_size = 100")) + { + if (++rows == 5) + { + break; + } + } + + _ = await client.ExecuteScalarAsync("SELECT 4"); + return rows; + } + + private static void WhatThePoolLinesSay(int abandonedAfter) + { + Console.WriteLine("\n Read that as four pairs. SELECT 1 opened a connection and SELECT 2 reused it — 'its 2"); + Console.WriteLine(" operation'. The cancelled query got 'its 3 operation' and then ended it: 'Closing a"); + Console.WriteLine(" returned connection rather than pooling it', so SELECT 3 had to open another."); + Console.WriteLine(); + Console.WriteLine($" Then the same thing with no token at all: a loop that read {abandonedAfter} rows of ten million"); + Console.WriteLine(" and broke out. The same two lines follow it, so abandoning a result is treated exactly"); + Console.WriteLine(" as cancelling one — the connection is closed, and SELECT 4 opened a fresh one."); + Console.WriteLine(); + Console.WriteLine(" So a cancellation costs a dial, and a loop that cancels every query keeps the pool"); + Console.WriteLine(" empty. It does not cost the client: every operation after one of these succeeded."); + Console.WriteLine(); + Console.WriteLine(" `break` inside an `await foreach` disposes the enumerator, which is what sends the"); + Console.WriteLine(" Cancel packet and returns the connection. So does `return`, and so does an exception"); + Console.WriteLine(" thrown from the loop body."); + Console.WriteLine(); + Console.WriteLine(" The one shape that does not is a hand-rolled enumerator that is never disposed:"); + Console.WriteLine(" var e = client.QueryAsync(sql).GetAsyncEnumerator(); // no await using"); + Console.WriteLine(" Its connection is neither returned nor closed, and nothing reclaims it — there is no"); + Console.WriteLine(" finalizer to free the pool slot, so it is gone for as long as the client lives. Use"); + Console.WriteLine(" `await foreach`, or `await using` on the enumerator."); + } + + private static async Task ExecuteAndStream() + { + Console.WriteLine("\n4. The same token on ExecuteAsync and StreamAsync\n"); + + await using var client = ExampleConfig.CreateTcpClient(); + + // ExecuteAsync drains the whole response before returning, so there is no loop to break out of and the + // token is the only way to stop waiting. + using (var deadline = new CancellationTokenSource(150)) + { + var clock = Stopwatch.StartNew(); + try + { + await client.ExecuteAsync("SELECT sleepEachRow(0.2) FROM numbers(5)", cancellationToken: deadline.Token); + Console.WriteLine(" ExecuteAsync returned, which is not what this example expected"); + } + catch (OperationCanceledException ex) + { + Console.WriteLine($" ExecuteAsync, token cancelled after 150 ms: {ex.GetType().Name} at {clock.ElapsedMilliseconds} ms"); + } + } + + // StreamAsync is the same contract one level down: the block being iterated is released, the enumerator + // is disposed by the loop, and the connection is closed. + using (var cancellation = new CancellationTokenSource()) + { + int blocks = 0; + try + { + await foreach (Block block in client.StreamAsync( + "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 4", + cancellationToken: cancellation.Token)) + { + blocks++; + cancellation.Cancel(); + } + } + catch (OperationCanceledException ex) + { + Console.WriteLine($" StreamAsync, cancelled after {blocks} block: {ex.GetType().Name}"); + } + } + + Console.WriteLine($" And afterwards: {await client.ExecuteScalarAsync("SELECT 'still usable'")}"); + Console.WriteLine(); + Console.WriteLine(" A token already cancelled when the call is made throws before the pool is touched, so"); + Console.WriteLine(" nothing is dialled and nothing is closed — the pool's log stays silent, and the next"); + Console.WriteLine(" operation reuses whatever was idle:"); + + using (var alreadyDone = new CancellationTokenSource()) + { + await alreadyDone.CancelAsync(); + try + { + _ = await client.ExecuteScalarAsync("SELECT 1", cancellationToken: alreadyDone.Token); + Console.WriteLine(" it ran anyway, which is not what this example expected"); + } + catch (OperationCanceledException ex) + { + Console.WriteLine($" {ex.GetType().Name} straight away"); + } + } + } + + private static void TheOtherWaysAnOperationEnds() + { + Console.WriteLine("\n5. Three ways to stop a query, and which one to reach for\n"); + Console.WriteLine(" CancellationToken The caller changed its mind: a request was abandoned, a"); + Console.WriteLine(" timeout of your own elapsed, the process is shutting down."); + Console.WriteLine(" Throws OperationCanceledException, costs the connection,"); + Console.WriteLine(" and the server is told (section 2)."); + Console.WriteLine(); + Console.WriteLine(" ReadTimeout The server went quiet. An idle deadline, not a time limit —"); + Console.WriteLine(" Tcp_019 measures it. Throws TimeoutException and also costs"); + Console.WriteLine(" the connection, because a socket that stopped answering"); + Console.WriteLine(" mid-response cannot be reused either."); + Console.WriteLine(); + Console.WriteLine(" max_execution_time The server gives up, as a per-query setting (Tcp_020). The"); + Console.WriteLine(" query stops server-side, the client gets an ordinary"); + Console.WriteLine(" ClickHouseTcpServerException with code 159, and the"); + Console.WriteLine(" connection survives, because the response completed — with"); + Console.WriteLine(" an error rather than rows. Tcp_023 covers it."); + Console.WriteLine(); + Console.WriteLine(" So the cheapest of the three is the server-side one: it is the only one that does not"); + Console.WriteLine(" end a connection. Prefer max_execution_time for 'this query must not run longer than N"); + Console.WriteLine(" seconds', and keep the token for 'this caller no longer wants the answer'."); + } + + /// + /// Reads one row out of system.query_log, retrying the flush and the read. A query's record is queued + /// independently of its response, so one flush straight after the query can miss it. + /// + private static async Task ReadLog(ClickHouseTcpClient client, string sql, string queryId) + { + var options = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "id", queryId } }, + }; + + for (int attempt = 1; attempt <= 5; attempt++) + { + await client.ExecuteAsync("SYSTEM FLUSH LOGS"); + await foreach (object[] row in client.QueryAsync(sql, options)) + { + return (string)row[0]; + } + + await Task.Delay(50); + } + + return "no row appeared in system.query_log after 5 attempts"; + } +} diff --git a/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs b/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs new file mode 100644 index 000000000..bef8c58fd --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs @@ -0,0 +1,396 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// The native client's exception hierarchy — , +/// and under +/// — how to branch on , and which failures +/// are worth retrying. +/// +/// +/// Retrying is where the two halves meet. A read is idempotent, so a retry costs a round trip and nothing else. An +/// insert is not: the same batch sent twice lands twice, unless the target table can deduplicate it and the insert +/// carries a token that says which insert it is. This example measures both. +/// +/// +public static class TcpErrorsAndRetries +{ + private const string PlainTable = "example_tcp_retry_plain"; + private const string DedupTable = "example_tcp_retry_dedup"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + TheHierarchy(); + await ServerErrors(client); + await NotServerErrors(); + await Transient(client); + await RetryingARead(client); + + try + { + await RetryingAnInsert(client); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {PlainTable}"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {DedupTable}"); + Console.WriteLine($"\nDropped {PlainTable} and {DedupTable}."); + } + } + + private static void TheHierarchy() + { + Console.WriteLine("1. Three exception types, one base, and what is deliberately not in it\n"); + Console.WriteLine(" ClickHouseTcpException : DbException catch this for 'anything between the"); + Console.WriteLine(" client and the server went wrong'"); + Console.WriteLine(" ClickHouseTcpServerException the server reported an error for a query,"); + Console.WriteLine(" a handshake or a ping"); + Console.WriteLine(" ClickHouseTcpTransportException the socket failed: refused, dropped, TLS"); + Console.WriteLine(" ClickHouseTcpProtocolException the bytes did not match the protocol"); + Console.WriteLine(); + Console.WriteLine(" The hierarchy is closed — the constructors are not visible outside the assembly — so a"); + Console.WriteLine(" caught ClickHouseTcpException is always one of the three."); + Console.WriteLine(); + Console.WriteLine(" Mistakes in the calling code keep the usual framework types, on purpose:"); + Console.WriteLine(" ArgumentException a bad option or a null argument (Tcp_019 has nine)"); + Console.WriteLine(" InvalidOperationException a misused object — a session running two operations"); + Console.WriteLine(" ObjectDisposedException use after disposal"); + Console.WriteLine(" OperationCanceledException the caller cancelled (Tcp_022)"); + Console.WriteLine(" TimeoutException a deadline elapsed: PoolTimeout, DialTimeout,"); + Console.WriteLine(" ReadTimeout (Tcp_019)"); + Console.WriteLine(); + Console.WriteLine(" So `catch (ClickHouseTcpException)` never swallows a bug in your own code, and never"); + Console.WriteLine(" swallows a cancellation."); + } + + private static async Task ServerErrors(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. ClickHouseTcpServerException: Code, RawCode, Name, ServerStackTrace\n"); + + (string What, string Sql)[] cases = + [ + ("a syntax error", "SELECT FROM WHERE"), + ("an unknown table", "SELECT * FROM does_not_exist_example_tcp_023"), + ("an unknown function", "SELECT no_such_function(1)"), + ("an unparseable value", "SELECT toUInt8('abc')"), + ("a division by zero", "SELECT intDiv(1, 0)"), + ]; + + Console.WriteLine($" {"",-22} {"Code",-28} {"RawCode",7} {"transient",9} message"); + foreach ((string what, string sql) in cases) + { + try + { + _ = await client.ExecuteScalarAsync(sql); + Console.WriteLine($" {what,-22} succeeded, which is not what this example expected"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" {what,-22} {ex.Code,-28} {ex.RawCode,7} {ex.IsTransient,9} {FirstLine(ex.Message, 40)}"); + } + } + + Console.WriteLine(); + Console.WriteLine(" RawCode is always the number the server sent. Code is that number as a named"); + Console.WriteLine(" constant, or Unknown when this client does not name it — the enum carries the codes"); + Console.WriteLine(" worth branching on, not all ~660 of them. Division by zero (153) is one it does not"); + Console.WriteLine(" name, so branch on RawCode for anything outside the list."); + Console.WriteLine(); + Console.WriteLine(" Every one of those left the connection usable: the server reported an error as part of"); + Console.WriteLine($" a complete response, so the pool keeps it. Proof — {await client.ExecuteScalarAsync("SELECT 'still here'")}."); + + // The two fields an operator asks for, on one error. + try + { + _ = await client.ExecuteScalarAsync("SELECT * FROM does_not_exist_example_tcp_023"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine("\n The whole of one error:"); + Console.WriteLine($" Code {ex.Code}"); + Console.WriteLine($" RawCode {ex.RawCode}"); + Console.WriteLine($" Name {ex.Name}"); + Console.WriteLine($" IsTransient {ex.IsTransient}"); + Console.WriteLine($" ErrorCode {ex.ErrorCode} (DbException's, the same number)"); + Console.WriteLine($" ServerStackTrace {ex.ServerStackTrace?.Length ?? 0} characters of the server's own C++ frames"); + Console.WriteLine($" Message {FirstLine(ex.Message, 90)}"); + Console.WriteLine(); + Console.WriteLine(" Message repeats Name, because the server puts its exception class in both."); + Console.WriteLine(" ServerStackTrace is for a bug report, not for a log line."); + + // Branching. A switch on Code is the readable form; the default arm has to exist, because a code the + // enum does not name arrives as Unknown. + string advice = ex.Code switch + { + ClickHouseErrorCode.UnknownTable or ClickHouseErrorCode.UnknownDatabase => "check the name and the database the client is pointed at", + ClickHouseErrorCode.SyntaxError => "the query text is wrong; do not retry it", + ClickHouseErrorCode.AccessDenied or ClickHouseErrorCode.AuthenticationFailed => "a grant or a credential problem", + ClickHouseErrorCode.TooManyParts or ClickHouseErrorCode.ServerOverloaded => "back off and try again", + _ => $"unrecognized; RawCode {ex.RawCode}", + }; + Console.WriteLine($"\n switch (ex.Code) -> {advice}"); + } + } + + private static async Task NotServerErrors() + { + Console.WriteLine("\n3. The other two: transport and protocol\n"); + + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); + + // Nothing listening: a socket failure, and a fresh connection may well work, so IsTransient is true. + try + { + await using var refused = new ClickHouseTcpClient(options with { Port = 1, DialTimeout = TimeSpan.FromSeconds(2) }); + await refused.PingAsync(); + Console.WriteLine(" Port 1 answered, which is not what this example expected"); + } + catch (ClickHouseTcpException ex) + { + Console.WriteLine($" nothing listening on the port {ex.GetType().Name}"); + Console.WriteLine($" IsTransient={ex.IsTransient}, InnerException={ex.InnerException?.GetType().Name}"); + Console.WriteLine(" Match the inner exception when the distinction"); + Console.WriteLine(" matters: SocketException, IOException,"); + Console.WriteLine(" EndOfStreamException, AuthenticationException."); + } + + // The HTTP port: a peer that answers, but not in this protocol. Not transient — retrying a + // misconfiguration just fails again. + try + { + await using var wrongPort = new ClickHouseTcpClient(options with { Port = ExampleConfig.HttpPort }); + await wrongPort.PingAsync(); + Console.WriteLine(" The HTTP port spoke the native protocol, which is not what this example expected"); + } + catch (ClickHouseTcpException ex) + { + Console.WriteLine($"\n the HTTP port ({ExampleConfig.HttpPort}) {ex.GetType().Name}"); + Console.WriteLine($" IsTransient={ex.IsTransient}"); + Console.WriteLine($" {ex.Message}"); + Console.WriteLine(" 72 is 'H', the first byte of an HTTP response."); + } + + Console.WriteLine(); + Console.WriteLine(" Both terminate the connection and it is never reused, which is why neither needs a"); + Console.WriteLine(" 'is the client still usable' check: the pool simply dials again."); + } + + private static async Task Transient(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. IsTransient, and what it does and does not promise\n"); + + // A real timeout: a scan far too large for the deadline. Transient by code, and completely deterministic. + await Report( + client, + "max_execution_time = 0.2 over a huge scan", + "SELECT count() FROM numbers(50000000000)", + new Dictionary { ["max_execution_time"] = "0.2" }); + + // Looks temporary, is not: the same query at the same size needs the same memory every time. + await Report( + client, + "max_memory_usage = 1 MB", + "SELECT groupArray(number) FROM numbers(10000000)", + new Dictionary { ["max_memory_usage"] = "1000000" }); + + Console.WriteLine(); + Console.WriteLine(" IsTransient reads the code, and it means 'retrying could plausibly succeed' — not"); + Console.WriteLine(" 'will'. TimeoutExceeded is transient because the server may be less busy next time,"); + Console.WriteLine(" and yet the query above will time out on every attempt, because the cause is its own"); + Console.WriteLine(" size. MemoryLimitExceeded is the opposite reading: it looks temporary and is judged"); + Console.WriteLine(" not transient, because the same query at the same size repeats it."); + Console.WriteLine(); + Console.WriteLine(" So cap the attempts, and prefer a failure whose cause is outside your query:"); + Console.WriteLine(" transient TimeoutExceeded(159) TooManySimultaneousQueries(202) NoFreeConnection(203)"); + Console.WriteLine(" SocketTimeout(209) NetworkError(210) TooManyParts(252)"); + Console.WriteLine(" AllConnectionTriesFailed(279) ServerOverloaded(745) KeeperException(999)"); + Console.WriteLine(" and every ClickHouseTcpTransportException"); + Console.WriteLine(" not syntax, unknown table or column, type mismatch, access denied,"); + Console.WriteLine(" memory limit, and every ClickHouseTcpProtocolException"); + } + + private static async Task Report( + ClickHouseTcpClient client, + string label, + string sql, + Dictionary settings) + { + try + { + _ = await client.ExecuteScalarAsync(sql, new ClickHouseTcpQueryOptions { Settings = settings }); + Console.WriteLine($" {label,-42} succeeded, which is not what this example expected"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine($" {label,-42} {ex.Code} ({ex.RawCode}), IsTransient={ex.IsTransient}"); + } + } + + private static async Task RetryingARead(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. Retrying a read, which is free\n"); + + // A real transient failure with nothing injected. max_concurrent_queries_for_user is checked when a query + // starts, and only against the query that declares it: the slow query below declares nothing, so it holds + // a slot and can never itself be refused, and only the retry loop can lose. That one-sidedness is what + // makes the outcome determined rather than raced. + // Started without Task.Run, so the query packet goes out on this thread before the poll below rather than + // whenever the thread pool gets to it. AsTask only wraps the operation already in flight. + string holderId = $"example-tcp-023-holder-{Guid.NewGuid():N}"; + Task holder = client + .ExecuteScalarAsync("SELECT sleepEachRow(0.08) FROM numbers(6)", new ClickHouseTcpQueryOptions { QueryId = holderId }) + .AsTask(); + + // A separate client, so that the two queries are really concurrent rather than queued behind one + // connection. + await using var second = new ClickHouseTcpClient(client.Options); + + // Waits until the server is really running it, so the first attempt below is refused rather than usually + // refused. Polling rather than a delay: a delay is the same race with a longer window. + await WaitUntilRunning(second, holderId); + + Console.WriteLine(" A slow query is running. A second one asks with max_concurrent_queries_for_user = 1,"); + Console.WriteLine(" so the server refuses it until the first has finished.\n"); + + var oneAtATime = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["max_concurrent_queries_for_user"] = "1" }, + }; + + for (int attempt = 1; attempt <= 8; attempt++) + { + try + { + object counted = await second.ExecuteScalarAsync("SELECT count() FROM numbers(1000)", oneAtATime); + Console.WriteLine($" attempt {attempt}: {counted}"); + break; + } + catch (ClickHouseTcpException ex) when (ex.IsTransient && attempt < 8) + { + string reason = ex is ClickHouseTcpServerException server ? $"{server.Code} ({server.RawCode})" : ex.GetType().Name; + Console.WriteLine($" attempt {attempt}: {reason} — transient, so try again"); + await Task.Delay(100 * attempt); + } + } + + await holder; + + Console.WriteLine(); + Console.WriteLine(" `catch (ClickHouseTcpException ex) when (ex.IsTransient)` is the whole filter, and the"); + Console.WriteLine(" attempt cap is what keeps a deterministic failure from becoming a loop. The read is"); + Console.WriteLine(" idempotent, so nothing had to be checked before trying again — which is the only"); + Console.WriteLine(" reason this retry is safe to write in three lines."); + Console.WriteLine(); + Console.WriteLine(" The limit travelled in the query packet, so it bounded those attempts and nothing"); + Console.WriteLine(" else — not the query it was waiting for, and not whatever runs next. Tcp_020 is about"); + Console.WriteLine(" that."); + } + + /// + /// Waits until system.processes shows the query. Deliberately sets nothing of its own, so this poll can + /// never be the query a concurrency limit refuses. + /// + private static async Task WaitUntilRunning(ClickHouseTcpClient client, string queryId) + { + var options = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "id", queryId } }, + }; + + for (int attempt = 0; attempt < 200; attempt++) + { + object running = await client.ExecuteScalarAsync( + "SELECT count() FROM system.processes WHERE query_id = {id:String}", + options); + if (Convert.ToUInt64(running) > 0) + { + return; + } + + await Task.Delay(10); + } + } + + private static async Task RetryingAnInsert(ClickHouseTcpClient client) + { + Console.WriteLine("\n6. Retrying an insert, which is not\n"); + + await client.ExecuteAsync($"DROP TABLE IF EXISTS {PlainTable}"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {DedupTable}"); + await client.ExecuteAsync($"CREATE TABLE {PlainTable} (id UInt64) ENGINE = MergeTree ORDER BY id"); + + // A non-replicated MergeTree deduplicates nothing unless it is told to keep a window of insert hashes. + await client.ExecuteAsync( + $"CREATE TABLE {DedupTable} (id UInt64) ENGINE = MergeTree ORDER BY id " + + "SETTINGS non_replicated_deduplication_window = 100"); + + object[][] batch = [[1UL], [2UL], [3UL]]; + + // The failure mode: a retry after a transport error that in fact delivered the rows. + await client.InsertRowsAsync($"INSERT INTO {PlainTable} (id) VALUES", batch); + await client.InsertRowsAsync($"INSERT INTO {PlainTable} (id) VALUES", batch); + Console.WriteLine($" plain MergeTree, the same 3 rows sent twice count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {PlainTable}")}"); + + var token = new ClickHouseTcpInsertOptions + { + Settings = new Dictionary { ["insert_deduplication_token"] = "example-tcp-023-batch-1" }, + }; + + await client.InsertRowsAsync($"INSERT INTO {PlainTable} (id) VALUES", batch, token); + await client.InsertRowsAsync($"INSERT INTO {PlainTable} (id) VALUES", batch, token); + Console.WriteLine($" ... twice more with one insert_deduplication_token count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {PlainTable}")}"); + Console.WriteLine(" The token did nothing: this table keeps no window of insert hashes to compare it"); + Console.WriteLine(" against, so there is nothing for it to match."); + + await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", batch, token); + await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", batch, token); + Console.WriteLine($"\n non_replicated_deduplication_window = 100, same token count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {DedupTable}")}"); + + // The token identifies the insert, not the bytes: a second attempt that produced different rows is still + // dropped, which is what makes a retry safe even when the data was rebuilt. + object[][] different = [[4UL], [5UL], [6UL]]; + await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", different, token); + Console.WriteLine($" ... and again with different rows, same token count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {DedupTable}")}"); + + // No token at all on the same table: the block's own hash is still compared, so a byte-identical retry is + // dropped too. Worth knowing, and not worth relying on. + await client.ExecuteAsync($"TRUNCATE TABLE {DedupTable}"); + await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", batch); + await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", batch); + Console.WriteLine($" ... and the same batch twice with no token at all count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {DedupTable}")}"); + + Console.WriteLine(); + Console.WriteLine(" So a safe insert retry needs two things, and one of them is not in the client:"); + Console.WriteLine(" - the table must deduplicate — a Replicated engine, or"); + Console.WriteLine(" non_replicated_deduplication_window on a plain MergeTree;"); + Console.WriteLine(" - the insert must carry insert_deduplication_token, one value per logical batch,"); + Console.WriteLine(" reused by every retry of it."); + Console.WriteLine(); + Console.WriteLine(" The last line is why the token matters even though a window alone dropped the"); + Console.WriteLine(" duplicate: that was the block's own hash matching, and a retry that rebuilt the batch"); + Console.WriteLine(" in a different order, or split it differently, hashes differently and lands twice."); + Console.WriteLine(); + Console.WriteLine(" And an insert that fails with a ClickHouseTcpTransportException may or may not have"); + Console.WriteLine(" been applied — the client cannot tell which side of the socket the failure was. That"); + Console.WriteLine(" is the case the token exists for."); + } + + /// + /// The server's message is one long line with its own multi-line detail appended, and it starts with the class + /// name that already carries. + /// + private static string FirstLine(string message, int width) + { + string text = message.Replace("DB::Exception: ", string.Empty); + int newline = text.IndexOf('\n'); + if (newline >= 0) + { + text = text[..newline]; + } + + return text.Length <= width ? text : text[..width] + "..."; + } +} diff --git a/examples/Tcp/Advanced/Tcp_024_Compression.cs b/examples/Tcp/Advanced/Tcp_024_Compression.cs new file mode 100644 index 000000000..49fe9701e --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_024_Compression.cs @@ -0,0 +1,350 @@ +using System.Net; +using System.Net.Sockets; +using ClickHouse.Driver.Compression; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// Block compression on the native protocol: the Compression=lz4|zstd|none connection-string key, +/// , and what each setting is worth in bytes on the wire. +/// +/// +/// LZ4 is the default, so blocks are compressed in both directions unless you say otherwise. The HTTP transport's +/// Compression key is a boolean; this one names a codec. +/// +/// +/// +/// How this example measures. It forwards the connection through a local socket that counts the bytes each +/// way, which is the only thing a client can observe honestly. Wall-clock time is not measured: everything here +/// runs over loopback, where there is no bandwidth to save, so a timing comparison would report the CPU cost of +/// compressing and none of the benefit. Wire size is deterministic and is the thing compression actually buys. +/// +/// +public static class TcpCompression +{ + private const string TableName = "example_tcp_compression"; + + // Big enough that the codec dominates the fixed cost of a handshake, small enough to stay quick. + private const int SelectRows = 200_000; + private const int InsertRows = 100_000; + + // Two columns, one of them compressible text, so the ratio is representative rather than a best case. + private static readonly string Query = $"SELECT number, toString(number % 97) AS text FROM numbers({SelectRows})"; + + public static async Task Run() + { + WhatIsInForce(); + + await using var client = ExampleConfig.CreateTcpClient(); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteAsync($"CREATE TABLE {TableName} (id UInt64, text String) ENGINE = MergeTree ORDER BY id"); + + try + { + await ReadingDirection(); + await WhoChoosesTheServersCodec(); + await WritingDirection(); + WhatItBuys(); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped {TableName}."); + } + } + + private static void WhatIsInForce() + { + Console.WriteLine("1. Which codec a client is using\n"); + + // The assembled connection string carries no Compression key, so this is the default. + ClickHouseTcpClientOptions fromDefaults = ClickHouseTcpClientOptions.FromConnectionString(ExampleConfig.TcpConnectionString); + + Console.WriteLine($" no Compression key Compressor = {Describe(fromDefaults.Compressor)}"); + foreach (string codec in new[] { "lz4", "zstd", "none" }) + { + ClickHouseTcpClientOptions options = Options(codec); + Console.WriteLine($" Compression={codec,-16}Compressor = {Describe(options.Compressor)}"); + } + + Console.WriteLine(); + Console.WriteLine(" 'none' leaves Compressor null, and null means the query asks the server for no"); + Console.WriteLine(" compression at all — which is not the same as a frame whose method byte says NONE."); + Console.WriteLine(); + Console.WriteLine(" Setting it in code instead of in a connection string takes the codec object:"); + Console.WriteLine(" options with { Compressor = ZstdCompressor.Default }"); + Console.WriteLine(" options with { Compressor = new ZstdCompressor(level: 9) }"); + Console.WriteLine(" options with { Compressor = null } // off"); + Console.WriteLine(); + + // A codec that only implements the HTTP body path cannot frame a block, and the client says so at + // construction rather than mid-query. + try + { + using var refused = new ClickHouseTcpClient(Options("lz4") with { Compressor = GZipCompressor.Default }); + Console.WriteLine(" A GZip codec was accepted, which is not what this example expected"); + } + catch (ArgumentException ex) + { + Console.WriteLine($" Not every IClickHouseCompressor will do: {ex.Message.Split(" (Parameter")[0]}"); + } + } + + private static async Task ReadingDirection() + { + Console.WriteLine($"\n2. Server to client: {SelectRows:N0} rows, measured on the wire\n"); + Console.WriteLine($" {Query}\n"); + Console.WriteLine($" {"client codec",-14}{"bytes from the server",22} {"vs none",8}"); + + long baseline = 0; + foreach (string codec in new[] { "none", "lz4", "zstd" }) + { + (long fromServer, _, long rows) = await Measure(Options(codec), null); + if (codec == "none") + { + baseline = fromServer; + } + + Console.WriteLine($" {codec,-14}{fromServer,22:N0} {(double)baseline / fromServer,7:0.00}x ({rows:N0} rows)"); + } + + Console.WriteLine(); + Console.WriteLine(" LZ4 cut it to about a third. ZSTD produced the same count as LZ4, to within the few"); + Console.WriteLine(" bytes of progress packets that vary between runs — which is the thing to understand"); + Console.WriteLine(" about this key."); + } + + private static async Task WhoChoosesTheServersCodec() + { + Console.WriteLine("\n3. The client's codec does not choose what the server sends\n"); + Console.WriteLine(" The query packet carries one flag: compressed, or not. Which codec the server then"); + Console.WriteLine(" frames its blocks with is the server's own choice, from its network_compression_method"); + Console.WriteLine(" setting — LZ4 by default. So asking for ZSTD on the client changed nothing above."); + Console.WriteLine(" Set it as a per-query setting to change it:\n"); + + Console.WriteLine($" {"network_compression_method",-30}{"bytes from the server",22}"); + foreach (string method in new[] { "LZ4", "ZSTD" }) + { + var options = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["network_compression_method"] = method }, + }; + (long fromServer, _, _) = await Measure(Options("lz4"), options); + Console.WriteLine($" {method,-30}{fromServer,22:N0}"); + } + + Console.WriteLine(); + Console.WriteLine(" The client decodes whatever arrives, whichever codec it asked for, so this is safe to"); + Console.WriteLine(" set per query. What the client's own Compressor decides is the direction the client"); + Console.WriteLine(" writes — which is an insert."); + } + + private static async Task WritingDirection() + { + Console.WriteLine($"\n4. Client to server: an insert of {InsertRows:N0} rows\n"); + Console.WriteLine($" {"client codec",-14}{"bytes to the server",22} {"vs none",8}"); + + long baseline = 0; + foreach (string codec in new[] { "none", "lz4", "zstd" }) + { + long toServer = await MeasureInsert(Options(codec)); + if (codec == "none") + { + baseline = toServer; + } + + Console.WriteLine($" {codec,-14}{toServer,22:N0} {(double)baseline / toServer,7:0.00}x"); + } + + Console.WriteLine(); + Console.WriteLine(" Here the key does what its name suggests, because these are the client's own frames."); + Console.WriteLine(" ZSTD is the smaller of the two and costs more CPU on the client to produce; LZ4 is the"); + Console.WriteLine(" cheaper one and is what the default gives you."); + } + + private static void WhatItBuys() + { + Console.WriteLine("\n5. What the numbers above do and do not tell you\n"); + Console.WriteLine(" They are bytes, and bytes are the honest measurement: run this example twice and the"); + Console.WriteLine(" counts differ only by the handful of progress packets the server chose to send."); + Console.WriteLine(); + Console.WriteLine(" There is deliberately no timing here. Every connection in this example is loopback,"); + Console.WriteLine(" where a saved byte saves nothing, so a wall-clock ranking of none/lz4/zstd would"); + Console.WriteLine(" measure the cost of compressing and none of the benefit — and would then read as an"); + Console.WriteLine(" argument for turning compression off. Where it pays is where the bytes have somewhere"); + Console.WriteLine(" to go: a link between availability zones, a metered egress bill, a saturated uplink,"); + Console.WriteLine(" or a server whose network is busier than its CPU."); + Console.WriteLine(); + Console.WriteLine(" Reasonable defaults, then:"); + Console.WriteLine(" lz4 leave it alone. Cheapest in CPU, lightest on the server, ~3x here."); + Console.WriteLine(" zstd a slow or metered link, and inserts large enough for the ratio to matter."); + Console.WriteLine(" none a client and a server on the same host, where the CPU is the scarce thing."); + Console.WriteLine(); + Console.WriteLine(" Compression is per query, not per connection, so nothing has to be restarted to change"); + Console.WriteLine(" it — but the codec lives on the client, so it takes a second client to run two."); + } + + /// Options for the configured server with one Compression value. + private static ClickHouseTcpClientOptions Options(string codec) + { + var builder = ExampleConfig.TcpBuilder(); + builder.Compression = codec; + return builder.ToOptions(); + } + + private static string Describe(IClickHouseCompressor compressor) + => compressor is null ? "null (no compression)" : compressor.GetType().Name; + + /// Runs the query through a counting proxy and reports the bytes each way. + private static async Task<(long FromServer, long ToServer, long Rows)> Measure( + ClickHouseTcpClientOptions options, + ClickHouseTcpQueryOptions? queryOptions) + { + await using var proxy = new CountingProxy(ExampleConfig.Host, ExampleConfig.TcpPort); + + long rows = 0; + + // The client is disposed before the counters are read, so every byte of the handshake, the query and the + // close is included. The handshake is a few hundred bytes and identical between the runs. + await using (var client = new ClickHouseTcpClient(options with { Host = "127.0.0.1", Port = proxy.Port })) + { + await foreach (Block block in client.StreamAsync(Query, queryOptions)) + { + rows += block.RowCount; + } + } + + return (proxy.FromServer, proxy.ToServer, rows); + } + + private static async Task MeasureInsert(ClickHouseTcpClientOptions options) + { + await using var proxy = new CountingProxy(ExampleConfig.Host, ExampleConfig.TcpPort); + + var ids = new ulong[InsertRows]; + var text = new string[InsertRows]; + for (int i = 0; i < InsertRows; i++) + { + ids[i] = (ulong)i; + text[i] = (i % 97).ToString(); + } + + await using (var client = new ClickHouseTcpClient(options with { Host = "127.0.0.1", Port = proxy.Port })) + { + await client.InsertAsync( + $"INSERT INTO {TableName} (id, text) VALUES", + [ClickHouseTcpColumn.Create("id", ids), ClickHouseTcpColumn.Create("text", text)]); + } + + return proxy.ToServer; + } + + /// + /// A local socket that forwards to the real server and counts the bytes each way. Nothing an application needs + /// — it is here because wire size is not otherwise observable from the client, and a byte count is the only + /// claim about compression that loopback can support. + /// + private sealed class CountingProxy : IAsyncDisposable + { + private readonly TcpListener listener; + private readonly CancellationTokenSource stopping = new(); + private readonly Task accepting; + private long fromServer; + private long toServer; + + public CountingProxy(string host, int port) + { + listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + Port = ((IPEndPoint)listener.LocalEndpoint).Port; + accepting = AcceptLoop(host, port); + } + + /// The loopback port to point a client at. + public int Port { get; } + + public long FromServer => Interlocked.Read(ref fromServer); + + public long ToServer => Interlocked.Read(ref toServer); + + public async ValueTask DisposeAsync() + { + await stopping.CancelAsync(); + listener.Stop(); + try + { + await accepting; + } + catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) + { + } + + stopping.Dispose(); + } + + private async Task AcceptLoop(string host, int port) + { + var sessions = new List(); + try + { + while (!stopping.IsCancellationRequested) + { + TcpClient accepted = await listener.AcceptTcpClientAsync(stopping.Token); + sessions.Add(Forward(accepted, host, port)); + } + } + catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) + { + // The listener was stopped, which is the normal ending here. + } + + foreach (Task session in sessions) + { + try + { + await session; + } + catch (Exception ex) when (ex is OperationCanceledException or IOException or SocketException or ObjectDisposedException) + { + } + } + } + + private async Task Forward(TcpClient downstream, string host, int port) + { + using TcpClient upstream = new(); + using (downstream) + { + await upstream.ConnectAsync(host, port, stopping.Token); + await Task.WhenAll( + Copy(downstream.GetStream(), upstream.GetStream(), towardsServer: true), + Copy(upstream.GetStream(), downstream.GetStream(), towardsServer: false)); + } + } + + private async Task Copy(Stream from, Stream to, bool towardsServer) + { + byte[] buffer = new byte[64 * 1024]; + try + { + while (true) + { + int read = await from.ReadAsync(buffer, stopping.Token); + if (read == 0) + { + break; + } + + Interlocked.Add(ref towardsServer ? ref toServer : ref fromServer, read); + await to.WriteAsync(buffer.AsMemory(0, read), stopping.Token); + await to.FlushAsync(stopping.Token); + } + } + catch (Exception ex) when (ex is OperationCanceledException or IOException or SocketException or ObjectDisposedException) + { + // Either side closing ends the copy; the counts up to that point are what matters. + } + } + } +} diff --git a/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs b/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs new file mode 100644 index 000000000..fbdfe96a6 --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs @@ -0,0 +1,187 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// +/// and +/// : what the server said about itself during the handshake, and the two +/// different things to gate a feature on — the protocol revision for anything the wire has to carry, and +/// the server version for anything SQL has to name. +/// +/// +/// Code that runs against one server you control needs none of this. Code that ships — a library, a migration +/// tool, an agent deployed across a fleet — meets 25.8 and 26.7 on the same afternoon, and the choice is between +/// asking first and catching an error afterwards. Asking is cheaper and says why in the log. +/// +/// +public static class TcpServerInfo +{ + /// The protocol revision that added the query-parameters list to the Query packet. + private const int ParametersRevision = 54459; + + /// The revision that introduced per-packet chunk framing, used here as a gate that does not pass. + private const int ChunkedFramingRevision = 54470; + + /// The oldest server this driver is tested against. + private static readonly Version SupportedFloor = new(25, 8); + + /// QBit(Int8, N) needs a newer server; QBit itself does not. Tcp_015 is about the type. + private static readonly Version QBitInt8From = new(26, 7); + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + // One call, and the answer came from the handshake rather than from a query — there is no round trip + // beyond opening a connection, so this is cheap enough to do at startup. + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + + EveryField(server); + await VersionAgainstSqlVersion(client, server); + await GatingOnTheRevision(client, server); + await GatingOnTheVersion(client, server); + WhichGateToUse(); + } + + private static void EveryField(ClickHouseTcpServerInfo server) + { + Console.WriteLine("1. What the handshake carried\n"); + Console.WriteLine($" ToString() {server}"); + Console.WriteLine($" Name {server.Name}"); + Console.WriteLine($" Version {server.Version}"); + Console.WriteLine($" VersionMajor {server.VersionMajor}"); + Console.WriteLine($" VersionMinor {server.VersionMinor}"); + Console.WriteLine($" VersionPatch {server.VersionPatch}"); + Console.WriteLine($" ProtocolRevision {server.ProtocolRevision}"); + Console.WriteLine($" Timezone {Quote(server.Timezone)}"); + Console.WriteLine($" DisplayName {Quote(server.DisplayName)}"); + Console.WriteLine(); + Console.WriteLine(" Timezone is the server's own, and it is what a bare DateTime column means — Tcp_012 is"); + Console.WriteLine(" about that. DisplayName is whatever display_name the server was configured with, which"); + Console.WriteLine(" is often the container's host name and is empty when nothing set it."); + Console.WriteLine(); + Console.WriteLine(" ClickHouseTcpServerInfo is a record, so two readings compare equal by value and it is"); + Console.WriteLine(" safe to cache. None of it changes for the life of a connection."); + } + + private static async Task VersionAgainstSqlVersion(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) + { + Console.WriteLine("\n2. Version, and the fourth number that is not in it\n"); + + object sqlVersion = await client.ExecuteScalarAsync("SELECT version()"); + object sqlTimezone = await client.ExecuteScalarAsync("SELECT timezone()"); + + Console.WriteLine($" server.Version {server.Version}"); + Console.WriteLine($" SELECT version() {sqlVersion}"); + Console.WriteLine($" server.Timezone {server.Timezone}"); + Console.WriteLine($" SELECT timezone() {sqlTimezone}"); + Console.WriteLine(); + Console.WriteLine(" The handshake carries three numbers, so Version is major.minor.patch and the build"); + Console.WriteLine(" number that version() shows has nowhere to go. Compare against Version for a feature"); + Console.WriteLine(" gate — the three numbers are what a release note names — and read version() only when"); + Console.WriteLine(" you want the exact build for a bug report."); + } + + private static async Task GatingOnTheRevision(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) + { + Console.WriteLine("\n3. Gating on ProtocolRevision, for what the wire has to carry\n"); + Console.WriteLine(" The revision is the lower of what the client and the server support, so it can be"); + Console.WriteLine(" below what either alone offers. Everything the protocol grew — a field, a packet, a"); + Console.WriteLine(" framing — is switched on by a number like these.\n"); + + Console.WriteLine($" negotiated {server.ProtocolRevision}"); + Console.WriteLine($" query parameters need {ParametersRevision} -> {Available(server.ProtocolRevision >= ParametersRevision)}"); + Console.WriteLine($" per-packet chunk framing needs {ChunkedFramingRevision} -> {Available(server.ProtocolRevision >= ChunkedFramingRevision)}"); + Console.WriteLine(); + + if (server.ProtocolRevision >= ParametersRevision) + { + var options = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "floor", 90UL } }, + }; + object count = await client.ExecuteScalarAsync( + "SELECT count() FROM numbers(100) WHERE number >= {floor:UInt64}", + options); + + Console.WriteLine($" So the parameterized query ran: count() = {count}."); + Console.WriteLine($" Below {ParametersRevision} the server has nowhere to read the parameters list from, and rejects"); + Console.WriteLine(" the query rather than running it unparameterized — which is the right failure, but"); + Console.WriteLine(" not one to discover in production. Tcp_007 covers parameters themselves."); + } + else + { + Console.WriteLine($" Skipped the parameterized query: this connection negotiated {server.ProtocolRevision}, and query"); + Console.WriteLine($" parameters need {ParametersRevision}. Interpolate the value into the SQL text instead, and quote it."); + } + + Console.WriteLine(); + Console.WriteLine(" The chunk-framing row is a real gate rather than a hypothetical one: it is above the"); + Console.WriteLine(" revision this connection negotiated, so nothing on this connection uses it. That is the"); + Console.WriteLine(" asymmetry to remember — a newer server alone does not raise the number, because the"); + Console.WriteLine(" client has to offer the revision too."); + } + + private static async Task GatingOnTheVersion(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) + { + Console.WriteLine("\n4. Gating on Version, for what SQL has to name\n"); + + // The passing direction: the floor the driver is tested against. + bool supported = server.Version >= SupportedFloor; + Console.WriteLine($" this driver is tested from {SupportedFloor} upwards, and the server is {server.Version} -> {Available(supported)}"); + + // The failing direction, with the same shape Tcp_015 uses. A type the server refuses outright cannot be + // caught cheaply: the CREATE TABLE fails, so ask first. + if (server.Version >= QBitInt8From) + { + const string table = "example_tcp_serverinfo_qbit"; + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + try + { + await client.ExecuteAsync($"CREATE TABLE {table} (v QBit(Int8, 8)) ENGINE = MergeTree ORDER BY tuple()"); + object declared = await client.ExecuteScalarAsync( + $"SELECT type FROM system.columns WHERE table = '{table}' AND name = 'v'"); + Console.WriteLine($" QBit(Int8, 8) declared as {declared}"); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + } + else + { + Console.WriteLine($" QBit(Int8, N) needs ClickHouse {QBitInt8From} or newer, and this server is {server.Version}:"); + Console.WriteLine(" skipped. On 26.6 the server refuses the type outright — 'QBit data type only"); + Console.WriteLine(" supports BFloat16, Float32, or Float64 as element type' — so a client that"); + Console.WriteLine(" offers Int8 vectors has to know before it writes the DDL. Tcp_015 is about QBit."); + } + + Console.WriteLine(); + Console.WriteLine(" A skip that prints why is worth more than a caught exception: the reason survives into"); + Console.WriteLine(" the log, and nothing had to be attempted against a server that would refuse it."); + } + + private static void WhichGateToUse() + { + Console.WriteLine("\n5. Which of the two to read\n"); + Console.WriteLine(" ProtocolRevision anything the wire carries: query parameters, the fields of a"); + Console.WriteLine(" progress packet, chunk framing, custom serialization. The client"); + Console.WriteLine(" already gates its own reads and writes on it, so this matters when"); + Console.WriteLine(" your own code depends on a protocol-level capability."); + Console.WriteLine(); + Console.WriteLine(" Version anything SQL names: a data type, a function, a table setting, a"); + Console.WriteLine(" SETTINGS key. None of it is visible in the revision, because the"); + Console.WriteLine(" protocol did not change to carry it."); + Console.WriteLine(); + Console.WriteLine(" Two things this record does not tell you, and where to get them:"); + Console.WriteLine(" the cluster SELECT * FROM system.clusters"); + Console.WriteLine(" the build SELECT * FROM system.build_options"); + Console.WriteLine(); + Console.WriteLine(" For a health check, prefer PingAsync: it is a protocol ping rather than a SELECT, so it"); + Console.WriteLine(" proves the connection without asking the server to plan anything."); + } + + private static string Available(bool yes) => yes ? "available" : "not on this connection"; + + private static string Quote(string value) => value.Length == 0 ? "(empty)" : $"'{value}'"; +} From 886184759221f477192b69543a56625ce59f5c13 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 16:38:38 +0200 Subject: [PATCH 11/16] Add the native protocol's observability examples Five under examples/Tcp/Observability/: the three log categories and what each reports, with two filter sets; OpenTelemetry, collecting the client's spans and showing the server's own joining the same trace; the three block-shaped callbacks Tcp_021 does not cover, copying out of a borrowed block; PingAsync as a health check, measured against SELECT 1; and a throwaway container reached over the native port. Tcp_030 waits on a handshake rather than on either wait strategy. Neither probe tests the native protocol - the HTTP one tests the other listener, and UntilInternalTcpPortIsAvailable only says the port is bound - so on a loaded machine the first handshake is refused after both have passed. Forced with a no-condition strategy that took 43 attempts over 4.4 seconds. Only a transport exception is retried: a server exception means it answered and rejected us. Tcp/README.md now records W3C trace context propagation under what only the native client does. The client sends the current Activity's ids with each query, so the server's spans join the caller's trace; the HTTP transport sends no traceparent. Co-Authored-By: Claude Opus 5 (1M context) --- examples/Program.cs | 25 ++ examples/README.md | 8 + examples/Tcp/Observability/Tcp_026_Logging.cs | 345 +++++++++++++++ .../Observability/Tcp_027_OpenTelemetry.cs | 393 ++++++++++++++++++ .../Observability/Tcp_028_MetadataBlocks.cs | 335 +++++++++++++++ .../Tcp/Observability/Tcp_029_HealthChecks.cs | 225 ++++++++++ .../Observability/Tcp_030_Testcontainers.cs | 141 +++++++ examples/Tcp/README.md | 3 + 8 files changed, 1475 insertions(+) create mode 100644 examples/Tcp/Observability/Tcp_026_Logging.cs create mode 100644 examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs create mode 100644 examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs create mode 100644 examples/Tcp/Observability/Tcp_029_HealthChecks.cs create mode 100644 examples/Tcp/Observability/Tcp_030_Testcontainers.cs diff --git a/examples/Program.cs b/examples/Program.cs index 8bf2497ab..934eadb16 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -471,6 +471,31 @@ private static async Task RunAllExamples(bool isInteractive) await TcpServerInfo.Run(); WaitForUser(isInteractive); + // Native Protocol: Observability + Console.WriteLine("\n\n" + new string('=', 70)); + Console.WriteLine("NATIVE PROTOCOL: OBSERVABILITY"); + Console.WriteLine(new string('=', 70) + "\n"); + + Console.WriteLine($"Running: {nameof(TcpLogging)}"); + await TcpLogging.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpOpenTelemetry)}"); + await TcpOpenTelemetry.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpMetadataBlocks)}"); + await TcpMetadataBlocks.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpHealthChecks)}"); + await TcpHealthChecks.Run(); + WaitForUser(isInteractive); + + Console.WriteLine($"\n\nRunning: {nameof(TcpTestcontainers)}"); + await TcpTestcontainers.Run(); + WaitForUser(isInteractive); + Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ALL EXAMPLES COMPLETED SUCCESSFULLY!"); Console.WriteLine(new string('=', 70)); diff --git a/examples/README.md b/examples/README.md index 772493f49..454e74654 100644 --- a/examples/README.md +++ b/examples/README.md @@ -142,6 +142,14 @@ These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/READ - [Tcp_024_Compression.cs](Tcp/Advanced/Tcp_024_Compression.cs) - `Compression=lz4|zstd|none` and `ClickHouseTcpClientOptions.Compressor`, measured in bytes on the wire: what LZ4 saves by default, why the client's codec does not choose what the server sends (`network_compression_method` does), what it does choose on an insert, and why loopback cannot measure the benefit - [Tcp_025_ServerInfo.cs](Tcp/Advanced/Tcp_025_ServerInfo.cs) - `GetServerInfoAsync` and every field of `ClickHouseTcpServerInfo`, the build number `Version` does not carry, gating on `ProtocolRevision` (query parameters need 54459) with one gate that passes and one that does not, and gating on the server version with a printed skip +### Native Protocol: Observability + +- [Tcp_026_Logging.cs](Tcp/Observability/Tcp_026_Logging.cs) - `ClickHouseTcpClientOptions.LoggerFactory` and the three categories `ClickHouseTcpDiagnostics.ClientLogCategory`, `.ConnectionLogCategory`, `.PoolLogCategory`: what each reports, which levels they use (nothing at `Information`), why a stock `ILoggerFactory` shows one line, and two measured filter sets — production against debugging a connection problem +- [Tcp_027_OpenTelemetry.cs](Tcp/Observability/Tcp_027_OpenTelemetry.cs) - `ClickHouseTcpDiagnostics.ActivitySourceName` and `IncludeSqlInActivityTags`, with spans collected and printed: the span names and attributes, `connect` nested under the statement that dialled, the W3C trace context the client propagates so the server's own spans join the trace, and why the source is separate from the HTTP transport's +- [Tcp_028_MetadataBlocks.cs](Tcp/Observability/Tcp_028_MetadataBlocks.cs) - The three `Block`-shaped callbacks Tcp_021 does not cover — `OnLog` with `send_logs_level`, `OnTotals` for `WITH TOTALS`, `OnExtremes` with `extremes` — the borrowed-block rule of copying inside the callback, the log priority scale, and what each `send_logs_level` costs +- [Tcp_029_HealthChecks.cs](Tcp/Observability/Tcp_029_HealthChecks.cs) - `PingAsync` as a health check over an `AddClickHouseTcpDataSource` registration: a protocol ping measured against `SELECT 1`, Healthy, Degraded and Unhealthy in one report, and what a Pong does and does not prove +- [Tcp_030_Testcontainers.cs](Tcp/Observability/Tcp_030_Testcontainers.cs) - A throwaway ClickHouse over the native protocol: the mapped 9000 rather than `GetConnectionString()`'s 8123, why a native-port wait strategy alone reports ready too early, and a query against the container + ## How to run ### Prerequisites diff --git a/examples/Tcp/Observability/Tcp_026_Logging.cs b/examples/Tcp/Observability/Tcp_026_Logging.cs new file mode 100644 index 000000000..427277146 --- /dev/null +++ b/examples/Tcp/Observability/Tcp_026_Logging.cs @@ -0,0 +1,345 @@ +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Driver.Examples; + +/// +/// and the three categories the native client logs under — +/// , +/// and : what each reports, at which level, and how to keep +/// one and drop the rest. +/// +/// +/// The client logs its own lifecycle. It never logs what the server reports; the server's log lines arrive +/// as a callback (Tcp_028). Nothing here reports pool state either — the pool's lines are the only window into it, +/// which is what Tcp_017 reads. +/// +/// +/// +/// The levels are the thing to plan around: the client writes almost everything at Debug or Trace, +/// and the statement text rides on a Debug line. So the two filter sets below are genuinely different +/// configurations, not one dialled up — and a stock , whose minimum level is +/// Information, shows nearly none of it. +/// +/// +public static class TcpLogging +{ + public static async Task Run() + { + await OneWorkloadThreeCategories(); + await WhichLevelsAreUsed(); + await AStockFactoryShowsAlmostNothing(); + await TwoFilterSets(); + await StatementTextRidesOnADebugLine(); + WhatIsNotHere(); + } + + private static async Task OneWorkloadThreeCategories() + { + Console.WriteLine("1. Three categories, one workload\n"); + + var recorder = new Recorder(); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder + .AddProvider(recorder) + .SetMinimumLevel(LogLevel.Trace)); + + await Workload(factory); + + Console.WriteLine(" Two queries, then one that names a table that does not exist:\n"); + foreach (Line line in recorder.Lines) + { + Console.WriteLine($" {line.ShortCategory,-10} {line.Level,-11} {line.Message}"); + } + + Console.WriteLine(); + Console.WriteLine(" Client what ran, how long it took, how it ended — and the statement text"); + Console.WriteLine(" Connection the dial, the TLS negotiation, and the handshake result"); + Console.WriteLine(" Pool checkouts, retirement, exhaustion, and the background work nobody awaits"); + Console.WriteLine(); + Console.WriteLine(" Each is a full logger category, so the usual per-category configuration applies:"); + Console.WriteLine(" appsettings' Logging:LogLevel section, AddFilter, or a filter predicate as below."); + } + + private static async Task WhichLevelsAreUsed() + { + Console.WriteLine("\n2. Which levels each category actually uses\n"); + + var recorder = new Recorder(); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder + .AddProvider(recorder) + .SetMinimumLevel(LogLevel.Trace)); + + // The same workload, plus the two things that log above Debug: a dial that fails, and a pool with nothing + // left to hand out. + await Workload(factory); + await FailedDial(factory); + await ExhaustedPool(factory); + + foreach (var group in recorder.Lines + .GroupBy(l => (l.ShortCategory, l.Level)) + .OrderBy(g => g.Key.ShortCategory, StringComparer.Ordinal) + .ThenByDescending(g => g.Key.Level)) + { + Console.WriteLine($" {group.Key.ShortCategory,-10} {group.Key.Level,-11} {group.Count(),2} line(s) e.g. {Trim(group.First().Message)}"); + } + + Console.WriteLine(); + Console.WriteLine(" Nothing is logged at Information or Critical. Warning is the top of the range and it is"); + Console.WriteLine(" reserved for four messages: a dial that failed, PoolTimeout, and the two background jobs"); + Console.WriteLine(" nobody awaits (a failed top-up towards MinPoolSize, a failed sweep) — which are reported"); + Console.WriteLine(" nowhere else at all. Error is a single message, an operation that threw — twice here,"); + Console.WriteLine(" once for the unknown table and once for the query that never got a connection."); + } + + private static async Task AStockFactoryShowsAlmostNothing() + { + Console.WriteLine("\n3. A factory with no minimum level set shows almost none of it\n"); + + var recorder = new Recorder(); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddProvider(recorder)); + + await Workload(factory); + + Console.WriteLine($" LoggerFactory.Create(b => b.AddProvider(...)) with no SetMinimumLevel: {recorder.Lines.Count} line(s) kept"); + foreach (Line line in recorder.Lines) + { + Console.WriteLine($" {line.ShortCategory,-10} {line.Level,-11} {line.Message}"); + } + + Console.WriteLine(); + Console.WriteLine(" Microsoft.Extensions.Logging defaults its minimum to Information, and the client logs"); + Console.WriteLine(" nothing there, so a factory that was wired up correctly still looks broken. Set the level"); + Console.WriteLine(" for the categories you want, not globally: Trace across the whole application is a lot of"); + Console.WriteLine(" log."); + } + + private static async Task TwoFilterSets() + { + Console.WriteLine("\n4. Two filter sets: one for production, one for a connection problem\n"); + + // Production: only the lines that mean something is wrong. No statement text, because the line that + // carries it is a Debug line. + var production = new Recorder(); + using (ILoggerFactory factory = LoggerFactory.Create(builder => builder + .AddProvider(production) + .AddFilter((category, level) => category?.StartsWith("ClickHouse.Driver.Tcp.", StringComparison.Ordinal) == true && level >= LogLevel.Warning) + .SetMinimumLevel(LogLevel.Warning))) + { + await Workload(factory); + await FailedDial(factory); + await ExhaustedPool(factory); + } + + Console.WriteLine($" Production — every category, Warning and worse: {production.Lines.Count} line(s)\n"); + foreach (Line line in production.Lines) + { + Console.WriteLine($" {line.ShortCategory,-10} {line.Level,-11} {Trim(line.Message)}"); + } + + // Debugging a connection problem: the two categories that know about sockets, at Trace, and nothing else. + var connections = new Recorder(); + using (ILoggerFactory factory = LoggerFactory.Create(builder => builder + .AddProvider(connections) + .AddFilter((category, level) => category switch + { + ClickHouseTcpDiagnostics.ConnectionLogCategory => level >= LogLevel.Trace, + ClickHouseTcpDiagnostics.PoolLogCategory => level >= LogLevel.Trace, + _ => false, + }) + .SetMinimumLevel(LogLevel.Trace))) + { + await Workload(factory); + await FailedDial(factory); + await ExhaustedPool(factory); + } + + Console.WriteLine($"\n Debugging a connection problem — Connection and Pool at Trace, nothing else: {connections.Lines.Count} line(s)\n"); + foreach (Line line in connections.Lines) + { + Console.WriteLine($" {line.ShortCategory,-10} {line.Level,-11} {Trim(line.Message)}"); + } + + Console.WriteLine(); + Console.WriteLine(" The second set answers the questions the first cannot: how many connections were opened,"); + Console.WriteLine(" whether a query reused one or dialled, how old the reused one was, and whether a returned"); + Console.WriteLine(" connection went back into the pool. Note that no Client line appears in it — the query"); + Console.WriteLine(" text is deliberately out, which is what makes the set safe to turn on against a live"); + Console.WriteLine(" system."); + Console.WriteLine(); + Console.WriteLine(" Both are predicates over the category string, so they can key on"); + Console.WriteLine(" ClickHouseTcpDiagnostics.ClientLogCategory and its two siblings rather than a literal."); + } + + private static async Task StatementTextRidesOnADebugLine() + { + Console.WriteLine("\n5. The statement text, and the one line that carries it\n"); + + const string sql = "SELECT 'the whole statement, or as much of it as StatementMaxLength allows'"; + + foreach (int max in new[] { 0, 30, 200 }) + { + var recorder = new Recorder(); + using ILoggerFactory factory = LoggerFactory.Create(builder => builder + .AddProvider(recorder) + .AddFilter((category, level) => category == ClickHouseTcpDiagnostics.ClientLogCategory && level >= LogLevel.Debug) + .SetMinimumLevel(LogLevel.Debug)); + + await using (var client = new ClickHouseTcpClient(Options() with + { + LoggerFactory = factory, + StatementMaxLength = max, + })) + { + _ = await client.ExecuteScalarAsync(sql); + } + + string running = recorder.Lines.First(l => l.Message.StartsWith("Running", StringComparison.Ordinal)).Message; + Console.WriteLine($" StatementMaxLength = {max,3} {running}"); + } + + Console.WriteLine(); + Console.WriteLine($" The statement was {sql.Length} characters. Zero keeps the text out of the log line while"); + Console.WriteLine(" leaving the line itself — which is the production recipe if you want a record of what ran"); + Console.WriteLine(" and how long it took without putting query text in your logs. The same knob caps the"); + Console.WriteLine(" db.query.text span attribute (Tcp_027), and Tcp_019 covers it as a limit."); + } + + private static void WhatIsNotHere() + { + Console.WriteLine("\n6. What these categories do not carry\n"); + Console.WriteLine(" The server's own log lines. Those come from the query, not the client, and reach you"); + Console.WriteLine(" through ClickHouseTcpQueryCallbacks.OnLog with send_logs_level set — Tcp_028. Bridging"); + Console.WriteLine(" them into an ILogger is a few lines, and yours to write."); + Console.WriteLine(); + Console.WriteLine(" Pool counters. There is no open/idle/in-use to read, so the Pool category's lines are the"); + Console.WriteLine(" only window into the pool; Tcp_017 measures it that way."); + Console.WriteLine(); + Console.WriteLine(" A connection identity. The reuse line carries a use count and an age, but nothing names"); + Console.WriteLine(" the connection, so two lines about the same socket cannot be tied together."); + Console.WriteLine(); + Console.WriteLine(" In an application you would not build the factory by hand at all: register logging in the"); + Console.WriteLine(" container and AddClickHouseTcpDataSource fills LoggerFactory in from it (Tcp_003)."); + } + + private static ClickHouseTcpClientOptions Options() => ExampleConfig.TcpBuilder().ToOptions(); + + /// Two queries that succeed and one that does not, on a client of this example's own. + private static async Task Workload(ILoggerFactory factory) + { + await using var client = new ClickHouseTcpClient(Options() with + { + LoggerFactory = factory, + StatementMaxLength = 60, + }); + + _ = await client.ExecuteScalarAsync("SELECT 'the first query has to open a connection'"); + _ = await client.ExecuteScalarAsync("SELECT 'the second reuses it'"); + + // An unknown table is reported after the server accepted the query, so the connection survives it and goes + // back into the pool. The client logs one Error line and rethrows. + try + { + _ = await client.ExecuteScalarAsync("SELECT * FROM example_tcp_logging_no_such_table"); + } + catch (ClickHouseTcpServerException) + { + } + } + + /// A dial that fails at once, for the Connection category's one Warning. + private static async Task FailedDial(ILoggerFactory factory) + { + await using var client = new ClickHouseTcpClient(Options() with + { + LoggerFactory = factory, + Port = 1, + DialTimeout = TimeSpan.FromSeconds(2), + }); + + try + { + await client.PingAsync(); + } + catch (ClickHouseTcpTransportException) + { + } + } + + /// A pool with its only connection pinned by a session, for the Pool category's PoolTimeout Warning. + private static async Task ExhaustedPool(ILoggerFactory factory) + { + await using var client = new ClickHouseTcpClient(Options() with + { + LoggerFactory = factory, + MaxPoolSize = 1, + PoolTimeout = TimeSpan.FromMilliseconds(200), + }); + + await using IClickHouseTcpSession session = await client.OpenSessionAsync(); + + try + { + _ = await client.ExecuteScalarAsync("SELECT 1"); + } + catch (TimeoutException) + { + } + } + + private static string Trim(string message) + => message.Length <= 96 ? message : message[..96] + "..."; + + private readonly record struct Line(string Category, LogLevel Level, string Message) + { + /// The part after the last dot — Client, Connection or Pool. + public string ShortCategory => Category[(Category.LastIndexOf('.') + 1)..]; + } + + /// + /// An that keeps the lines rather than printing them, so a section can report + /// what its filter kept. Registered with AddProvider, so the builder's filters really do apply — a + /// factory that only wraps would bypass them and prove nothing. + /// + private sealed class Recorder : ILoggerProvider + { + private readonly List lines = []; + + public IReadOnlyList Lines + { + get + { + lock (lines) + { + return lines.ToArray(); + } + } + } + + public ILogger CreateLogger(string categoryName) => new Sink(categoryName, lines); + + public void Dispose() + { + } + + private sealed class Sink(string category, List lines) : ILogger + { + public bool IsEnabled(LogLevel logLevel) => true; + + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + lock (lines) + { + lines.Add(new Line(category, logLevel, formatter(state, exception))); + } + } + } + } +} diff --git a/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs b/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs new file mode 100644 index 000000000..a6b2df7d6 --- /dev/null +++ b/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs @@ -0,0 +1,393 @@ +using System.Diagnostics; +using ClickHouse.Driver.Diagnostic; +using ClickHouse.Driver.Tcp; +using ClickHouse.Driver.Utility; +using OpenTelemetry; +using OpenTelemetry.Trace; + +namespace ClickHouse.Driver.Examples; + +/// +/// Tracing the native client: , the spans it emits and +/// the attributes they carry, and . +/// +/// +/// The spans are collected here by an exporter that keeps them, and printed, because a console exporter's output +/// is too wide to read next to the code that produced it. The wiring is otherwise exactly what an application +/// does — Sdk.CreateTracerProviderBuilder().AddSource(ClickHouseTcpDiagnostics.ActivitySourceName) — so +/// swapping in AddOtlpExporter() is the only change needed to send these spans somewhere real. +/// +/// +/// +/// The attribute names are the current OpenTelemetry database conventions (db.system.name, +/// db.namespace, db.query.text, server.address), which is where this transport differs from +/// the HTTP one: it still emits the older db.system/db.statement set. The two also use different +/// source names, so either can be collected without the other. +/// +/// +public static class TcpOpenTelemetry +{ + private const string TableName = "example_tcp_open_telemetry"; + + /// Stands in for the application's own instrumentation, so the client's spans have a parent. + private static readonly ActivitySource AppSource = new("ClickHouse.Driver.Examples.Tcp027"); + + public static async Task Run() + { + Console.WriteLine($"The native client's ActivitySource: {ClickHouseTcpDiagnostics.ActivitySourceName}"); + Console.WriteLine($"The HTTP transport's, for comparison: {ClickHouseDiagnosticsOptions.ActivitySourceName}\n"); + + await using var client = ExampleConfig.CreateTcpClient(); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + await client.ExecuteAsync($"CREATE TABLE {TableName} (id UInt64, note String) ENGINE = MergeTree ORDER BY id"); + + try + { + await OneSpanPerOperation(); + await TheParentChildShape(); + await StatementTextIsOptIn(); + await TheServerJoinsTheSameTrace(client); + await TwoTransportsTwoSources(); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + Console.WriteLine($"\nDropped {TableName}."); + } + } + + private static async Task OneSpanPerOperation() + { + Console.WriteLine("1. One span per operation, named after the statement\n"); + + var collector = new SpanCollector(); + using (TracerProvider provider = Collect(collector, ClickHouseTcpDiagnostics.ActivitySourceName)) + { + await using var client = new ClickHouseTcpClient(Options() with + { + IncludeSqlInActivityTags = true, + StatementMaxLength = 120, + }); + + await client.PingAsync(); + _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(50000)"); + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, note) VALUES", + [[1UL, "one"], [2UL, "two"], [3UL, "three"]]); + + // An error after the server accepted the query, so the span records a failure and the connection + // stays usable. + try + { + _ = await client.ExecuteScalarAsync("SELECT * FROM example_tcp_open_telemetry_no_such_table"); + } + catch (ClickHouseTcpServerException) + { + } + } + + foreach (Activity span in collector.Spans) + { + Print(span); + } + + Console.WriteLine(" The span name is the statement's leading keyword, uppercased, which keeps it low"); + Console.WriteLine(" cardinality — a generated statement that does not start with a word is named 'query'."); + Console.WriteLine(" A Ping is its own span, and so is a dial (below)."); + Console.WriteLine(); + Console.WriteLine(" db.clickhouse.read_rows and read_bytes come from the server's Progress packets, so they"); + Console.WriteLine(" describe what the query read rather than what it returned; result_rows and result_bytes"); + Console.WriteLine(" are the execution summary. An insert has neither pair: the server sends no Progress for"); + Console.WriteLine(" rows streamed to it, so db.clickhouse.written_rows is the client's own count."); + } + + private static async Task TheParentChildShape() + { + Console.WriteLine("\n2. Where the client's spans sit in a trace\n"); + + var collector = new SpanCollector(); + using (TracerProvider provider = Collect(collector, ClickHouseTcpDiagnostics.ActivitySourceName, AppSource.Name)) + { + // The application's own span. Everything the client starts while it is current becomes a descendant. + using (Activity? request = AppSource.StartActivity("handle-request")) + { + // A client of its own, so its pool is empty and the first operation has to dial. + await using var client = new ClickHouseTcpClient(Options() with + { + IncludeSqlInActivityTags = true, + StatementMaxLength = 60, + }); + + _ = await client.ExecuteScalarAsync("SELECT 'first, so this one dials'"); + _ = await client.ExecuteScalarAsync("SELECT 'second, so this one does not'"); + } + } + + PrintTree(collector.Spans); + + Console.WriteLine(); + Console.WriteLine(" 'connect' covers the socket connect, the TLS negotiation and the handshake, and it is a"); + Console.WriteLine(" child of whichever operation had to wait for the connection — so a slow first request"); + Console.WriteLine(" shows why in the trace rather than only in the total. The second statement has no such"); + Console.WriteLine(" child because it reused the pooled connection."); + Console.WriteLine(); + Console.WriteLine(" With no ambient Activity the client's spans are roots, one trace each — which is what"); + Console.WriteLine(" section 1 above produced. A parent is also what the server is told about, so the shape"); + Console.WriteLine(" above reaches further than this process: section 4."); + } + + private static async Task StatementTextIsOptIn() + { + Console.WriteLine("\n3. IncludeSqlInActivityTags, and how much text it lets through\n"); + + const string sql = "SELECT 'a statement long enough that StatementMaxLength has something to cut'"; + + (bool include, int max)[] cases = + [ + (false, 200), + (true, 40), + (true, 200), + (true, 0), + ]; + + foreach ((bool include, int max) in cases) + { + var collector = new SpanCollector(); + using (TracerProvider provider = Collect(collector, ClickHouseTcpDiagnostics.ActivitySourceName)) + { + await using var client = new ClickHouseTcpClient(Options() with + { + IncludeSqlInActivityTags = include, + StatementMaxLength = max, + }); + + _ = await client.ExecuteScalarAsync(sql); + } + + Activity span = collector.Spans.First(s => s.OperationName == "SELECT"); + object? text = span.GetTagItem("db.query.text"); + Console.WriteLine($" IncludeSqlInActivityTags = {include,-5} StatementMaxLength = {max,3} db.query.text = {(text is null ? "(not set)" : "\"" + text + "\"")}"); + } + + Console.WriteLine(); + Console.WriteLine($" The statement was {sql.Length} characters. Off is the default, because a statement can carry"); + Console.WriteLine(" data a trace is not meant to hold — a literal in a WHERE clause is often the very value"); + Console.WriteLine(" you are not allowed to export. StatementMaxLength defaults to 5, a stub rather than a"); + Console.WriteLine(" statement, so recording query text takes both settings; zero suppresses the attribute"); + Console.WriteLine(" even with the opt-in on. It caps the Debug log line by the same rule (Tcp_026)."); + } + + /// + /// The client writes the current span's W3C trace context into the Query packet, so the spans the server + /// records for the same query land under the caller's trace id. + /// + /// A client for reading the server's span log, whose own spans are not collected. + private static async Task TheServerJoinsTheSameTrace(ClickHouseTcpClient reader) + { + Console.WriteLine("\n4. The server's own spans join the same trace\n"); + + var collector = new SpanCollector(); + string traceId; + + using (TracerProvider provider = Collect(collector, ClickHouseTcpDiagnostics.ActivitySourceName, AppSource.Name)) + { + await using var client = new ClickHouseTcpClient(Options()); + + using Activity? request = AppSource.StartActivity("handle-request"); + traceId = request!.TraceId.ToHexString(); + _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(100000)"); + } + + Console.WriteLine($" Trace id on this side: {traceId}"); + Console.WriteLine($" Spans collected here: {string.Join(", ", collector.Spans.Select(s => s.OperationName))}"); + + // The server's spans are queued like any system log, so the flush and the read are retried rather than + // read once — the same shape Tcp_020 uses for system.query_log. + long serverSpans = 0; + var names = new List(); + for (int attempt = 1; attempt <= 5 && serverSpans == 0; attempt++) + { + await reader.ExecuteAsync("SYSTEM FLUSH LOGS"); + serverSpans = Convert.ToInt64(await reader.ExecuteScalarAsync( + "SELECT count() FROM system.opentelemetry_span_log WHERE lower(hex(trace_id)) = {trace:String}", + new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "trace", traceId } }, + })); + + if (serverSpans == 0) + { + await Task.Delay(50); + } + } + + await foreach (object[] row in reader.QueryAsync( + "SELECT DISTINCT operation_name FROM system.opentelemetry_span_log " + + "WHERE lower(hex(trace_id)) = {trace:String} ORDER BY operation_name LIMIT 6", + new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "trace", traceId } }, + })) + { + names.Add((string)row[0]); + } + + Console.WriteLine($" Spans the server recorded under the same trace id: {serverSpans}"); + Console.WriteLine($" {string.Join(", ", names)}"); + Console.WriteLine(); + Console.WriteLine(" The Query packet's ClientInfo carries the W3C trace context of Activity.Current when the"); + Console.WriteLine(" negotiated protocol revision is 54442 or newer, which every supported server is. So the"); + Console.WriteLine(" server's account of the query — every stage, in system.opentelemetry_span_log — is part"); + Console.WriteLine(" of the same trace as the request that issued it, with no header to set and nothing to"); + Console.WriteLine(" correlate by hand. That is the strongest reason to give the client an ambient Activity."); + Console.WriteLine(); + Console.WriteLine(" Two conditions. The current Activity's id has to be W3C, which it is unless something"); + Console.WriteLine(" set ActivityIdFormat.Hierarchical; and the server has to have its span log switched on,"); + Console.WriteLine(" which the stock configuration does. The flush above is only so this example can read the"); + Console.WriteLine(" table immediately; nothing about the propagation needs it."); + Console.WriteLine(); + Console.WriteLine(" db.clickhouse.query_id is the other join, and it needs a QueryId you chose (Tcp_020):"); + Console.WriteLine(" when you supply none, the id the server assigns never reaches the client."); + } + + private static async Task TwoTransportsTwoSources() + { + Console.WriteLine("\n5. The two transports are separate sources\n"); + + // Only the native source. The HTTP query below runs, and is not collected. + var nativeOnly = new SpanCollector(); + using (TracerProvider provider = Collect(nativeOnly, ClickHouseTcpDiagnostics.ActivitySourceName)) + { + await BothTransports(); + } + + // Both sources, same workload. + var both = new SpanCollector(); + using (TracerProvider provider = Collect( + both, + ClickHouseTcpDiagnostics.ActivitySourceName, + ClickHouseDiagnosticsOptions.ActivitySourceName)) + { + await BothTransports(); + } + + Console.WriteLine(" One native query and one HTTP query, collected twice:\n"); + Report("AddSource(native)", nativeOnly); + Report("AddSource(native, http)", both); + + Console.WriteLine(); + Console.WriteLine(" So a service that has moved its reads to the native client and left its writes on HTTP"); + Console.WriteLine(" can trace one, the other, or both, and tell them apart in the backend by source. The"); + Console.WriteLine(" attribute sets differ as well: the HTTP transport emits db.system and db.statement, this"); + Console.WriteLine(" one db.system.name and db.query.text, so a dashboard built on one does not read the"); + Console.WriteLine(" other without a rule for each. The span names differ too — the HTTP one is named after"); + Console.WriteLine(" the driver method that ran, this one after the statement's keyword."); + Console.WriteLine(); + Console.WriteLine(" Their opt-ins are separate too, and shaped differently: the HTTP transport's live on the"); + Console.WriteLine(" static ClickHouseDiagnosticsOptions, so they are process-wide, while the native client's"); + Console.WriteLine(" are per client, on the options record."); + + static void Report(string label, SpanCollector collector) + { + IEnumerable byTransport = collector.Spans + .GroupBy(s => s.Source.Name) + .OrderBy(g => g.Key, StringComparer.Ordinal) + .Select(g => $"{g.Key} -> {string.Join(", ", g.Select(s => s.OperationName))}"); + + Console.WriteLine($" {label,-24} {collector.Spans.Count} span(s): {string.Join("; ", byTransport)}"); + } + } + + /// One query over each transport, so a collector can be asked which of them it saw. + private static async Task BothTransports() + { + await using var tcp = new ClickHouseTcpClient(Options()); + _ = await tcp.ExecuteScalarAsync("SELECT 'over the native protocol'"); + + using var http = ExampleConfig.CreateHttpClient(); + _ = await http.ExecuteScalarAsync("SELECT 'over HTTP'"); + } + + private static ClickHouseTcpClientOptions Options() => ExampleConfig.TcpBuilder().ToOptions(); + + /// + /// The wiring an application writes, with an exporter that keeps the spans instead of printing them. + /// + private static TracerProvider Collect(SpanCollector collector, params string[] sources) + => Sdk.CreateTracerProviderBuilder() + .AddSource(sources) + .AddProcessor(new SimpleActivityExportProcessor(collector)) + .Build()!; + + private static void Print(Activity span) + { + Console.WriteLine($" {span.OperationName,-8} {span.Kind,-6} {span.Status,-5} {span.Duration.TotalMilliseconds,7:0.0} ms"); + foreach (KeyValuePair tag in span.TagObjects) + { + Console.WriteLine($" {tag.Key,-30} {tag.Value}"); + } + + foreach (ActivityEvent e in span.Events) + { + Console.WriteLine($" event {e.Name,-24} {e.Tags.FirstOrDefault(t => t.Key == "exception.type").Value}"); + } + + Console.WriteLine(); + } + + /// Prints the spans indented by depth, which is what a trace viewer draws. + private static void PrintTree(IReadOnlyList spans) + { + var byId = spans.ToDictionary(s => s.SpanId.ToHexString(), StringComparer.Ordinal); + + foreach (Activity span in spans.OrderBy(s => s.StartTimeUtc)) + { + int depth = 0; + for (Activity? walk = span; walk is not null && depth < 8;) + { + walk = byId.TryGetValue(walk.ParentSpanId.ToHexString(), out Activity? parent) ? parent : null; + if (walk is not null) + { + depth++; + } + } + + string? sql = span.GetTagItem("db.query.text") as string; + Console.WriteLine($" {new string(' ', depth * 3)}{span.OperationName,-8} {span.Duration.TotalMilliseconds,7:0.0##} ms{(sql is null ? string.Empty : " " + sql)}"); + } + } + + /// + /// A that keeps what it is given. Registered through + /// , so each span is handed over as it ends and nothing has to be + /// flushed before a section prints. + /// + private sealed class SpanCollector : BaseExporter + { + private readonly List spans = []; + + public IReadOnlyList Spans + { + get + { + lock (spans) + { + return spans.ToArray(); + } + } + } + + public override ExportResult Export(in Batch batch) + { + lock (spans) + { + foreach (Activity span in batch) + { + spans.Add(span); + } + } + + return ExportResult.Success; + } + } +} diff --git a/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs b/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs new file mode 100644 index 000000000..187c36b2e --- /dev/null +++ b/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs @@ -0,0 +1,335 @@ +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Driver.Examples; + +/// +/// The three -shaped callbacks on : +/// with send_logs_level, +/// for WITH TOTALS, and +/// with the extremes setting. Tcp_021 covers the other +/// three, which hand over structs rather than blocks. +/// +/// +/// Every one of these blocks is borrowed. Its columns are views over pooled buffers that are released as +/// soon as the callback returns, so the rule for all three is the same: copy out what you need inside the +/// callback, and keep neither the block, its columns, nor a span over them. Every section below does the copying +/// in the callback and the printing afterwards, which is also what an application does — a callback runs +/// synchronously on the thread draining the response, so the less it does the better. +/// +/// +/// +/// The contract otherwise is the one Tcp_021 states: in packet order, on the reading thread, and never allowed to +/// throw — an exception propagates out of the operation and terminates the connection. Copying a few values +/// cannot throw, which is one more reason to copy and leave. +/// +/// +public static class TcpMetadataBlocks +{ + public static async Task Run() + { + WhatTurnsEachOneOn(); + + await using var client = ExampleConfig.CreateTcpClient(); + + await ServerLogLines(client); + await BridgingThemIntoALogger(client); + await HowMuchEachLevelSays(client); + await TheTotalsRow(client); + await TheExtremesRows(client); + await NothingFiresWhenThereIsNothingToSend(client); + } + + private static void WhatTurnsEachOneOn() + { + Console.WriteLine("Three callbacks, and what each one needs before the server sends anything:\n"); + Console.WriteLine(" OnLog Settings[\"send_logs_level\"] = \"debug\" (or trace) — the default, fatal, is silent"); + Console.WriteLine(" OnTotals WITH TOTALS in the query, right after GROUP BY"); + Console.WriteLine(" OnExtremes Settings[\"extremes\"] = \"1\""); + Console.WriteLine(); + Console.WriteLine("Setting the callback alone gets you nothing, and so does turning the feature on without the"); + Console.WriteLine("callback: the block is decoded either way, to keep the connection aligned, and then dropped."); + } + + private static async Task ServerLogLines(ClickHouseTcpClient client) + { + Console.WriteLine("\n1. OnLog: the server's own log, for this query only\n"); + + // The copies. Everything that outlives the callback is in here, and nothing in here points into a block. + var lines = new List<(sbyte Priority, string Source, string Text, uint EventTime)>(); + int blocks = 0; + + var options = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["send_logs_level"] = "trace" }, + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnLog = block => + { + blocks++; + + // Named columns, so the order on the wire does not matter. The span is borrowed; the strings + // an IColumn hands back are already copies. + ReadOnlySpan priority = block.Column("priority").Values; + ReadOnlySpan eventTime = block.Column("event_time").Values; + IColumn source = block.Column("source"); + IColumn text = block.Column("text"); + + for (int row = 0; row < block.RowCount; row++) + { + lines.Add((priority[row], source[row], text[row], eventTime[row])); + } + }, + }, + }; + + _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(200000)", options); + + Console.WriteLine($" {blocks} log blocks, {lines.Count} lines, all of them copied out before the blocks went back:\n"); + foreach ((sbyte priority, string source, string text, uint _) in lines.Take(8)) + { + Console.WriteLine($" {priority} {source,-22} {(text.Length <= 78 ? text : text[..78] + "...")}"); + } + + if (lines.Count > 8) + { + Console.WriteLine($" ... and {lines.Count - 8} more"); + } + + Console.WriteLine(); + Console.WriteLine(" The columns are event_time, event_time_microseconds, host_name, query_id, thread_id,"); + Console.WriteLine(" priority, source and text. event_time is a DateTime column, which on this tier is the"); + Console.WriteLine($" integer the wire carried — {lines[0].EventTime} whole Unix seconds, so"); + Console.WriteLine($" DateTimeOffset.FromUnixTimeSeconds gives {DateTimeOffset.FromUnixTimeSeconds(lines[0].EventTime):HH:mm:ss} UTC, with the sub-second part in the"); + Console.WriteLine(" microseconds column beside it."); + Console.WriteLine(); + Console.WriteLine(" These are the same lines the server writes to its own log, so this is how a client gets"); + Console.WriteLine(" the server's account of one query without access to the server's log file — which is what"); + Console.WriteLine(" makes it useful when a query is slow on someone else's cluster."); + } + + private static async Task BridgingThemIntoALogger(ClickHouseTcpClient client) + { + Console.WriteLine("\n2. priority is a Poco severity, so a lower number is more severe\n"); + Console.WriteLine(" 1 fatal 2 critical 3 error 4 warning 5 notice"); + Console.WriteLine(" 6 information 7 debug 8 trace 9 test\n"); + + var seen = new SortedDictionary(); + + var options = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["send_logs_level"] = "trace" }, + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnLog = block => + { + ReadOnlySpan priority = block.Column("priority").Values; + IColumn source = block.Column("source"); + + for (int row = 0; row < block.RowCount; row++) + { + seen.TryGetValue(priority[row], out (int Count, string Example) soFar); + seen[priority[row]] = (soFar.Count + 1, source[row]); + } + }, + }, + }; + + _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(200000)", options); + + Console.WriteLine(" What this query reported, and where each line would go in an ILogger:\n"); + foreach ((sbyte priority, (int count, string example)) in seen) + { + Console.WriteLine($" priority {priority} {count,2} line(s) ILogger level {ToLogLevel(priority),-11} e.g. from {example}"); + } + + Console.WriteLine(); + Console.WriteLine(" So filter with <=, and treat anything outside 1..9 as unknown rather than as severe. A"); + Console.WriteLine(" query that runs cleanly says nothing above debug, which is why raising send_logs_level to"); + Console.WriteLine(" warning is a way to be told only about the queries that had a problem."); + Console.WriteLine(); + Console.WriteLine(" Forwarding these to an ILogger is a few lines and yours to write: the client logs its own"); + Console.WriteLine(" lifecycle only (Tcp_026) and never what the server says."); + } + + private static async Task HowMuchEachLevelSays(ClickHouseTcpClient client) + { + Console.WriteLine("\n3. What each send_logs_level costs\n"); + + foreach (string level in new[] { "none", "warning", "information", "debug", "trace" }) + { + int blocks = 0; + int rows = 0; + + _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(200000)", new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["send_logs_level"] = level }, + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnLog = block => + { + blocks++; + rows += block.RowCount; + }, + }, + }); + + Console.WriteLine($" send_logs_level = {level,-12} {blocks} block(s), {rows,2} line(s)"); + } + + Console.WriteLine(); + Console.WriteLine(" The lines are packets on the same connection as the result, so they are not free: text"); + Console.WriteLine(" the server would otherwise only write to its own log crosses the wire. trace on a busy"); + Console.WriteLine(" client is a lot of it. debug on the queries you are investigating is the usable setting,"); + Console.WriteLine(" and it can be set per query rather than on the client (Tcp_020)."); + } + + private static async Task TheTotalsRow(ClickHouseTcpClient client) + { + Console.WriteLine("\n4. OnTotals: the WITH TOTALS row, in the query's own shape\n"); + + const string sql = + "SELECT number % 3 AS bucket, count() AS rows, sum(number) AS total " + + "FROM numbers(30) GROUP BY bucket WITH TOTALS ORDER BY bucket"; + + string[] names = []; + object?[] totals = []; + int calls = 0; + + var options = new ClickHouseTcpQueryOptions + { + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnTotals = block => + { + calls++; + + // ColumnNames is computed and owned, so it is safe to keep; the columns are not. One row, and + // every column here is a scalar, so GetValue boxes a copy of the value rather than a view of + // the buffer. A composite column would need materializing on purpose. + names = [.. block.ColumnNames]; + totals = [.. block.Columns.Select(column => column.GetValue(0))]; + }, + }, + }; + + Console.WriteLine($" {sql}\n"); + await foreach (object[] row in client.QueryAsync(sql, options)) + { + Console.WriteLine($" row {string.Join(" ", row.Select(v => $"{v,8}"))}"); + } + + Console.WriteLine($" names {string.Join(" ", names.Select(n => $"{n,8}"))}"); + Console.WriteLine($" totals {string.Join(" ", totals.Select(v => $"{v,8}"))}"); + Console.WriteLine(); + Console.WriteLine($" Called {calls} time, after the last row: the server sends the totals block once the result"); + Console.WriteLine(" is complete. The shape is the query's own, so the aggregate columns hold the totals over"); + Console.WriteLine(" every group, and the grouping key holds a default rather than anything meaningful."); + Console.WriteLine(); + Console.WriteLine(" It arrives on its own packet, not as an extra row, so a caller reading rows never has to"); + Console.WriteLine(" filter it out — which is the difference from reading WITH TOTALS over HTTP in a row-shaped"); + Console.WriteLine(" format."); + } + + private static async Task TheExtremesRows(ClickHouseTcpClient client) + { + Console.WriteLine("\n5. OnExtremes: two rows, the minimum and the maximum\n"); + + const string sql = "SELECT number AS n, toString(number) AS text FROM numbers(1, 12)"; + + var rows = new List(); + string[] names = []; + + var options = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["extremes"] = "1" }, + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnExtremes = block => + { + names = [.. block.ColumnNames]; + for (int row = 0; row < block.RowCount; row++) + { + rows.Add([.. block.Columns.Select(column => column.GetValue(row))]); + } + }, + }, + }; + + int count = 0; + await foreach (object[] row in client.QueryAsync(sql, options)) + { + count++; + } + + Console.WriteLine($" {sql} ({count} rows)\n"); + Console.WriteLine($" {"",-8} {string.Join(" ", names.Select(n => $"{n,6}"))}"); + Console.WriteLine($" {"minimum",-8} {string.Join(" ", rows[0].Select(v => $"{v,6}"))}"); + Console.WriteLine($" {"maximum",-8} {string.Join(" ", rows[1].Select(v => $"{v,6}"))}"); + Console.WriteLine(); + Console.WriteLine(" Row 0 is the minimum and row 1 the maximum, per column and independently, so the pair is"); + Console.WriteLine(" not two rows of the result. Each column is compared in its own type's order, which for"); + Console.WriteLine($" the String column above is lexicographic — hence \"{rows[1][1]}\" as the maximum of 1..12."); + } + + private static async Task NothingFiresWhenThereIsNothingToSend(ClickHouseTcpClient client) + { + Console.WriteLine("\n6. A callback that never fires\n"); + + int log = 0; + int totals = 0; + int extremes = 0; + + var callbacks = new ClickHouseTcpQueryCallbacks + { + OnLog = _ => log++, + OnTotals = _ => totals++, + OnExtremes = _ => extremes++, + }; + + // Nothing turned on: no send_logs_level, no WITH TOTALS, no extremes. + await foreach (object[] row in client.QueryAsync( + "SELECT number FROM numbers(5)", new ClickHouseTcpQueryOptions { Callbacks = callbacks })) + { + } + + Console.WriteLine($" A plain query with all three set: OnLog {log}, OnTotals {totals}, OnExtremes {extremes} calls."); + + // All three turned on at once, on one query. + await foreach (object[] row in client.QueryAsync( + "SELECT number % 2 AS bucket, count() AS rows FROM numbers(20) GROUP BY bucket WITH TOTALS ORDER BY bucket", + new ClickHouseTcpQueryOptions + { + Settings = new Dictionary + { + ["send_logs_level"] = "debug", + ["extremes"] = "1", + }, + Callbacks = callbacks, + })) + { + } + + Console.WriteLine($" The same callbacks on a query with all three on: OnLog {log}, OnTotals {totals}, OnExtremes {extremes} calls."); + Console.WriteLine(); + Console.WriteLine(" There is no \"none arrived\" reading to look for, because there is no block to hand over,"); + Console.WriteLine(" so a caller that needs to know whether totals came keeps its own flag or counter — the"); + Console.WriteLine(" same shape Tcp_021 uses to show OnProfileInfo is called exactly once."); + } + + /// + /// Maps a server log line's Poco severity onto an level. Unknown numbers become + /// rather than something alarming. + /// + private static LogLevel ToLogLevel(sbyte priority) => priority switch + { + 1 => LogLevel.Critical, + 2 => LogLevel.Critical, + 3 => LogLevel.Error, + 4 => LogLevel.Warning, + 5 or 6 => LogLevel.Information, + 7 => LogLevel.Debug, + 8 or 9 => LogLevel.Trace, + _ => LogLevel.Information, + }; +} diff --git a/examples/Tcp/Observability/Tcp_029_HealthChecks.cs b/examples/Tcp/Observability/Tcp_029_HealthChecks.cs new file mode 100644 index 000000000..5e74a35d4 --- /dev/null +++ b/examples/Tcp/Observability/Tcp_029_HealthChecks.cs @@ -0,0 +1,225 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Driver.Examples; + +/// +/// PingAsync as a health check, wired into Microsoft.Extensions.Diagnostics.HealthChecks over an +/// AddClickHouseTcpDataSource registration. +/// +/// +/// A Ping is a protocol packet pair, not a statement: the server answers Pong and nothing is parsed, planned, +/// executed or written to system.query_log. That makes it the cheapest liveness probe this transport has — +/// the numbers are below — and it is the reason a native health check does not look like the HTTP one, which has +/// to send SELECT 1. +/// +/// +/// +/// Cheap also means narrow. A Pong says the socket reaches a ClickHouse that is still talking; it does not say a +/// query will succeed. Section 4 is the list of what it does and does not prove, which matters because a probe +/// that answers the wrong question is worse than none. +/// +/// +public static class TcpHealthChecks +{ + public static async Task Run() + { + await WhatAPingCosts(); + await ThreeEndpointsOneReport(); + WhatAPongProves(); + } + + private static async Task WhatAPingCosts() + { + Console.WriteLine("1. What a Ping costs against what SELECT 1 costs\n"); + + await using var client = ExampleConfig.CreateTcpClient(); + + // Both warmed up first, so neither measurement pays for the handshake. + await client.PingAsync(); + _ = await client.ExecuteScalarAsync("SELECT 1"); + + const int rounds = 50; + + var clock = Stopwatch.StartNew(); + for (int i = 0; i < rounds; i++) + { + await client.PingAsync(); + } + + double pings = clock.Elapsed.TotalMilliseconds; + + clock.Restart(); + for (int i = 0; i < rounds; i++) + { + _ = await client.ExecuteScalarAsync("SELECT 1"); + } + + double selects = clock.Elapsed.TotalMilliseconds; + + Console.WriteLine($" {$"{rounds} × PingAsync()",-38} {pings,7:0.0} ms {pings / rounds,5:0.00} ms each"); + Console.WriteLine($" {$"{rounds} × ExecuteScalarAsync(\"SELECT 1\")",-38} {selects,7:0.0} ms {selects / rounds,5:0.00} ms each"); + Console.WriteLine(); + Console.WriteLine(" Against a loopback server the difference is almost all server-side work, because the"); + Console.WriteLine(" round trip costs the same either way: SELECT 1 is parsed, planned, executed, and recorded"); + Console.WriteLine(" in system.query_log, and a Ping is one packet answered by one packet. Across a real"); + Console.WriteLine(" network the round trip dominates both and the gap narrows — the reason to prefer the Ping"); + Console.WriteLine(" is then that a probe every few seconds from every instance leaves no trace in the query"); + Console.WriteLine(" log to read past."); + Console.WriteLine(); + Console.WriteLine(" A Ping still needs a pool connection, so the first one after a cold start pays for a dial"); + Console.WriteLine(" and a handshake like any other operation."); + } + + private static async Task ThreeEndpointsOneReport() + { + Console.WriteLine("\n2. Registered as a health check, over three endpoints\n"); + + var services = new ServiceCollection(); + services.AddLogging(logging => logging.SetMinimumLevel(LogLevel.None)); + + // The endpoint that works, registered without a key, exactly as Tcp_003 does. + services.AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString); + + // One that nothing is listening on, so its dial is refused at once. + services.AddClickHouseTcpDataSource( + ExampleConfig.TcpBuilder().ToOptions() with { Port = 1, DialTimeout = TimeSpan.FromSeconds(2) }, + serviceKey: "unreachable"); + + // One whose pool holds a single connection, which a session below will pin — a healthy server that this + // process cannot reach a connection to. + services.AddClickHouseTcpDataSource( + ExampleConfig.TcpBuilder().ToOptions() with + { + MaxPoolSize = 1, + PoolTimeout = TimeSpan.FromMilliseconds(200), + }, + serviceKey: "saturated"); + + services.AddHealthChecks() + .AddClickHouseTcpPing("clickhouse") + .AddClickHouseTcpPing("clickhouse-unreachable", serviceKey: "unreachable") + .AddClickHouseTcpPing("clickhouse-saturated", serviceKey: "saturated"); + + await using ServiceProvider provider = services.BuildServiceProvider(); + + // Hold the saturated pool's only connection for the duration of the report. + var saturated = provider.GetRequiredKeyedService("saturated"); + await using IClickHouseTcpSession pinned = await saturated.OpenSessionAsync(); + + var health = provider.GetRequiredService(); + HealthReport report = await health.CheckHealthAsync(); + + Console.WriteLine($" Overall: {report.Status}, in {report.TotalDuration.TotalMilliseconds:0} ms\n"); + foreach ((string name, HealthReportEntry entry) in report.Entries.OrderBy(e => e.Key, StringComparer.Ordinal)) + { + Console.WriteLine($" {name,-24} {entry.Status,-9} {entry.Duration.TotalMilliseconds,6:0} ms"); + Console.WriteLine($" {Trim(entry.Description)}"); + foreach (KeyValuePair item in entry.Data) + { + Console.WriteLine($" {item.Key,-10} {item.Value}"); + } + } + + Console.WriteLine(); + Console.WriteLine(" Three states, and the middle one is the point: a pool with nothing free throws"); + Console.WriteLine(" TimeoutException, which says nothing about the server, so reporting it as Unhealthy would"); + Console.WriteLine(" take an instance out of rotation for being busy. Degraded is the honest answer. The"); + Console.WriteLine(" distinction has to be made on the exception type, because there is no status on the"); + Console.WriteLine(" client to ask."); + Console.WriteLine(); + Console.WriteLine(" Every registration resolves the client the data source owns, so the check runs on the"); + Console.WriteLine(" application's own pool and measures the path a request would take. That is also why the"); + Console.WriteLine(" check must never dispose it (Tcp_003): the pool is shared, and the container owns it."); + Console.WriteLine(); + Console.WriteLine(" In ASP.NET Core the rest is MapHealthChecks(\"/health\") plus, if you want the endpoints"); + Console.WriteLine(" split, a predicate over the tags each registration carries."); + } + + private static void WhatAPongProves() + { + Console.WriteLine("\n3. What a Pong does and does not prove\n"); + Console.WriteLine(" It proves:"); + Console.WriteLine(" - a connection to the endpoint exists, or could be opened inside DialTimeout;"); + Console.WriteLine(" - the process answering it speaks the native protocol;"); + Console.WriteLine(" - if the pool had to dial, that the credentials were accepted — the handshake"); + Console.WriteLine(" authenticates, so a wrong password fails there rather than at the Ping;"); + Console.WriteLine(" - the server is not so stalled that it cannot answer a packet."); + Console.WriteLine(); + Console.WriteLine(" It does not prove:"); + Console.WriteLine(" - that a query will succeed. No table is read, no permission is checked, no memory or"); + Console.WriteLine(" concurrency limit is tested, and a server refusing queries still Pongs;"); + Console.WriteLine(" - that the database the client is configured for exists;"); + Console.WriteLine(" - that replication is caught up, or that any part of a cluster beyond this one node is"); + Console.WriteLine(" reachable;"); + Console.WriteLine(" - anything at all about credentials on a warm pool, where the Ping travels over a"); + Console.WriteLine(" connection whose handshake happened minutes ago."); + Console.WriteLine(); + Console.WriteLine(" So a Ping is a liveness probe. For readiness — should this instance take traffic — pick a"); + Console.WriteLine(" statement that touches what the service actually needs (SELECT 1, or a count against one"); + Console.WriteLine(" table), accept that it costs a query, and run it far less often."); + } + + private static string Trim(string? description) + => description is null ? "(no description)" + : description.Length <= 100 ? description + : description[..100] + "..."; + + /// + /// Registers a health check that Pings whichever registration names. Kept in + /// this example rather than shipped by the driver, so that the mapping from exception to + /// stays the application's decision. + /// + /// The health check builder. + /// The name the entry appears under in the report. + /// The keyed registration to check, or null for the unkeyed one. + /// The builder, for chaining. + private static IHealthChecksBuilder AddClickHouseTcpPing( + this IHealthChecksBuilder builder, + string name, + string? serviceKey = null) + => builder.Add(new HealthCheckRegistration( + name, + provider => new TcpPingHealthCheck(serviceKey is null + ? provider.GetRequiredService() + : provider.GetRequiredKeyedService(serviceKey)), + failureStatus: HealthStatus.Unhealthy, + tags: ["clickhouse", "native"])); + + /// + /// Takes the interface rather than the concrete client, and never disposes it: the container owns the pool. + /// + private sealed class TcpPingHealthCheck(IClickHouseTcpClient client) : IHealthCheck + { + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + var clock = Stopwatch.StartNew(); + + // ToString() on the options is the only public rendering that resolves the port, and it is written to + // be safe to log — it leaves the password out. + var data = new Dictionary { ["endpoint"] = client.Options.ToString() }; + + try + { + await client.PingAsync(cancellationToken); + data["pong_ms"] = Math.Round(clock.Elapsed.TotalMilliseconds, 1); + return HealthCheckResult.Healthy("Pong", data); + } + catch (TimeoutException e) + { + // No connection came free within PoolTimeout. The server has not been reached, so this is a + // statement about this process, not about ClickHouse. + return HealthCheckResult.Degraded("No pooled connection was free", e, data); + } + catch (Exception e) when (e is not OperationCanceledException) + { + return new HealthCheckResult(context.Registration.FailureStatus, e.Message, e, data); + } + } + } +} diff --git a/examples/Tcp/Observability/Tcp_030_Testcontainers.cs b/examples/Tcp/Observability/Tcp_030_Testcontainers.cs new file mode 100644 index 000000000..1f735c2a4 --- /dev/null +++ b/examples/Tcp/Observability/Tcp_030_Testcontainers.cs @@ -0,0 +1,141 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; +using DotNet.Testcontainers.Builders; +using Testcontainers.ClickHouse; + +namespace ClickHouse.Driver.Examples; + +/// +/// Running the native client against a throwaway ClickHouse from Testcontainers. The HTTP counterpart is +/// Testing_001_Testcontainers; everything here is about the one difference that matters, which is the port. +/// +/// +/// The container publishes 8123 and 9000 on two random host ports. GetConnectionString() describes the +/// HTTP one, so a native client cannot use it: take the native port from +/// GetMappedPublicPort(9000) and build the connection string yourself. +/// +/// +/// +/// Readiness is the second difference. The ClickHouse module's own wait strategy probes the HTTP interface, and +/// WithWaitStrategy replaces it rather than adding to it — so the strategy below asks for the HTTP +/// probe back and then also waits on 9000. A port check alone is not enough: measured on this image, a wait on +/// 9000 by itself reported ready about three seconds before the server would complete a native handshake, because +/// the port is bound before it is served. +/// +/// +/// +/// Both probes together are still not proof. On a busy machine the first native handshake can be refused after +/// each one has passed, because neither tests the native protocol — one tests the HTTP listener and the other +/// tests only that the port is bound. So the client waits on a handshake instead, which is the one check that +/// succeeds exactly when the client can work. Forced with a no-condition wait strategy, that took 43 attempts +/// over 4.4 seconds. +/// +/// +/// +/// This example starts its own server, so it is one of the few that does not take its endpoint from +/// ExampleConfig. It needs a working Docker daemon and will not run on a macOS runner. +/// +/// +public static class TcpTestcontainers +{ + /// Pinned, so the example tests one known server rather than whatever latest is today. + private const string Image = "clickhouse/clickhouse-server:25.12-alpine"; + + private const ushort NativePort = 9000; + private const ushort HttpPort = 8123; + + // The module creates this user from environment variables at startup. There is no GetUsername()/GetPassword() + // on the container, so setting them here is also how the test gets to know them. + private const string User = "example"; + private const string Password = "example"; + + public static async Task Run() + { + Console.WriteLine($"Starting {Image}. First run pulls the image, which takes a while.\n"); + + await using ClickHouseContainer container = new ClickHouseBuilder(Image) + .WithUsername(User) + .WithPassword(Password) + + // Both probes: the HTTP one is what says the server is really up, and the native one is what says the + // port this client dials is bound. Testcontainers polls both; nothing here sleeps. + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilHttpRequestIsSucceeded(request => request.ForPath("/ping").ForPort(HttpPort)) + .UntilInternalTcpPortIsAvailable(NativePort)) + .Build(); + + await container.StartAsync(); + + Console.WriteLine($" Container started. GetConnectionString() describes the HTTP interface only:"); + Console.WriteLine($" {container.GetConnectionString()}"); + + // The native endpoint, assembled from the mapped port. ClickHouseTcpConnectionStringBuilder rather than a + // literal, so the escaping of anything in the password is not this example's problem. + var builder = new ClickHouseTcpConnectionStringBuilder + { + Host = container.Hostname, + Port = container.GetMappedPublicPort(NativePort), + Username = User, + Password = Password, + Database = "default", + }; + + Console.WriteLine($"\n Native endpoint, from GetMappedPublicPort({NativePort}):"); + Console.WriteLine($" Host={builder.Host};Port={builder.Port};Username={builder.Username};Database={builder.Database}"); + Console.WriteLine($" HTTP is on the other mapped port, {container.GetMappedPublicPort(HttpPort)} — the two are separate listeners."); + + await using var client = new ClickHouseTcpClient(builder.ToOptions()); + + // No wait strategy can prove the native protocol is accepting. Both probes above test something adjacent: + // /ping answers on the HTTP listener, and UntilInternalTcpPortIsAvailable only says the port is bound. + // A bound port is not an accepting server, so on a loaded machine the first handshake can still be + // refused. Waiting on a handshake is the only check that tests the thing being waited for. + ClickHouseTcpServerInfo info = await HandshakeWhenReady(client); + Console.WriteLine($"\n Handshaken: {info}, protocol revision {info.ProtocolRevision}, timezone {info.Timezone}"); + Console.WriteLine($" currentUser(): {await client.ExecuteScalarAsync("SELECT currentUser()")}"); + + await client.ExecuteAsync("CREATE TABLE probe (id UInt64, note String) ENGINE = MergeTree ORDER BY id"); + await client.InsertRowsAsync("INSERT INTO probe (id, note) VALUES", [[1UL, "from the native client"]]); + Console.WriteLine($" Inserted and read back: {await client.ExecuteScalarAsync("SELECT note FROM probe WHERE id = 1")}"); + + // No DROP TABLE: the container goes with the example, and so does everything in it. That is the whole + // point of a throwaway server, and it is why a test suite built on one needs no cleanup between tests + // beyond what the container's lifetime gives it. + Console.WriteLine("\n Nothing is dropped: DisposeAsync removes the container and the table with it."); + Console.WriteLine(" In a test project the container is started once for the run (an NUnit [OneTimeSetUp],"); + Console.WriteLine(" an xUnit fixture) and shared, because a handshake is cheap and a container start is not."); + } + + /// + /// Handshakes as soon as the server will, retrying while the native listener refuses the connection. + /// This is the readiness check a test fixture wants: it succeeds exactly when the client can work. + /// + private static async Task HandshakeWhenReady(ClickHouseTcpClient client) + { + var deadline = TimeSpan.FromSeconds(30); + var started = Stopwatch.StartNew(); + + for (int attempt = 1; ; attempt++) + { + try + { + ClickHouseTcpServerInfo info = await client.GetServerInfoAsync(); + + if (attempt > 1) + { + Console.WriteLine($"\n The first handshake was refused: it took {attempt} attempts over " + + $"{started.ElapsedMilliseconds} ms before the native listener accepted one, " + + "even though both wait strategies had already passed."); + } + + return info; + } + catch (ClickHouseTcpTransportException) when (started.Elapsed < deadline) + { + // Only a transport failure is worth retrying: the listener is not accepting yet. A server + // exception would mean it answered and rejected us, which no amount of waiting fixes. + await Task.Delay(100); + } + } + } +} diff --git a/examples/Tcp/README.md b/examples/Tcp/README.md index fd4a0b270..e1455652b 100644 --- a/examples/Tcp/README.md +++ b/examples/Tcp/README.md @@ -70,3 +70,6 @@ scale the column type declared. `ClickHouseTcpQueryCallbacks`, rather than as headers after the fact. - **Block compression** on the wire, LZ4 by default. - **Bit-plane access to `QBit` columns**, through `IQBitColumn`. +- **W3C trace context propagation.** The client sends the current `Activity`'s trace and span ids with + each query, so the spans the server records in `system.opentelemetry_span_log` join the same trace as + the caller's. The HTTP transport sends no `traceparent`. From c8313cedba7a460b8a3b67ac99dd80d27dd17e96 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 19:08:00 +0200 Subject: [PATCH 12/16] Act on the examples review Runner and configuration: - A whole-string CLICKHOUSE_{HTTP,TCP}_CONNECTION_STRING override now reaches ExampleConfig.HttpBuilder()/TcpBuilder(), which rebuilt from the component variables and so pointed an example at a different server than preflight checked. Host/HttpPort/TcpPort are private; HttpEndpoint/TcpEndpoint answer for the effective endpoint. - --http and --tcp no longer include the three examples that need a cluster, Cloud credentials or a token. --list still shows them, marked. - A transport flag now narrows --filter instead of being ignored. - The two Testcontainers examples start their own server, so preflight no longer holds them up on the configured endpoint. - TcpOpenTelemetry queries both interfaces, so it is registered as cross-transport and both endpoints are checked. Native examples: - Gate QBit (Tcp_015) and Geometry (Tcp_013) on 25.11, and QBit on the HTTP side too: both fail outright on 25.8, the floor of the support matrix. - Every fixed-name table is dropped before it is created, so a run interrupted before its finally does not break the next one. - Drop the trailing break from 42 single-block loops. Stopping a StreamAsync enumeration early makes the client cancel the query and discard the connection, which is not the pattern to teach. - Tcp_012 pins session_timezone=UTC in the comparison that reads a Kind=Unspecified DateTime, which otherwise means something different per server. - Tcp_017's concurrency marker and the deliberately-missing table names in Tcp_026 and Tcp_027 are unique per run. - Tcp_020 polls its own table instead of issuing SYSTEM FLUSH ASYNC INSERT QUEUE, which flushes every client's pending inserts. - WaitUntilRunning throws instead of returning after its last poll, and Tcp_023 reports retry exhaustion instead of throwing out of the example. - Tcp_016 creates its role and user inside the try that drops them. - Tcp_030 retries only a socket failure; the same exception type also carries TLS and DNS failures. Corrected claims: values in an object[] row are boxed only when the column is a value type; MaxRowsPerBlock is block geometry and MaxSendBufferBytes is the memory bound; toString of a NULL is NULL, not an empty string; BFloat16, IPv4, IPv6 and String are CLR surfaces rather than the wire bytes; a parameterized query below revision 54459 is refused by the client before it is sent; a TimeoutException does not say whether the pool, the dial or the read timed out; a callback that copies values can still throw; GetServerInfoAsync describes the connection it borrowed. Also fixes three HTTP examples that built settings from Host = "localhost" instead of ExampleConfig. Co-Authored-By: Claude Opus 5 (1M context) --- examples/AGENTS.md | 26 +++-- examples/ExampleConfig.cs | 100 +++++++++++------- examples/ExamplePreflight.cs | 15 ++- examples/ExampleRunner.cs | 55 ++++++++-- .../Advanced/Advanced_002_SessionIdUsage.cs | 3 +- .../Http/Advanced/Advanced_011_Compression.cs | 3 +- .../Core/Core_004_HttpClientConfiguration.cs | 6 +- .../Vector_001_QBitSimilaritySearch.cs | 12 +++ examples/Program.cs | 8 +- examples/README.md | 13 ++- .../Advanced/Tcp_020_SettingsAndQueryId.cs | 31 +++++- .../Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs | 23 +++- examples/Tcp/Advanced/Tcp_024_Compression.cs | 4 +- examples/Tcp/Advanced/Tcp_025_ServerInfo.cs | 13 ++- examples/Tcp/Connection/Tcp_016_Sessions.cs | 16 +-- examples/Tcp/Connection/Tcp_017_PoolTuning.cs | 4 +- examples/Tcp/Connection/Tcp_019_Timeouts.cs | 4 +- examples/Tcp/Core/Tcp_001_BasicUsage.cs | 12 ++- examples/Tcp/Core/Tcp_002_ConnectionString.cs | 8 +- .../Tcp/Core/Tcp_004_MigratingFromHttp.cs | 1 + examples/Tcp/Observability/Tcp_026_Logging.cs | 4 +- .../Observability/Tcp_027_OpenTelemetry.cs | 4 +- .../Observability/Tcp_028_MetadataBlocks.cs | 5 +- .../Tcp/Observability/Tcp_029_HealthChecks.cs | 17 +-- .../Observability/Tcp_030_Testcontainers.cs | 9 +- examples/Tcp/Read/Tcp_005_ReadTiers.cs | 9 +- examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs | 10 +- examples/Tcp/Read/Tcp_007_Parameters.cs | 6 +- examples/Tcp/Read/Tcp_008_Poco.cs | 1 + examples/Tcp/Types/Tcp_011_ScalarTypes.cs | 33 +++--- .../Tcp/Types/Tcp_012_DateTimeAndTimezones.cs | 25 +++-- examples/Tcp/Types/Tcp_013_CompositeRead.cs | 35 +++--- .../Tcp/Types/Tcp_014_VariantDynamicJson.cs | 24 ++--- .../Tcp/Types/Tcp_015_QBitVectorSearch.cs | 23 ++-- examples/Tcp/Write/Tcp_009_ColumnarInsert.cs | 15 ++- examples/Tcp/Write/Tcp_010_CompositeWrites.cs | 15 ++- 36 files changed, 372 insertions(+), 220 deletions(-) diff --git a/examples/AGENTS.md b/examples/AGENTS.md index be3fe7071..bf62da49a 100644 --- a/examples/AGENTS.md +++ b/examples/AGENTS.md @@ -49,10 +49,14 @@ Take the server from `ExampleConfig`, which resolves environment variables over those would produce a duplicate. - `ExampleConfig.HttpBuilder()` to *change* one of those five. It returns a fresh builder each call. -Three examples are exempt, because configuration is their subject or they start their own server: +Four examples are exempt, because configuration is their subject or they start their own server: `Core_002_ConnectionStringConfiguration`, `Core_003_DependencyInjection`, -`Testing_001_Testcontainers`. A literal connection string inside a comment, shown to teach the -reader what one looks like, is also fine. +`Testing_001_Testcontainers`, `Tcp_030_Testcontainers`. A literal connection string inside a comment, +shown to teach the reader what one looks like, is also fine. + +Ask `ExampleConfig` for the endpoint with `HttpEndpoint` or `TcpEndpoint` when an example has to name +or dial it itself. There are no `Host`/`Port` properties: a whole-string override would not reach +them, so an example reading one could print an endpoint it did not connect to. ## Examples deliberately left out of `RunAllExamples` @@ -63,13 +67,23 @@ by explicit filter: - `Tables_003_CreateTableCloud` — needs ClickHouse Cloud credentials. - `Auth_001_JwtAuthentication` — needs a JWT. -If you add an example of that kind, leave it out of `RunAllExamples` and list it here. Otherwise the -omission is indistinguishable from having forgotten step 4. +Their class names are also in `ExampleRunner._optIn`, which is what keeps `--http` and `--tcp` from +running them: those select by reflection, so leaving an example out of `RunAllExamples` does not on +its own keep a transport run from picking it up. + +If you add an example of that kind, leave it out of `RunAllExamples`, add its class name to +`_optIn`, and list it here. Otherwise the omission is indistinguishable from having forgotten step 4. ## Style - Console output is the teaching surface. Print what you did and what came back, not just "OK". - Show one thing well rather than covering an API exhaustively. A reader who wants the full surface reads the docs. -- Drop any table you create. +- Drop any table you create, in a `finally`, and `DROP TABLE IF EXISTS` it before the `CREATE` as well. The + second one is what lets a run survive an earlier run that was interrupted before its `finally`. +- Name a table `example_`, fixed, not unique per run. This project is not the test suites: it assumes + one suite run at a time against a server, so it does not need `CreateTableName`. Two things do need a + per-run `Guid`: a name the example needs the server *not* to have (a deliberately missing table), and a + marker an example counts in `system.processes` or `system.query_log`. A second run would otherwise make + the first one's measurement or expected failure wrong. - Comments explain why the server or the driver behaves as it does, not what the line does. diff --git a/examples/ExampleConfig.cs b/examples/ExampleConfig.cs index 785b786cc..2dbf0063d 100644 --- a/examples/ExampleConfig.cs +++ b/examples/ExampleConfig.cs @@ -22,70 +22,76 @@ namespace ClickHouse.Driver.Examples; /// CLICKHOUSE_DATABASEdefault default /// /// -/// CLICKHOUSE_HTTP_CONNECTION_STRING replaces the whole assembled string, for a server the -/// pieces above cannot describe — TLS, a cloud endpoint, an extra setting. +/// CLICKHOUSE_HTTP_CONNECTION_STRING and CLICKHOUSE_TCP_CONNECTION_STRING replace the +/// whole assembled string for their transport, for a server the pieces above cannot describe — TLS, a +/// cloud endpoint, an extra setting. Either one also becomes what and +/// return, so an example that changes one key still starts from the override. /// /// -/// Three examples deliberately do not use this: Core_002_ConnectionStringConfiguration and +/// Four examples deliberately do not use this: Core_002_ConnectionStringConfiguration and /// Core_003_DependencyInjection, whose subject is configuration itself, and -/// Testing_001_Testcontainers, which starts its own server. +/// Testing_001_Testcontainers and Tcp_030_Testcontainers, which start their own servers. /// /// public static class ExampleConfig { - /// The server host name or address. - public static string Host { get; } = Env("CLICKHOUSE_HOST") ?? "localhost"; - - /// The HTTP interface port. - public static ushort HttpPort { get; } = ushort.Parse(Env("CLICKHOUSE_HTTP_PORT") ?? "8123"); - - /// The native protocol port. Not interchangeable with . - public static ushort TcpPort { get; } = ushort.Parse(Env("CLICKHOUSE_TCP_PORT") ?? "9000"); - - /// The user to authenticate as. - public static string Username { get; } = Env("CLICKHOUSE_USER") ?? "default"; - - /// The password, empty for a server with no password set. - public static string Password { get; } = Env("CLICKHOUSE_PASSWORD") ?? string.Empty; - - /// The default database for queries. - public static string Database { get; } = Env("CLICKHOUSE_DATABASE") ?? "default"; + // Private, and read only by the two component builders below. A whole-string override does not + // reach them, so an example that asked one of these for the endpoint would print, proxy or dial + // somewhere other than where it connected. HttpEndpoint and TcpEndpoint answer that question. + private static readonly string Host = Env("CLICKHOUSE_HOST") ?? "localhost"; + private static readonly ushort HttpPort = ushort.Parse(Env("CLICKHOUSE_HTTP_PORT") ?? "8123"); + private static readonly ushort TcpPort = ushort.Parse(Env("CLICKHOUSE_TCP_PORT") ?? "9000"); + private static readonly string Username = Env("CLICKHOUSE_USER") ?? "default"; + private static readonly string Password = Env("CLICKHOUSE_PASSWORD") ?? string.Empty; + private static readonly string Database = Env("CLICKHOUSE_DATABASE") ?? "default"; /// The connection string for the HTTP transport. public static string HttpConnectionString { get; } = - Env("CLICKHOUSE_HTTP_CONNECTION_STRING") ?? HttpBuilder().ConnectionString; + Env("CLICKHOUSE_HTTP_CONNECTION_STRING") ?? FromComponents().ConnectionString; /// The connection string for the native protocol. public static string TcpConnectionString { get; } = - Env("CLICKHOUSE_TCP_CONNECTION_STRING") ?? TcpBuilder().ToString(); + Env("CLICKHOUSE_TCP_CONNECTION_STRING") ?? TcpFromComponents().ToString(); /// /// A builder pre-filled with the configured endpoint and credentials, for an example that has to /// change one key. Each call returns a fresh builder. /// /// A builder describing the configured HTTP endpoint. - public static ClickHouseConnectionStringBuilder HttpBuilder() => new() - { - Host = Host, - Port = HttpPort, - Username = Username, - Password = Password, - Database = Database, - }; + public static ClickHouseConnectionStringBuilder HttpBuilder() => new(HttpConnectionString); /// /// A builder pre-filled with the configured endpoint and credentials for the native protocol, for /// an example that has to change one key. Each call returns a fresh builder. /// /// A builder describing the configured native endpoint. - public static ClickHouseTcpConnectionStringBuilder TcpBuilder() => new() + public static ClickHouseTcpConnectionStringBuilder TcpBuilder() => new(TcpConnectionString); + + /// + /// The host and port the HTTP examples reach, for an example that has to name or dial the endpoint + /// itself rather than hand a connection string to a client. + /// + public static (string Host, ushort Port) HttpEndpoint { - Host = Host, - Port = TcpPort, - Username = Username, - Password = Password, - Database = Database, - }; + get + { + var builder = HttpBuilder(); + return (builder.Host, builder.Port); + } + } + + /// The host and port the native examples dial. Not interchangeable with . + public static (string Host, int Port) TcpEndpoint + { + get + { + var builder = TcpBuilder(); + + // The builder reports no port when the connection string omits one, which is the native + // default rather than an absence. + return (builder.Host, builder.Port ?? 9000); + } + } /// Creates a client against the configured server. The caller disposes it. /// A client for the configured HTTP endpoint. @@ -102,6 +108,24 @@ public static class ExampleConfig /// A connection for the configured HTTP endpoint. public static ClickHouseConnection CreateHttpConnection() => new(HttpConnectionString); + private static ClickHouseConnectionStringBuilder FromComponents() => new() + { + Host = Host, + Port = HttpPort, + Username = Username, + Password = Password, + Database = Database, + }; + + private static ClickHouseTcpConnectionStringBuilder TcpFromComponents() => new() + { + Host = Host, + Port = TcpPort, + Username = Username, + Password = Password, + Database = Database, + }; + private static string? Env(string name) { var value = Environment.GetEnvironmentVariable(name); diff --git a/examples/ExamplePreflight.cs b/examples/ExamplePreflight.cs index e1913e595..87a37fd11 100644 --- a/examples/ExamplePreflight.cs +++ b/examples/ExamplePreflight.cs @@ -80,18 +80,23 @@ public static async Task CheckAsync(params ExampleTransport[] transports) private static void Report(ExampleTransport transport, string failure) { - var (name, endpoint, port, source) = transport == ExampleTransport.Http - ? ("HTTP interface", $"{ExampleConfig.Host}:{ExampleConfig.HttpPort}", "CLICKHOUSE_HTTP_PORT", "CLICKHOUSE_HTTP_CONNECTION_STRING") - : ("native protocol", $"{ExampleConfig.Host}:{ExampleConfig.TcpPort}", "CLICKHOUSE_TCP_PORT", "CLICKHOUSE_TCP_CONNECTION_STRING"); + // Reported from the effective endpoint, so that a whole-string override is described as itself + // rather than as whatever the component variables say. + var http = ExampleConfig.HttpEndpoint; + var tcp = ExampleConfig.TcpEndpoint; + + var (name, endpoint, user, port, source) = transport == ExampleTransport.Http + ? ("HTTP interface", $"{http.Host}:{http.Port}", ExampleConfig.HttpBuilder().Username, "CLICKHOUSE_HTTP_PORT", "CLICKHOUSE_HTTP_CONNECTION_STRING") + : ("native protocol", $"{tcp.Host}:{tcp.Port}", ExampleConfig.TcpBuilder().Username, "CLICKHOUSE_TCP_PORT", "CLICKHOUSE_TCP_CONNECTION_STRING"); Console.WriteLine(); - Console.WriteLine($"Cannot reach ClickHouse on the {name} at {endpoint} as user '{ExampleConfig.Username}'."); + Console.WriteLine($"Cannot reach ClickHouse on the {name} at {endpoint} as user '{user}'."); Console.WriteLine($" {failure}"); Console.WriteLine(); if (transport == ExampleTransport.Tcp) { - Console.WriteLine($" The native protocol listens on port 9000 by default, not on the HTTP port ({ExampleConfig.HttpPort})."); + Console.WriteLine($" The native protocol listens on port 9000 by default, not on the HTTP port ({http.Port})."); Console.WriteLine(); } diff --git a/examples/ExampleRunner.cs b/examples/ExampleRunner.cs index 3486392ce..7b7502f21 100644 --- a/examples/ExampleRunner.cs +++ b/examples/ExampleRunner.cs @@ -15,6 +15,29 @@ public static class ExampleRunner private static readonly HashSet _crossTransport = new(StringComparer.Ordinal) { "TcpMigratingFromHttp", + "TcpOpenTelemetry", + }; + + /// + /// Examples that start their own server, so the configured endpoint is not theirs and preflight must not + /// hold them up. Declared before _examples for the same reason as _crossTransport. + /// + private static readonly HashSet _selfContained = new(StringComparer.Ordinal) + { + "Testcontainers", + "TcpTestcontainers", + }; + + /// + /// Examples needing infrastructure an ordinary server does not have, so neither an unfiltered run + /// nor a transport run includes them. An explicit --filter still reaches them. Keep this in + /// step with the list in AGENTS.md. + /// + private static readonly HashSet _optIn = new(StringComparer.Ordinal) + { + "CreateTableCluster", + "CreateTableCloud", + "JwtAuthentication", }; private static readonly List _examples = DiscoverExamples(); @@ -42,9 +65,17 @@ public record ExampleInfo(string ClassName, Type Type, MethodInfo RunMethod) /// Every endpoint the example needs to reach, which is not always the one it is filed under: /// an example comparing the two transports needs both. /// - public IReadOnlyList RequiredTransports { get; } = _crossTransport.Contains(ClassName) - ? [ExampleTransport.Http, ExampleTransport.Tcp] - : [ClassName.StartsWith("Tcp", StringComparison.Ordinal) ? ExampleTransport.Tcp : ExampleTransport.Http]; + public IReadOnlyList RequiredTransports { get; } = _selfContained.Contains(ClassName) + ? [] + : _crossTransport.Contains(ClassName) + ? [ExampleTransport.Http, ExampleTransport.Tcp] + : [ClassName.StartsWith("Tcp", StringComparison.Ordinal) ? ExampleTransport.Tcp : ExampleTransport.Http]; + + /// + /// Whether a run that names no example includes it. False for one needing a cluster, Cloud + /// credentials or a token, which only an explicit filter should reach. + /// + public bool RunsByDefault { get; } = !_optIn.Contains(ClassName); } /// @@ -53,22 +84,26 @@ public record ExampleInfo(string ClassName, Type Type, MethodInfo RunMethod) public static IReadOnlyList Examples => _examples; /// - /// Gets the examples that use one transport. + /// Gets the examples that use one transport and that a run naming no example includes. /// /// The transport to select. /// The matching examples, in class-name order. public static List ForTransport(ExampleTransport transport) - => _examples.Where(e => e.Transport == transport).ToList(); + => _examples.Where(e => e.Transport == transport && e.RunsByDefault).ToList(); /// /// Finds examples matching the given filter using fuzzy matching. /// Matches against any substring of the normalized class name. /// - public static List FindMatches(string filter) + /// The pattern to match. + /// The transport to restrict the match to, or null for either. + /// The matching examples, in class-name order. + public static List FindMatches(string filter, ExampleTransport? transport = null) { var normalizedFilter = Normalize(filter); return _examples .Where(e => e.NormalizedName.Contains(normalizedFilter)) + .Where(e => transport is null || e.Transport == transport) .ToList(); } @@ -86,13 +121,17 @@ public static async Task RunExample(ExampleInfo example) /// public static void ListExamples(ExampleTransport? transport = null) { - var listed = transport is { } only ? ForTransport(only) : _examples.ToList(); + // Lists the opt-in examples too, marked. They are what a reader is most likely to be looking + // for by name, since no unfiltered run ever prints them. + var listed = _examples.Where(e => transport is null || e.Transport == transport).ToList(); Console.WriteLine(transport is { } named ? $"Available {named} examples:\n" : "Available examples:\n"); foreach (var example in listed.OrderBy(e => e.ClassName)) { - Console.WriteLine($" - {example.ClassName}"); + Console.WriteLine(example.RunsByDefault + ? $" - {example.ClassName}" + : $" - {example.ClassName} (--filter only: needs a cluster, Cloud, or a token)"); } Console.WriteLine(); diff --git a/examples/Http/Advanced/Advanced_002_SessionIdUsage.cs b/examples/Http/Advanced/Advanced_002_SessionIdUsage.cs index 67a401e0f..1205dde62 100644 --- a/examples/Http/Advanced/Advanced_002_SessionIdUsage.cs +++ b/examples/Http/Advanced/Advanced_002_SessionIdUsage.cs @@ -16,9 +16,8 @@ public static async Task Run() Console.WriteLine("Session ID Usage Examples\n"); // To use temporary tables, you must enable sessions - var settings = new ClickHouseClientSettings + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { - Host = "localhost", UseSession = true, // If you don't set SessionId, a GUID will be automatically generated }; diff --git a/examples/Http/Advanced/Advanced_011_Compression.cs b/examples/Http/Advanced/Advanced_011_Compression.cs index 4e55a713d..f874af1d6 100644 --- a/examples/Http/Advanced/Advanced_011_Compression.cs +++ b/examples/Http/Advanced/Advanced_011_Compression.cs @@ -86,9 +86,8 @@ public static async Task Run() // Using ClickHouseClientSettings Console.WriteLine("3. Via ClickHouseClientSettings:"); - var settings = new ClickHouseClientSettings + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { - Host = "localhost", UseCompression = false // Disable compression }; using (var client = new ClickHouseClient(settings)) diff --git a/examples/Http/Core/Core_004_HttpClientConfiguration.cs b/examples/Http/Core/Core_004_HttpClientConfiguration.cs index b80a9f307..b9a307465 100644 --- a/examples/Http/Core/Core_004_HttpClientConfiguration.cs +++ b/examples/Http/Core/Core_004_HttpClientConfiguration.cs @@ -82,9 +82,8 @@ private static async Task Example1_CustomHttpClient() }; // Pass the HttpClient via settings - var settings = new ClickHouseClientSettings + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { - Host = "localhost", HttpClient = httpClient, }; @@ -207,9 +206,8 @@ private static async Task Example4_HttpClientFactory() // You can create a simple factory implementation var factory = new SimpleHttpClientFactory(); - var settings = new ClickHouseClientSettings + var settings = new ClickHouseClientSettings(ExampleConfig.HttpConnectionString) { - Host = "localhost", HttpClientFactory = factory, HttpClientName = "ClickHouseClient", // Optional: factory can use this name }; diff --git a/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs b/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs index 621d629fe..88a6fc131 100644 --- a/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs +++ b/examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs @@ -15,6 +15,18 @@ public static async Task Run() Console.WriteLine("=== QBit Similarity Search with Different Precision Levels ===\n"); + // QBit arrived in 25.10 with limitations, so this asks for 25.11 as the driver's own suites do. The + // comparison is left to the server, which spares the client parsing a four-part version string. + var supported = Convert.ToBoolean(await client.ExecuteScalarAsync( + "SELECT (toUInt32(splitByChar('.', version())[1]), toUInt32(splitByChar('.', version())[2])) >= (25, 11)")); + + if (!supported) + { + var reported = await client.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($"Skipped: QBit needs ClickHouse 25.11 or newer, and this server is {reported}."); + return; + } + var tableName = "example_qbit_similarity"; await client.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {tableName}"); diff --git a/examples/Program.cs b/examples/Program.cs index 934eadb16..f15a01a9c 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -22,7 +22,7 @@ static async Task Main(string[] args) if (filter != null) { - await RunFiltered(filter, isInteractive); + await RunFiltered(filter, transport, isInteractive); } else if (transport is { } only) { @@ -46,9 +46,11 @@ static async Task Main(string[] args) } } - private static async Task RunFiltered(string filter, bool isInteractive) + private static async Task RunFiltered(string filter, ExampleTransport? transport, bool isInteractive) { - var matches = ExampleRunner.FindMatches(filter); + // A transport flag narrows the filter rather than being ignored: '--tcp --filter basicusage' + // otherwise also selects the HTTP BasicUsage and asks for an endpoint the caller did not name. + var matches = ExampleRunner.FindMatches(filter, transport); if (matches.Count == 0) { diff --git a/examples/README.md b/examples/README.md index 454e74654..39dbb8fd7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -192,9 +192,9 @@ The filter matches the example's class name, which `--list` prints. A class name ### Connection configuration -Every example takes its server from [ExampleConfig.cs](ExampleConfig.cs), so one environment variable -points the whole suite somewhere else. The defaults are what a stock server container exposes on -localhost, and the examples run with nothing set. +Every example takes its server from [ExampleConfig.cs](ExampleConfig.cs), apart from the four listed +below, so one environment variable points the whole suite somewhere else. The defaults are what a +stock server container exposes on localhost, and the examples run with nothing set. | Variable | Default | | --- | --- | @@ -212,9 +212,12 @@ For an endpoint those pieces cannot describe — TLS, a cloud host, an extra set CLICKHOUSE_HOST=my-server CLICKHOUSE_PASSWORD=secret dotnet run -- basicusage ``` -Two examples keep literal connection strings, because configuration is what they teach: +Four examples do not read `ExampleConfig`. Two keep literal connection strings because configuration +is what they teach: [Core_002_ConnectionStringConfiguration.cs](Http/Core/Core_002_ConnectionStringConfiguration.cs) and -[Core_003_DependencyInjection.cs](Http/Core/Core_003_DependencyInjection.cs). +[Core_003_DependencyInjection.cs](Http/Core/Core_003_DependencyInjection.cs). Two start their own +server and address that: [Testing_001_Testcontainers.cs](Http/Testing/Testing_001_Testcontainers.cs) +and [Tcp_030_Testcontainers.cs](Tcp/Observability/Tcp_030_Testcontainers.cs). ### ClickHouse Cloud diff --git a/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs b/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs index 5c7ff11de..7107d3fea 100644 --- a/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs +++ b/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs @@ -242,15 +242,14 @@ private static async Task AsyncInsert(ClickHouseTcpClient client) long notWaited = clock.ElapsedMilliseconds; object immediately = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); - // The only way to make the second batch's visibility deterministic. Without it, the count above is 2 or 4 - // depending on whether the buffer happened to flush, which is exactly the guarantee being given up. - await client.ExecuteAsync("SYSTEM FLUSH ASYNC INSERT QUEUE"); - object afterFlush = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + // Polling this one table, not SYSTEM FLUSH ASYNC INSERT QUEUE: that command flushes every client's + // pending async inserts on the server, so an example must not issue it on a shared one. + long afterFlush = await CountWhenItReaches(client, 4); Console.WriteLine($"\n async_insert=1, wait_for_async_insert=0: returned after {notWaited} ms."); Console.WriteLine($" count() straight afterwards = {immediately}. That number is 2 on one run and 4 on the next:"); Console.WriteLine(" the buffer flushes on its own schedule and the call no longer waits for it."); - Console.WriteLine($" After SYSTEM FLUSH ASYNC INSERT QUEUE: count() = {afterFlush}."); + Console.WriteLine($" Counting again until the buffer has been written: count() = {afterFlush}."); Console.WriteLine(); Console.WriteLine(" Two things the pair changes, neither of which is visible in the API:"); Console.WriteLine(" - a returned InsertRowsAsync no longer means the rows are stored, so a failure"); @@ -286,6 +285,25 @@ private static async Task ReadLog(ClickHouseTcpClient client, string sql return "no row appeared in system.query_log after 5 attempts"; } + /// Counts the table until it holds rows, or gives up. + private static async Task CountWhenItReaches(ClickHouseTcpClient client, long expected) + { + long count = 0; + + for (int attempt = 0; attempt < 300; attempt++) + { + count = Convert.ToInt64(await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}")); + if (count >= expected) + { + return count; + } + + await Task.Delay(10); + } + + return count; + } + /// Waits until system.processes shows the query, so a race cannot decide the next assertion. private static async Task WaitUntilRunning(ClickHouseTcpClient client, string queryId) { @@ -306,6 +324,9 @@ private static async Task WaitUntilRunning(ClickHouseTcpClient client, string qu await Task.Delay(10); } + + throw new TimeoutException( + $"query {queryId} did not appear in system.processes, so the next step's precondition does not hold"); } /// The server's message is one long line with its own detail appended; the first line is the fact. diff --git a/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs b/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs index bef8c58fd..8f733774f 100644 --- a/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs +++ b/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs @@ -161,13 +161,13 @@ private static async Task NotServerErrors() // misconfiguration just fails again. try { - await using var wrongPort = new ClickHouseTcpClient(options with { Port = ExampleConfig.HttpPort }); + await using var wrongPort = new ClickHouseTcpClient(options with { Port = ExampleConfig.HttpEndpoint.Port }); await wrongPort.PingAsync(); Console.WriteLine(" The HTTP port spoke the native protocol, which is not what this example expected"); } catch (ClickHouseTcpException ex) { - Console.WriteLine($"\n the HTTP port ({ExampleConfig.HttpPort}) {ex.GetType().Name}"); + Console.WriteLine($"\n the HTTP port ({ExampleConfig.HttpEndpoint.Port}) {ex.GetType().Name}"); Console.WriteLine($" IsTransient={ex.IsTransient}"); Console.WriteLine($" {ex.Message}"); Console.WriteLine(" 72 is 'H', the first byte of an HTTP response."); @@ -260,7 +260,9 @@ private static async Task RetryingARead(ClickHouseTcpClient client) Settings = new Dictionary { ["max_concurrent_queries_for_user"] = "1" }, }; - for (int attempt = 1; attempt <= 8; attempt++) + const int attempts = 8; + + for (int attempt = 1; attempt <= attempts; attempt++) { try { @@ -268,10 +270,20 @@ private static async Task RetryingARead(ClickHouseTcpClient client) Console.WriteLine($" attempt {attempt}: {counted}"); break; } - catch (ClickHouseTcpException ex) when (ex.IsTransient && attempt < 8) + catch (ClickHouseTcpException ex) when (ex.IsTransient) { string reason = ex is ClickHouseTcpServerException server ? $"{server.Code} ({server.RawCode})" : ex.GetType().Name; Console.WriteLine($" attempt {attempt}: {reason} — transient, so try again"); + + // The last attempt is caught too. The limit counts every query this user is running, so anything + // else on the server under the same user can keep refusing this one, and an example must report + // that rather than throw out of the demonstration. + if (attempt == attempts) + { + Console.WriteLine($" gave up after {attempts}: the cap is what stops a retry becoming a loop."); + break; + } + await Task.Delay(100 * attempt); } } @@ -312,6 +324,9 @@ private static async Task WaitUntilRunning(ClickHouseTcpClient client, string qu await Task.Delay(10); } + + throw new TimeoutException( + $"query {queryId} did not appear in system.processes, so the next step's precondition does not hold"); } private static async Task RetryingAnInsert(ClickHouseTcpClient client) diff --git a/examples/Tcp/Advanced/Tcp_024_Compression.cs b/examples/Tcp/Advanced/Tcp_024_Compression.cs index 49fe9701e..4dec13c93 100644 --- a/examples/Tcp/Advanced/Tcp_024_Compression.cs +++ b/examples/Tcp/Advanced/Tcp_024_Compression.cs @@ -201,7 +201,7 @@ private static string Describe(IClickHouseCompressor compressor) ClickHouseTcpClientOptions options, ClickHouseTcpQueryOptions? queryOptions) { - await using var proxy = new CountingProxy(ExampleConfig.Host, ExampleConfig.TcpPort); + await using var proxy = new CountingProxy(ExampleConfig.TcpEndpoint.Host, ExampleConfig.TcpEndpoint.Port); long rows = 0; @@ -220,7 +220,7 @@ private static string Describe(IClickHouseCompressor compressor) private static async Task MeasureInsert(ClickHouseTcpClientOptions options) { - await using var proxy = new CountingProxy(ExampleConfig.Host, ExampleConfig.TcpPort); + await using var proxy = new CountingProxy(ExampleConfig.TcpEndpoint.Host, ExampleConfig.TcpEndpoint.Port); var ids = new ulong[InsertRows]; var text = new string[InsertRows]; diff --git a/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs b/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs index fbdfe96a6..e0940d777 100644 --- a/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs +++ b/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs @@ -34,6 +34,10 @@ public static async Task Run() // One call, and the answer came from the handshake rather than from a query — there is no round trip // beyond opening a connection, so this is cheap enough to do at startup. + // + // It describes the connection this call borrowed, not the client. Every query below borrows its own, so + // behind a load balancer over mixed versions a gate decided here can be wrong for the query it gates. + // Open a session when that matters: one session is one connection for its whole life. ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); EveryField(server); @@ -105,9 +109,9 @@ private static async Task GatingOnTheRevision(ClickHouseTcpClient client, ClickH options); Console.WriteLine($" So the parameterized query ran: count() = {count}."); - Console.WriteLine($" Below {ParametersRevision} the server has nowhere to read the parameters list from, and rejects"); - Console.WriteLine(" the query rather than running it unparameterized — which is the right failure, but"); - Console.WriteLine(" not one to discover in production. Tcp_007 covers parameters themselves."); + Console.WriteLine($" Below {ParametersRevision} the query packet has no field for the parameters list, so the client"); + Console.WriteLine(" throws NotSupportedException before it sends anything, rather than sending the query"); + Console.WriteLine(" unparameterized. The connection stays usable. Tcp_007 covers parameters themselves."); } else { @@ -139,8 +143,9 @@ private static async Task GatingOnTheVersion(ClickHouseTcpClient client, ClickHo try { await client.ExecuteAsync($"CREATE TABLE {table} (v QBit(Int8, 8)) ENGINE = MergeTree ORDER BY tuple()"); + // Qualified by database as well as by name: system.columns spans every database on the server. object declared = await client.ExecuteScalarAsync( - $"SELECT type FROM system.columns WHERE table = '{table}' AND name = 'v'"); + $"SELECT type FROM system.columns WHERE database = currentDatabase() AND table = '{table}' AND name = 'v'"); Console.WriteLine($" QBit(Int8, 8) declared as {declared}"); } finally diff --git a/examples/Tcp/Connection/Tcp_016_Sessions.cs b/examples/Tcp/Connection/Tcp_016_Sessions.cs index 048667fe7..f71152f11 100644 --- a/examples/Tcp/Connection/Tcp_016_Sessions.cs +++ b/examples/Tcp/Connection/Tcp_016_Sessions.cs @@ -136,15 +136,17 @@ private static async Task SetRoleInASession(ClickHouseTcpClient admin) Console.WriteLine(" ClickHouseTcpQueryOptions carries no Roles: there is nowhere on the wire to put one per"); Console.WriteLine(" query. A session is the equivalent, and it is per connection rather than per query.\n"); - await admin.ExecuteAsync($"CREATE OR REPLACE TABLE {RoleTable} (id UInt64) ENGINE = MergeTree ORDER BY id"); - await admin.ExecuteAsync($"CREATE ROLE OR REPLACE {RoleName}"); - await admin.ExecuteAsync($"GRANT SELECT ON {RoleTable} TO {RoleName}"); - await admin.ExecuteAsync($"CREATE USER OR REPLACE {RoleUser} IDENTIFIED WITH plaintext_password BY '{RoleUserPassword}'"); - await admin.ExecuteAsync($"GRANT {RoleName} TO {RoleUser}"); - Console.WriteLine($" Created user '{RoleUser}', role '{RoleName}' holding SELECT on '{RoleTable}'"); - try { + // Inside the try, so that a failure part way through this sequence still reaches the finally and + // drops whatever it did create. + await admin.ExecuteAsync($"CREATE OR REPLACE TABLE {RoleTable} (id UInt64) ENGINE = MergeTree ORDER BY id"); + await admin.ExecuteAsync($"CREATE ROLE OR REPLACE {RoleName}"); + await admin.ExecuteAsync($"GRANT SELECT ON {RoleTable} TO {RoleName}"); + await admin.ExecuteAsync($"CREATE USER OR REPLACE {RoleUser} IDENTIFIED WITH plaintext_password BY '{RoleUserPassword}'"); + await admin.ExecuteAsync($"GRANT {RoleName} TO {RoleUser}"); + Console.WriteLine($" Created user '{RoleUser}', role '{RoleName}' holding SELECT on '{RoleTable}'"); + // Same server as everything else here; only the credentials differ. var asUser = ExampleConfig.TcpBuilder(); asUser.Username = RoleUser; diff --git a/examples/Tcp/Connection/Tcp_017_PoolTuning.cs b/examples/Tcp/Connection/Tcp_017_PoolTuning.cs index 56d7fe7be..020ee92dc 100644 --- a/examples/Tcp/Connection/Tcp_017_PoolTuning.cs +++ b/examples/Tcp/Connection/Tcp_017_PoolTuning.cs @@ -100,7 +100,9 @@ private static async Task MaxPoolSizeCapsConcurrency() private static async Task Measure(int maxPoolSize) { - string marker = $"example_tcp_pool_cap_{maxPoolSize}"; + // Unique per call: the count below matches on the marker, so a second run of this example would + // otherwise be counted too and could report more running queries than this client's pool allows. + string marker = $"example_tcp_pool_cap_{maxPoolSize}_{Guid.NewGuid():N}"; var capture = new LogCapture(); await using var observer = ExampleConfig.CreateTcpClient(); diff --git a/examples/Tcp/Connection/Tcp_019_Timeouts.cs b/examples/Tcp/Connection/Tcp_019_Timeouts.cs index dd7076022..d98b4ae58 100644 --- a/examples/Tcp/Connection/Tcp_019_Timeouts.cs +++ b/examples/Tcp/Connection/Tcp_019_Timeouts.cs @@ -68,8 +68,8 @@ private static async Task DialingTheWrongThing() // The HTTP port. Both interfaces are ClickHouse, but they speak different protocols, and the native client // reads the HTTP server's reply as a protocol packet. clock.Restart(); - Exception? wrongPort = await Failing(options with { Port = ExampleConfig.HttpPort, DialTimeout = TimeSpan.FromSeconds(2) }); - Console.WriteLine($"\n The HTTP port ({ExampleConfig.HttpPort}) instead of the native one, after {clock.ElapsedMilliseconds} ms:"); + Exception? wrongPort = await Failing(options with { Port = ExampleConfig.HttpEndpoint.Port, DialTimeout = TimeSpan.FromSeconds(2) }); + Console.WriteLine($"\n The HTTP port ({ExampleConfig.HttpEndpoint.Port}) instead of the native one, after {clock.ElapsedMilliseconds} ms:"); Console.WriteLine($" {Describe(wrongPort)}"); Console.WriteLine(" Packet type 72 is 'H', the first byte of the HTTP response. Not a timeout either."); diff --git a/examples/Tcp/Core/Tcp_001_BasicUsage.cs b/examples/Tcp/Core/Tcp_001_BasicUsage.cs index a26aff45d..fb568f604 100644 --- a/examples/Tcp/Core/Tcp_001_BasicUsage.cs +++ b/examples/Tcp/Core/Tcp_001_BasicUsage.cs @@ -14,7 +14,8 @@ namespace ClickHouse.Driver.Examples; /// public static class TcpBasicUsage { - // These examples are not the test suite, so a fixed name is fine. It is dropped even if a step throws. + // Fixed, like every example_* table in this project: the suite runs one at a time against a server, so + // it does not need the unique names the test suites do. It is dropped even if a step throws. private const string TableName = "example_tcp_basic_usage"; public static async Task Run() @@ -26,9 +27,11 @@ public static async Task Run() // 'await using', not 'using': the client is IAsyncDisposable, and disposal closes sockets. await using var client = ExampleConfig.CreateTcpClient(); - Console.WriteLine($"Native protocol endpoint: {ExampleConfig.Host}:{ExampleConfig.TcpPort}, user '{ExampleConfig.Username}'"); + var endpoint = ExampleConfig.TcpEndpoint; + Console.WriteLine($"Native protocol endpoint: {endpoint.Host}:{endpoint.Port}, user '{ExampleConfig.TcpBuilder().Username}'"); - // Read out of the handshake the connection already made, so this costs no query. + // Read out of the handshake rather than from a query. A newly built client holds no connection, so + // the first call here dials and handshakes; later ones read what that handshake recorded. var server = await client.GetServerInfoAsync(); Console.WriteLine($"Server: {server} (protocol revision {server.ProtocolRevision}, timezone {server.Timezone})"); @@ -50,6 +53,7 @@ public static async Task Run() private static async Task CreateTable(ClickHouseTcpClient client) { // ExecuteAsync is for anything that returns no rows: DDL, and DML other than INSERT ... VALUES. + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( @@ -109,7 +113,7 @@ private static async Task ReadOneValue(ClickHouseTcpClient client) private static void ShowTheReadTiers() { Console.WriteLine("\nThree read tiers, all on this client:"); - Console.WriteLine(" QueryAsync one object[] per row, every value boxed"); + Console.WriteLine(" QueryAsync one object[] per row, value-type columns boxed"); Console.WriteLine(" QueryAsync one POCO per row, filled by column name"); Console.WriteLine(" StreamAsync whole Blocks, typed columns, no per-row boxing"); Console.WriteLine(); diff --git a/examples/Tcp/Core/Tcp_002_ConnectionString.cs b/examples/Tcp/Core/Tcp_002_ConnectionString.cs index 67f52061f..5d93714ea 100644 --- a/examples/Tcp/Core/Tcp_002_ConnectionString.cs +++ b/examples/Tcp/Core/Tcp_002_ConnectionString.cs @@ -137,7 +137,7 @@ private static void ShowTlsKeys() { _ = new ClickHouseTcpClient(new ClickHouseTcpClientOptions { - Host = ExampleConfig.Host, + Host = ExampleConfig.TcpEndpoint.Host, TlsAllowInvalidCertificates = true, }); } @@ -156,7 +156,11 @@ private static async Task ConnectWithThem(ClickHouseTcpClientOptions options) await using var client = new ClickHouseTcpClient(options); var server = await client.GetServerInfoAsync(); - Console.WriteLine($" Connected to {server}, blocks framed with {Describe(options.Compressor)}"); + + // Compressor names the codec this client writes its own blocks with. It does not choose the codec + // for the blocks coming back: the query packet carries a compression flag and no codec name, so + // the server picks that one itself. + Console.WriteLine($" Connected to {server}, blocks this client writes framed with {Describe(options.Compressor)}"); // set_max_threads became a client-level setting, so the server sees it on every operation. object maxThreads = await client.ExecuteScalarAsync("SELECT getSetting('max_threads')"); diff --git a/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs b/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs index 5448b0a70..a86668fdd 100644 --- a/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs +++ b/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs @@ -69,6 +69,7 @@ private static async Task SameTaskBothWays(ClickHouseClient http, ClickHouseTcpC private static async Task CompareTransports(ClickHouseClient http, ClickHouseTcpClient tcp) { // DDL. HTTP: ExecuteNonQueryAsync, which returns the affected-row count ADO.NET expects. + await http.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {TableName}"); await http.ExecuteNonQueryAsync($@" CREATE TABLE {TableName} (id UInt64, name String, source String) ENGINE = MergeTree() ORDER BY id"); diff --git a/examples/Tcp/Observability/Tcp_026_Logging.cs b/examples/Tcp/Observability/Tcp_026_Logging.cs index 427277146..b24195dd8 100644 --- a/examples/Tcp/Observability/Tcp_026_Logging.cs +++ b/examples/Tcp/Observability/Tcp_026_Logging.cs @@ -239,7 +239,9 @@ private static async Task Workload(ILoggerFactory factory) // back into the pool. The client logs one Error line and rethrows. try { - _ = await client.ExecuteScalarAsync("SELECT * FROM example_tcp_logging_no_such_table"); + // Unique, so the query fails because the table does not exist rather than because it happens not + // to exist: a fixed name is one CREATE TABLE away from making this demonstration succeed. + _ = await client.ExecuteScalarAsync($"SELECT * FROM example_tcp_logging_no_such_table_{Guid.NewGuid():N}"); } catch (ClickHouseTcpServerException) { diff --git a/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs b/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs index a6b2df7d6..defbec037 100644 --- a/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs +++ b/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs @@ -79,7 +79,9 @@ await client.InsertRowsAsync( // stays usable. try { - _ = await client.ExecuteScalarAsync("SELECT * FROM example_tcp_open_telemetry_no_such_table"); + // Unique for the same reason as in Tcp_026: the failure has to be the missing table. + _ = await client.ExecuteScalarAsync( + $"SELECT * FROM example_tcp_open_telemetry_no_such_table_{Guid.NewGuid():N}"); } catch (ClickHouseTcpServerException) { diff --git a/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs b/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs index 187c36b2e..7b901e099 100644 --- a/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs +++ b/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs @@ -20,8 +20,9 @@ namespace ClickHouse.Driver.Examples; /// /// /// The contract otherwise is the one Tcp_021 states: in packet order, on the reading thread, and never allowed to -/// throw — an exception propagates out of the operation and terminates the connection. Copying a few values -/// cannot throw, which is one more reason to copy and leave. +/// throw — an exception propagates out of the operation and terminates the connection. So keep the callback to +/// copying values out, and do the parsing that can fail somewhere it is allowed to. Even a copy has to be written +/// with that in mind: a named column lookup or a span index throws if the name or the row is not there. /// /// public static class TcpMetadataBlocks diff --git a/examples/Tcp/Observability/Tcp_029_HealthChecks.cs b/examples/Tcp/Observability/Tcp_029_HealthChecks.cs index 5e74a35d4..20ee613c7 100644 --- a/examples/Tcp/Observability/Tcp_029_HealthChecks.cs +++ b/examples/Tcp/Observability/Tcp_029_HealthChecks.cs @@ -125,11 +125,11 @@ private static async Task ThreeEndpointsOneReport() } Console.WriteLine(); - Console.WriteLine(" Three states, and the middle one is the point: a pool with nothing free throws"); - Console.WriteLine(" TimeoutException, which says nothing about the server, so reporting it as Unhealthy would"); - Console.WriteLine(" take an instance out of rotation for being busy. Degraded is the honest answer. The"); - Console.WriteLine(" distinction has to be made on the exception type, because there is no status on the"); - Console.WriteLine(" client to ask."); + Console.WriteLine(" Three states, and the middle one is the point: a timeout says nothing about the server,"); + Console.WriteLine(" so reporting it as Unhealthy would take an instance out of rotation for being busy."); + Console.WriteLine(" Degraded is the honest answer. Note what the check cannot know: TimeoutException covers"); + Console.WriteLine(" waiting for a pool slot, dialing, and reading the pong alike, and the client exposes"); + Console.WriteLine(" nothing that separates them, so treat it as 'did not finish' and no more."); Console.WriteLine(); Console.WriteLine(" Every registration resolves the client the data source owns, so the check runs on the"); Console.WriteLine(" application's own pool and measures the path a request would take. That is also why the"); @@ -212,9 +212,10 @@ public async Task CheckHealthAsync( } catch (TimeoutException e) { - // No connection came free within PoolTimeout. The server has not been reached, so this is a - // statement about this process, not about ClickHouse. - return HealthCheckResult.Degraded("No pooled connection was free", e, data); + // One type covers three deadlines: waiting for a pool slot, dialing, and reading the pong. None + // of them is proof the server is down, and none of them can be told apart here, so the honest + // report is that the check did not finish in time. + return HealthCheckResult.Degraded("Timed out before a pong", e, data); } catch (Exception e) when (e is not OperationCanceledException) { diff --git a/examples/Tcp/Observability/Tcp_030_Testcontainers.cs b/examples/Tcp/Observability/Tcp_030_Testcontainers.cs index 1f735c2a4..ecc09a09c 100644 --- a/examples/Tcp/Observability/Tcp_030_Testcontainers.cs +++ b/examples/Tcp/Observability/Tcp_030_Testcontainers.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Net.Sockets; using ClickHouse.Driver.Tcp; using DotNet.Testcontainers.Builders; using Testcontainers.ClickHouse; @@ -130,10 +131,12 @@ private static async Task HandshakeWhenReady(ClickHouse return info; } - catch (ClickHouseTcpTransportException) when (started.Elapsed < deadline) + catch (ClickHouseTcpTransportException e) + when (e.InnerException is SocketException && started.Elapsed < deadline) { - // Only a transport failure is worth retrying: the listener is not accepting yet. A server - // exception would mean it answered and rejected us, which no amount of waiting fixes. + // Only a socket failure is worth retrying: the listener is not accepting yet. The same exception + // type also carries a TLS or DNS failure, which no amount of waiting fixes, so it is the inner + // exception that decides. A server exception would mean the server answered and rejected us. await Task.Delay(100); } } diff --git a/examples/Tcp/Read/Tcp_005_ReadTiers.cs b/examples/Tcp/Read/Tcp_005_ReadTiers.cs index cf4bc512e..d97cae29d 100644 --- a/examples/Tcp/Read/Tcp_005_ReadTiers.cs +++ b/examples/Tcp/Read/Tcp_005_ReadTiers.cs @@ -5,7 +5,7 @@ namespace ClickHouse.Driver.Examples; /// /// The three ways the native client reads a result, run one after another over the same query: QueryAsync -/// (one object[] per row, every value boxed), QueryAsync<T> (one POCO per row, values +/// (one object[] per row, value-type columns boxed), QueryAsync<T> (one POCO per row, values /// converted) and StreamAsync (whole s, typed columns, no per-row work at all). /// /// @@ -46,6 +46,7 @@ private static async Task Seed(ClickHouseTcpClient client) { // recorded_at declares its timezone. A bare DateTime would take the server's, which is what the block // tier reports as the column's TimeZone; naming UTC makes this example's output the same everywhere. + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( @@ -78,7 +79,7 @@ await client.InsertRowsAsync( private static async Task RowTier(ClickHouseTcpClient client) { - Console.WriteLine("\n1. QueryAsync — one object[] per row, every value boxed\n"); + Console.WriteLine("\n1. QueryAsync — one object[] per row, value-type columns boxed\n"); Console.WriteLine(" ID City Temp recorded_at CLR types"); Console.WriteLine(" -- --------- ----- ----------- ---------"); @@ -272,8 +273,8 @@ private static void ShowTheChoice() Console.WriteLine(" Block.ColumnNames if you need them. Date and time columns arrive raw."); Console.WriteLine(); Console.WriteLine(" QueryAsync The default for application code. One object per row instead of an"); - Console.WriteLine(" array plus a box per value, values converted to the property's type,"); - Console.WriteLine(" and each row owns its values, so a row can be kept or returned."); + Console.WriteLine(" array plus a box per value-type column, values converted to the"); + Console.WriteLine(" property's type, and each row owns its values, so a row can be kept."); Console.WriteLine(); Console.WriteLine(" StreamAsync Aggregating, scanning, or handing a column to something that wants a"); Console.WriteLine(" span. Nothing is materialized per row, and a column read out of one"); diff --git a/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs b/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs index cd1baf856..e4ea319a3 100644 --- a/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs +++ b/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs @@ -44,6 +44,7 @@ private static async Task Seed(ClickHouseTcpClient client) { // captured_at names a zone with an offset and a daylight-saving rule, so the block tier has something to // report; uptime is a Time, which is a count from midnight and has no zone at all. + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( @@ -156,7 +157,6 @@ private static async Task AddressingAColumn(ClickHouseTcpClient client) // The first block is enough for the sections that follow. Breaking out is allowed: the client tells the // server the result is abandoned, and drops that connection instead of returning it to the pool. - break; } } @@ -173,8 +173,6 @@ private static async Task WhatAColumnReports(ClickHouseTcpClient client) Console.WriteLine( $" {column.Name,-11} {column.TypeName,-35} {Describe(column.ElementType),-11} {column.RowCount,4} {ExtraInterface(column)}"); } - - break; } Console.WriteLine(); @@ -213,8 +211,6 @@ private static async Task ValuesAsSpans(ClickHouseTcpClient client) Console.WriteLine($" A String column is a span too, of references: sensor.Values = [{string.Join(", ", sensor.Values.ToArray())}]"); Console.WriteLine(" Reading it decodes one string per value, so the block tier saves less on String than"); Console.WriteLine(" on a fixed-width type. It still saves the object[] and the boxes."); - - break; } } @@ -257,8 +253,6 @@ private static async Task DateAndTimeColumns(ClickHouseTcpClient client) Console.WriteLine(" The count is signed and is not clamped to one day, so a TimeSpan here can be"); Console.WriteLine(" negative or longer than 24 hours."); } - - break; } Console.WriteLine(); @@ -343,8 +337,6 @@ private static async Task ArrayColumns(ClickHouseTcpClient client) Console.WriteLine($" Values[0] = [{string.Join(", ", rows.Values[0])}] (Values: every row's array, built at once)"); Console.WriteLine(" Those arrays outlive the block. The span holding them does not, being a span."); Console.WriteLine(" So prefer the indexer when only a few rows out of a tall block are wanted."); - - break; } Console.WriteLine(); diff --git a/examples/Tcp/Read/Tcp_007_Parameters.cs b/examples/Tcp/Read/Tcp_007_Parameters.cs index 0b039fbf5..f5afefcb4 100644 --- a/examples/Tcp/Read/Tcp_007_Parameters.cs +++ b/examples/Tcp/Read/Tcp_007_Parameters.cs @@ -42,6 +42,7 @@ public static async Task Run() private static async Task Seed(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( @@ -275,8 +276,9 @@ private static async Task NamesThatCollideWithSettings(ClickHouseTcpClient clien // A client of its own for the query that is meant to fail. The server rejects this one while it is still // reading the settings list, and then closes the socket, so the connection it was on is dead even though - // the client saw an ordinary server error. Disposing this client throws that connection away with it; - // running the query on the shared client would leave a dead connection in its pool. + // the client saw an ordinary server error. The pool checks a connection for a closed socket both on + // return and on checkout, so it usually discards this one; a close notice that arrives after both checks + // can still be handed out. Disposing a throwaway client keeps that race out of the shared pool. await using (ClickHouseTcpClient throwaway = ExampleConfig.CreateTcpClient()) { try diff --git a/examples/Tcp/Read/Tcp_008_Poco.cs b/examples/Tcp/Read/Tcp_008_Poco.cs index 6a37e3ab9..dd8aa1304 100644 --- a/examples/Tcp/Read/Tcp_008_Poco.cs +++ b/examples/Tcp/Read/Tcp_008_Poco.cs @@ -42,6 +42,7 @@ public static async Task Run() private static async Task CreateTable(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( diff --git a/examples/Tcp/Types/Tcp_011_ScalarTypes.cs b/examples/Tcp/Types/Tcp_011_ScalarTypes.cs index 00d7918c6..28b0812a7 100644 --- a/examples/Tcp/Types/Tcp_011_ScalarTypes.cs +++ b/examples/Tcp/Types/Tcp_011_ScalarTypes.cs @@ -10,13 +10,19 @@ namespace ClickHouse.Driver.Examples; /// the 256-bit integers, the decimals, BFloat16, and the enums. /// /// -/// One rule underlies all of it: the client hands back the value the wire carried, in the narrowest CLR -/// type that holds it without loss. So UInt8 is a and not an , +/// One rule underlies most of it: the client hands back the value the wire carried, in the narrowest CLR +/// type that holds it. So UInt8 is a and not an , /// FixedString(N) is a [] and not a , and an Enum8 is its /// ordinal and not its label. The same type is what an insert column must hold, in both directions. /// /// /// +/// Three types are a chosen CLR surface rather than the wire bytes: BFloat16 widens to a +/// , IPv4 and IPv6 become an IPAddress, and String is decoded as +/// UTF-8, which a ClickHouse String is not required to be. Sections 4 and 5 show what that costs. +/// +/// +/// /// Tcp_012 covers the date and time family, Tcp_013 the composites. This one assumes the block tier /// from Tcp_006. /// @@ -54,6 +60,7 @@ public static async Task Run() private static async Task Seed(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( @@ -134,13 +141,12 @@ private static async Task TheWholeMap(ClickHouseTcpClient client) object value = column.GetValue(0); Console.WriteLine($" {column.TypeName,-36} {Describe(column.ElementType),-20} {Render(value)}"); } - - break; } Console.WriteLine(); - Console.WriteLine(" Nothing in that table is a conversion. ElementType is the type the wire's bytes are,"); - Console.WriteLine(" so a read costs a copy at most, and the same type is what an insert column must hold."); + Console.WriteLine(" ElementType is what an insert column must hold as well as what a read gives back. For most"); + Console.WriteLine(" of that table it is the wire's own type, so a read costs a copy at most. BFloat16, IPv4,"); + Console.WriteLine(" IPv6 and String are the exceptions: each is a CLR surface built from the wire bytes."); } private static async Task WideIntegers(ClickHouseTcpClient client) @@ -176,8 +182,6 @@ private static async Task WideIntegers(ClickHouseTcpClient client) signed.WriteLittleEndian(raw); Console.WriteLine($" WriteLittleEndian {Convert.ToHexString(raw)}"); Console.WriteLine($" ReadLittleEndian round trip {Int256.ReadLittleEndian(raw) == signed}"); - - break; } } @@ -197,7 +201,6 @@ private static async Task Decimals(ClickHouseTcpClient client) } Console.WriteLine(" Both hold 1.25. Only the declared precision differs."); - break; } Console.WriteLine(); @@ -214,8 +217,6 @@ private static async Task Decimals(ClickHouseTcpClient client) Console.WriteLine($" Scale {value.Scale}, Sign {value.Sign}, ToString() {value}"); Console.WriteLine($" TryToDecimal {narrows}{(narrows ? $" -> {narrowed}" : " (out of a System.Decimal's range)")}"); } - - break; } Console.WriteLine(); @@ -232,8 +233,6 @@ private static async Task Decimals(ClickHouseTcpClient client) var value = (ClickHouseTcpDecimal)column.GetValue(0); Console.WriteLine($" {column.Name,-18} {column.TypeName,-16} TryToDecimal {value.TryToDecimal(out _),-5} {value}"); } - - break; } // Two values of different scale can be the same number, and comparison says so. @@ -262,7 +261,6 @@ private static async Task Floats(ClickHouseTcpClient client) Console.WriteLine($" Float64 wrote -2.25 read {block.Column("f64")[0]}"); Console.WriteLine($" BFloat16 wrote 0.1f read {block.Column("bf16")[0]:R}"); Console.WriteLine(" 7 stored mantissa bits, so 0.1 is not representable and the nearest value comes back."); - break; } } @@ -281,7 +279,6 @@ private static async Task StringsAndBytes(ClickHouseTcpClient client) Console.WriteLine($" IPv4 -> IPAddress {block.Column("ip4")[0]}"); Console.WriteLine($" IPv6 -> IPAddress {block.Column("ip6")[0]}"); Console.WriteLine(" One CLR type for both, told apart by AddressFamily."); - break; } await foreach (Block block in client.StreamAsync( @@ -296,7 +293,6 @@ private static async Task StringsAndBytes(ClickHouseTcpClient client) } Console.WriteLine(" An IPv4 address in an IPv6 column is the mapped form, ::ffff:a.b.c.d."); - break; } Console.WriteLine(); @@ -312,7 +308,6 @@ private static async Task StringsAndBytes(ClickHouseTcpClient client) } Console.WriteLine(" 0xFFFE came back as two replacement characters. Use FixedString(N) for bytes."); - break; } Console.WriteLine(); @@ -345,8 +340,6 @@ private static async Task Enums(ClickHouseTcpClient client) Console.WriteLine($" {column.Name,-4} {column.TypeName}"); Console.WriteLine($" reads as {Describe(column.ElementType)} = {column.GetValue(0)}"); } - - break; } Console.WriteLine(); @@ -389,8 +382,6 @@ private static async Task Nothing(ClickHouseTcpClient client) { Console.WriteLine($" {column.Name,-14} {column.TypeName,-18} reads as {Describe(column.ElementType),-10} value {Render(column.GetValue(0))}"); } - - break; } Console.WriteLine(); diff --git a/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs b/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs index ceb0771e2..ee46bc49d 100644 --- a/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs +++ b/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs @@ -66,6 +66,7 @@ public static async Task Run() private static async Task Seed(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( @@ -122,8 +123,6 @@ private static async Task SixTypes(ClickHouseTcpClient client) Console.WriteLine( $" {column.TypeName,-28} {Describe(column.ElementType),-10} {Raw(column.GetValue(0)),-19} {Extra(column)}"); } - - break; } Console.WriteLine(); @@ -160,8 +159,6 @@ private static async Task WhatTheWireCarries(ClickHouseTcpClient client) Console.WriteLine(" A Time carries no timezone at all, because it is not an instant:"); Console.WriteLine($" t {block["t"].TypeName,-30} Scale {t.Scale}, GetTimeSpan(0) {t.GetTimeSpan(0)}"); Console.WriteLine($" dt64_tz {block["dt64_tz"].TypeName,-30} Scale {dt64.Scale}, TimeZone {dt64.TimeZone.Id}"); - - break; } } @@ -188,7 +185,6 @@ private static async Task WhereThePresentationTimezoneComesFrom(ClickHouseTcpCli var declared = (IDateTimeColumn)block["declared"]; Console.WriteLine( $" {(zone.Length == 0 ? "(not set)" : zone),-20} {block.Column("bare")[0],-14} {Format(bare.GetDateTimeOffset(0)),-30} {Format(declared.GetDateTimeOffset(0))}"); - break; } } @@ -231,8 +227,6 @@ private static async Task Scale(ClickHouseTcpClient client) var instants = (IDateTimeColumn)column; Console.WriteLine($" {column.TypeName,-20} {column.GetValue(0),-21} {instants.GetDateTimeOffset(0):yyyy-MM-dd HH:mm:ss.fffffff}"); } - - break; } Console.WriteLine(); @@ -278,7 +272,6 @@ private static async Task KindOnTheWritePath(ClickHouseTcpClient client) { var stored = (IDateTimeColumn)block["t"]; Console.WriteLine($" {what,-22} -> count {block.Column("t")[0]}, presented {Format(stored.GetDateTimeOffset(0))}"); - break; } await client.ExecuteAsync($"TRUNCATE TABLE {KindTable}"); @@ -336,16 +329,24 @@ await client.ExecuteScalarAsync( ("{t:DateTime}", DateTime.SpecifyKind(new DateTime(2026, 6, 1, 12, 0, 0), DateTimeKind.Unspecified), "Kind=Unspecified — a wall clock, so no timezone is needed"), }) { + // session_timezone is pinned, because the last case below is read in it and a server left on its + // own default would make this comparison say something different on every machine. object? epoch = await client.ExecuteScalarAsync( $"SELECT toUnixTimestamp(toDateTime({placeholder}, 'UTC'))", - new ClickHouseTcpQueryOptions { Parameters = new ClickHouseTcpParameterCollection { { "t", value } } }); + new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "t", value } }, + Settings = new Dictionary { ["session_timezone"] = "UTC" }, + }); Console.WriteLine($" {placeholder,-27} -> {epoch} {note}"); } Console.WriteLine(); Console.WriteLine(" The last row is the one to notice: with Kind=Unspecified the count is whatever the"); - Console.WriteLine(" session timezone makes of 12:00, so it agrees with the others only because this session"); - Console.WriteLine(" is UTC. That is exactly the ambiguity the refusal above protects an instant from."); + Console.WriteLine(" session timezone makes of 12:00, so it agrees with the others only because these queries"); + Console.WriteLine(" set session_timezone=UTC. Without that it follows the server, and the same value means a"); + Console.WriteLine(" different instant on a differently configured one. That is the ambiguity the refusal"); + Console.WriteLine(" above protects an instant from."); Console.WriteLine(); Console.WriteLine(" Same rule for DateTime64: {t:DateTime64(3, 'UTC')} declares one, {t:DateTime64(3)} does"); Console.WriteLine(" not. Date, Date32, Time and Time64 have no timezone to declare, so none of this applies"); @@ -371,8 +372,6 @@ private static async Task TimeIsNotATimeOfDay(ClickHouseTcpClient client) var times = (ITimeColumn)column; Console.WriteLine($" {column.Name,-16} {column.GetValue(0),-10} {times.GetTimeSpan(0)}"); } - - break; } Console.WriteLine(); diff --git a/examples/Tcp/Types/Tcp_013_CompositeRead.cs b/examples/Tcp/Types/Tcp_013_CompositeRead.cs index 9c2ace2d6..4794debfe 100644 --- a/examples/Tcp/Types/Tcp_013_CompositeRead.cs +++ b/examples/Tcp/Types/Tcp_013_CompositeRead.cs @@ -30,9 +30,13 @@ public static class TcpCompositeRead private const string Columns = "id, readings, attrs, point, named_point, score, city, nick, matrix, tagged, buckets"; + // Geometry, the Variant over the six geo aliases, is newer than the rest of this example. + private static readonly Version GeometryFrom = new(25, 11); + public static async Task Run() { await using var client = ExampleConfig.CreateTcpClient(); + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); try { @@ -45,7 +49,7 @@ public static async Task Run() await Nesting(client); await NestedColumns(client); await GeoAliases(client); - await Geometry(client); + await Geometry(client, server); } finally { @@ -57,6 +61,7 @@ public static async Task Run() private static async Task Seed(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( @@ -139,15 +144,12 @@ private static async Task WhichViewEachCompositeOffers(ClickHouseTcpClient clien { Console.WriteLine($" {column.TypeName,-32} {Describe(column.ElementType),-30} {View(column)}"); } - - break; } await foreach (Block block in client.StreamAsync($"SELECT items FROM {NestedTable} ORDER BY id")) { IColumn column = block["items"]; Console.WriteLine($" {column.TypeName,-32} {Describe(column.ElementType),-30} {View(column)}"); - break; } Console.WriteLine(); @@ -196,8 +198,6 @@ private static async Task MapsAndArrays(ClickHouseTcpClient client) Console.WriteLine(" Taking only the keys, or only the values, therefore costs nothing:"); Console.WriteLine($" distinct keys across every row = {string.Join(", ", keys.Values.ToArray().Distinct())}"); } - - break; } } @@ -226,8 +226,6 @@ private static async Task Tuples(ClickHouseTcpClient client) var named = (ITupleColumn)block["named_point"]; IColumn xs = (IColumn)named.Children[0]; Console.WriteLine($" Children[0].Values = [{string.Join(", ", xs.Values.ToArray())}] (the x of every row, no ValueTuple built)"); - - break; } } @@ -258,8 +256,6 @@ private static async Task Nulls(ClickHouseTcpClient client) Console.WriteLine(" Do not read Inner without the null map. The value at a NULL position is the inner"); Console.WriteLine(" codec's placeholder, not data — here it is 0, which is a perfectly plausible score."); } - - break; } } @@ -296,8 +292,6 @@ private static async Task LowCardinalities(ClickHouseTcpClient client) } } } - - break; } Console.WriteLine(); @@ -350,8 +344,6 @@ private static async Task Nesting(ClickHouseTcpClient client) Console.WriteLine(" The one thing to know is that each match needs the child's element type spelled out,"); Console.WriteLine(" which IColumn.ElementType on the parent tells you: Array(Array(Int32)) reports int[][],"); Console.WriteLine(" so the outer view is IArrayColumn and the inner one IArrayColumn."); - - break; } } @@ -393,8 +385,6 @@ private static async Task NestedColumns(ClickHouseTcpClient client) Console.WriteLine($" The materialized row is an object[][] — one object[] per entry, boxed, so the"); Console.WriteLine($" field columns are the way to read it: items.GetValue(0) = {Render(block["items"].GetValue(0))}"); } - - break; } Console.WriteLine(); @@ -440,8 +430,6 @@ SELECT CAST((1.0, 2.0), 'Point') AS p, Console.WriteLine($" Children[1] [{string.Join(", ", latitudes.Values.ToArray())}]"); Console.WriteLine($" row 0 {Render(block["r"].GetValue(0))}"); } - - break; } Console.WriteLine(); @@ -453,9 +441,16 @@ SELECT CAST((1.0, 2.0), 'Point') AS p, Console.WriteLine(" client, as are Polygon and MultiLineString. Only the name tells them apart."); } - private static async Task Geometry(ClickHouseTcpClient client) + private static async Task Geometry(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) { Console.WriteLine("\n9. Geometry is the one alias that is not a nested array\n"); + + if (server.Version < GeometryFrom) + { + Console.WriteLine($" Skipped: needs ClickHouse {GeometryFrom} or newer, this server is {server.Version}."); + return; + } + Console.WriteLine(" It names a Variant over the six above, so one column holds rows of different shapes. The"); Console.WriteLine(" header carries only 'Geometry', so the client expands the alternatives itself, in the"); Console.WriteLine(" server's own name-sorted discriminator order:\n"); @@ -484,8 +479,6 @@ SELECT g FROM (SELECT arrayJoin([ Console.WriteLine($" row {row}: discriminator {geometry.Discriminators[row]} -> {child.TypeName,-14} value {Render(block["g"].GetValue(row))}"); } } - - break; } Console.WriteLine(); diff --git a/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs b/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs index 6ae9ae57e..1d7c8676d 100644 --- a/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs +++ b/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs @@ -8,10 +8,11 @@ namespace ClickHouse.Driver.Examples; /// JSON. /// /// -/// Variant and Dynamic are discriminated unions. Both read as IColumn<object>, so every -/// row read that way is boxed, and both expose a columnar view instead — a per-row discriminator plus one typed -/// child column per alternative. They differ in where the alternative list comes from: a Variant declares -/// it in the type string, a Dynamic discovers it per block and reports it as +/// Variant and Dynamic are discriminated unions. Both read as IColumn<object>, which +/// loses the static type and boxes every row whose alternative is a value type, and both expose a columnar view +/// instead — a per-row discriminator plus one typed child column per alternative. They differ in where the +/// alternative list comes from: a Variant declares it in the type string, a Dynamic discovers it per +/// block and reports it as /// . They also differ in how NULL is marked, which is the one detail that /// will bite you. /// @@ -55,14 +56,17 @@ public static async Task Run() private static async Task Seed(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {VariantTable}"); await client.ExecuteAsync($@" CREATE TABLE {VariantTable} (id UInt64, v Variant(String, UInt64, Array(Int32))) ENGINE = MergeTree() ORDER BY id"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {DynamicTable}"); await client.ExecuteAsync($@" CREATE TABLE {DynamicTable} (id UInt64, d Dynamic) ENGINE = MergeTree() ORDER BY id"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {JsonTable}"); await client.ExecuteAsync($@" CREATE TABLE {JsonTable} (id UInt64, doc JSON) ENGINE = MergeTree() ORDER BY id"); @@ -159,14 +163,13 @@ private static async Task Variants(ClickHouseTcpClient client) Console.WriteLine(); Console.WriteLine($" The materialized surface is IColumn: ElementType is {column.ElementType.Name}, so"); - Console.WriteLine(" GetValue boxes every row — including the ones whose alternative is a value type:"); + Console.WriteLine(" the static type is gone and a value-type alternative is boxed. A String or an Array is"); + Console.WriteLine(" already a reference, so it costs nothing beyond the object[] the caller sees:"); for (int row = 0; row < column.RowCount; row++) { object? value = column.GetValue(row); Console.WriteLine($" GetValue({row}) -> {(value is null ? "null" : $"{Describe(value.GetType())} {Render(value)}")}"); } - - break; } Console.WriteLine(); @@ -225,8 +228,6 @@ private static async Task Dynamics(ClickHouseTcpClient client) Console.WriteLine(" child to, so a caller can bind IColumn per alternative without inspecting a"); Console.WriteLine(" single value."); } - - break; } Console.WriteLine(); @@ -286,8 +287,6 @@ private static async Task JsonIsText(ClickHouseTcpClient client) { Console.WriteLine($" {column.Name,-11} {column.TypeName,-16} reads as {Describe(column.ElementType),-9} {Render(column.GetValue(0))}"); } - - break; } Console.WriteLine(); @@ -305,9 +304,10 @@ private static async Task JsonIsText(ClickHouseTcpClient client) Settings = new Dictionary { ["output_format_native_write_json_as_string"] = "0" }, }; + // Drained rather than broken out of: the throw is what this demonstrates, and stopping early + // would discard the connection on the way to it. await foreach (Block _ in client.StreamAsync(@"SELECT CAST('{""a"":1}', 'JSON') AS j", withoutTheSetting)) { - break; } Console.WriteLine(" accepted, which this example did not expect"); diff --git a/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs b/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs index b1c04c4e5..28937f2cb 100644 --- a/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs +++ b/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs @@ -27,6 +27,9 @@ public static class TcpQBitVectorSearch private const string TableName = "example_tcp_qbit"; private const string WideTable = "example_tcp_qbit_wide"; + // QBit arrived in 25.10 with limitations, so the driver's own suites gate it at 25.11 and so does this. + private static readonly Version QBitFrom = new(25, 11); + // Int8 elements and the strided QBit(T, N, stride) form both need a newer server. private static readonly Version StridedAndInt8From = new(26, 7); @@ -45,6 +48,13 @@ public static async Task Run() await using var client = ExampleConfig.CreateTcpClient(); ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + if (server.Version < QBitFrom) + { + Console.WriteLine($"QBit needs ClickHouse {QBitFrom} or newer, and this server is {server.Version}."); + Console.WriteLine("Nothing here runs on it: the CREATE TABLE is the first thing that would fail."); + return; + } + try { await Seed(client); @@ -65,6 +75,7 @@ public static async Task Run() private static async Task Seed(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} (word String, vec QBit(Float32, 5)) ENGINE = MergeTree() ORDER BY word"); @@ -112,8 +123,6 @@ private static async Task TheGeometry(ClickHouseTcpClient client) Console.WriteLine(" BitWidth is the width of the STORED element, not of the CLR one: a"); Console.WriteLine(" QBit(BFloat16, N) has 16 planes and still reads as float[]. Section 5."); } - - break; } } @@ -195,8 +204,6 @@ private static async Task ReadingAPlane(ClickHouseTcpClient client) } } } - - break; } } @@ -208,6 +215,7 @@ private static async Task ByteOrderWithinABitmap(ClickHouseTcpClient client) Console.WriteLine(" big-endian encoding of a BytesPerRow-byte integer whose bit i is element i. That is"); Console.WriteLine(" invisible at 5 elements and not at 12:\n"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {WideTable}"); await client.ExecuteAsync($@" CREATE TABLE {WideTable} (v QBit(Float32, 12)) ENGINE = MergeTree() ORDER BY tuple()"); @@ -238,8 +246,6 @@ await client.InsertAsync( Console.WriteLine(); Console.WriteLine($" With Stride not a multiple of 8, the {(wide.BytesPerRow * 8) - wide.Dimension} unused bits are the high bits of byte 0."); } - - break; } } @@ -279,8 +285,6 @@ private static async Task ReducedPrecision(ClickHouseTcpClient client) Console.WriteLine($" The top row is exact, and equals what the IColumn view hands back:"); Console.WriteLine($" [{string.Join(", ", materialized.Select(value => value.ToString("0.####", CultureInfo.InvariantCulture)))}]"); } - - break; } Console.WriteLine(); @@ -348,7 +352,6 @@ private static async Task ElementTypes(ClickHouseTcpClient client, ClickHouseTcp }; Console.WriteLine($" {block["v"].TypeName,-18} {qbit.BitWidth,-8} {Describe(block["v"].ElementType),-9} {note}"); Console.WriteLine($" row 0 = [{string.Join(", ", ((System.Collections.IEnumerable)block["v"].GetValue(0)!).Cast().Select(Number))}]"); - break; } } catch (ClickHouseTcpServerException ex) @@ -401,9 +404,9 @@ private static async Task Strided(ClickHouseTcpClient client, ClickHouseTcpServe await client.ExecuteAsync($"CREATE TABLE {WideTable}_strided (v QBit(Float32, 8, 4)) ENGINE = MergeTree() ORDER BY tuple()"); await client.ExecuteAsync($"INSERT INTO {WideTable}_strided VALUES ([1, 2, 3, 4, 5, 6, 7, 8])"); + // Drained: the read is expected to fail, and the failure is the point. await foreach (Block _ in client.StreamAsync($"SELECT v FROM {WideTable}_strided")) { - break; } Console.WriteLine(" read, which this example did not expect"); diff --git a/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs b/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs index db2d5a64b..4f6e38572 100644 --- a/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs +++ b/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs @@ -49,6 +49,7 @@ public static async Task Run() private static async Task OneColumnPerTargetColumn(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); await client.ExecuteAsync($@" CREATE TABLE {TableName} ( @@ -137,6 +138,7 @@ await ShowRejection( private static async Task ANamedSubset(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {DefaultsTable}"); await client.ExecuteAsync($@" CREATE TABLE {DefaultsTable} ( @@ -169,6 +171,7 @@ await client.InsertAsync( private static async Task TheServerStatesTheType(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {InstantsTable}"); await client.ExecuteAsync($@" CREATE TABLE {InstantsTable} ( @@ -228,6 +231,7 @@ await ShowRejection( private static async Task BlockGeometry(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {BulkTable}"); await client.ExecuteAsync($@" CREATE TABLE {BulkTable} ( @@ -240,8 +244,8 @@ score Float64 Console.WriteLine("\n5. ClickHouseTcpInsertOptions.MaxRowsPerBlock\n"); Console.WriteLine(" One InsertAsync call is one statement, but not necessarily one wire block. The cap"); - Console.WriteLine(" splits the rows into blocks of at most that many, which bounds what the client holds"); - Console.WriteLine(" encoded at once. It defaults to 1,000,000 rows; null writes one block of any height.\n"); + Console.WriteLine(" splits the rows into blocks of at most that many. It is block geometry, not a memory"); + Console.WriteLine(" bound: it defaults to 1,000,000 rows, and null writes one block of any height.\n"); Console.WriteLine(" The same six rows, once split into three blocks and once written as one:\n"); Console.WriteLine(" MaxRowsPerBlock Rows stored Active parts"); @@ -254,7 +258,10 @@ score Float64 Console.WriteLine(" The cap is a client-side concern only. This server recombines the blocks of one insert"); Console.WriteLine(" before it writes, so the six rows land as one part either way: lowering the cap does not"); Console.WriteLine(" create parts and raising it does not remove them."); - Console.WriteLine(" Lower it to bound client memory on a very tall insert, and leave it alone otherwise."); + Console.WriteLine(" Leave it alone unless you want a particular block height. To bound the memory a large"); + Console.WriteLine(" insert costs, set ClickHouseTcpClientOptions.MaxSendBufferBytes: it flushes the buffered"); + Console.WriteLine(" bytes to the socket whenever they pass the cap, independent of block height. One column"); + Console.WriteLine(" larger than the cap still buffers in full."); } private static async Task SixRowsAndCountParts(ClickHouseTcpClient client, int? maxRowsPerBlock) @@ -284,7 +291,7 @@ private static async Task TheRowTierForComparison(ClickHouseTcpClient client) Console.WriteLine(" InsertRowsAsync takes one object[] per row and the same statement. It is the right"); Console.WriteLine(" call when the data really is row-shaped, and it differs in three ways:\n"); Console.WriteLine(" values are matched to the target columns by POSITION, not by name;"); - Console.WriteLine(" every value is boxed, which the caller pays for when it builds the rows;"); + Console.WriteLine(" each row is an object[], so every value-type value is boxed as the caller builds it;"); Console.WriteLine(" the client then transposes those rows into one typed column per target.\n"); await client.ExecuteAsync($"TRUNCATE TABLE {BulkTable}"); diff --git a/examples/Tcp/Write/Tcp_010_CompositeWrites.cs b/examples/Tcp/Write/Tcp_010_CompositeWrites.cs index 8d7af4d8f..2190060d9 100644 --- a/examples/Tcp/Write/Tcp_010_CompositeWrites.cs +++ b/examples/Tcp/Write/Tcp_010_CompositeWrites.cs @@ -56,6 +56,8 @@ public static async Task Run() private static async Task JaggedArrays(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {ArraysTable}"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {DenseTable}"); await client.ExecuteAsync(ArrayDdl(ArraysTable)); await client.ExecuteAsync(ArrayDdl(DenseTable)); @@ -167,6 +169,8 @@ private static async Task DenseArraysAndTheRoundTrip(ClickHouseTcpClient client) private static async Task TheOtherComposites(ClickHouseTcpClient client) { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {OthersTable}"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {OthersDenseTable}"); await client.ExecuteAsync(OthersDdl(OthersTable)); await client.ExecuteAsync(OthersDdl(OthersDenseTable)); @@ -276,9 +280,9 @@ private static void WhatToRemember() Console.WriteLine(" Nullable(T), and the plain value for LowCardinality(T)."); Console.WriteLine(" A row of Array(T) or Map(K, V) is never null. Use an empty array, or make the elements"); Console.WriteLine(" nullable."); - Console.WriteLine(" A column read out of a block is a valid insert column, and the fastest one: it is"); - Console.WriteLine(" already in the layout the codec writes from. Re-insert it inside the iteration that"); - Console.WriteLine(" yielded it, because the block is borrowed."); + Console.WriteLine(" A column read out of a block is a valid insert column, and re-inserts without its"); + Console.WriteLine(" composite layout being rebuilt: it is already in the layout the codec writes from."); + Console.WriteLine(" Re-insert it inside the iteration that yielded it, because the block is borrowed."); Console.WriteLine(" Match the column's name to the target, in the SELECT if need be."); } @@ -329,8 +333,9 @@ private static async Task ShowOthers(ClickHouseTcpClient client, string table) } } - // toString of a NULL is the empty string, which is indistinguishable from an empty string in a table. - private static string Text(object value) => value is string { Length: 0 } ? "NULL" : value?.ToString() ?? "NULL"; + // toString of a NULL is NULL, not the empty string: toString keeps the argument's nullability, so the row + // tier hands back a null reference here. + private static string Text(object value) => value?.ToString() ?? "NULL"; // Runs an insert that is expected to be rejected client-side and prints the reason. private static async Task ShowRejection(ClickHouseTcpClient client, string what, string sql, IReadOnlyList columns) From 084aafe3d3f3555880d7d38e33304956e01fc58b Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 20:20:31 +0200 Subject: [PATCH 13/16] Clarify and renumber TCP examples --- .../ClickHouseRawResultDecompressionTests.cs | 2 +- docs/overview.mdx | 4 +- examples/AGENTS.md | 6 +- examples/ExampleConfig.cs | 66 +- examples/ExamplePreflight.cs | 42 +- examples/ExampleRunner.cs | 77 +-- examples/Program.cs | 222 ++----- examples/README.md | 84 +-- .../Advanced/Tcp_001_SettingsAndQueryId.cs | 43 ++ .../Advanced/Tcp_002_ProgressAndStatistics.cs | 53 ++ examples/Tcp/Advanced/Tcp_003_Cancellation.cs | 49 ++ .../Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs | 98 +++ examples/Tcp/Advanced/Tcp_005_Compression.cs | 27 + examples/Tcp/Advanced/Tcp_006_ServerInfo.cs | 48 ++ .../Advanced/Tcp_020_SettingsAndQueryId.cs | 344 ----------- .../Advanced/Tcp_021_ProgressAndStatistics.cs | 247 -------- examples/Tcp/Advanced/Tcp_022_Cancellation.cs | 303 ---------- .../Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs | 411 ------------- examples/Tcp/Advanced/Tcp_024_Compression.cs | 350 ----------- examples/Tcp/Advanced/Tcp_025_ServerInfo.cs | 192 ------ examples/Tcp/Connection/Tcp_001_Sessions.cs | 43 ++ examples/Tcp/Connection/Tcp_002_PoolTuning.cs | 49 ++ examples/Tcp/Connection/Tcp_003_Tls.cs | 39 ++ examples/Tcp/Connection/Tcp_004_Timeouts.cs | 55 ++ examples/Tcp/Connection/Tcp_016_Sessions.cs | 209 ------- examples/Tcp/Connection/Tcp_017_PoolTuning.cs | 375 ------------ examples/Tcp/Connection/Tcp_018_Tls.cs | 259 -------- examples/Tcp/Connection/Tcp_019_Timeouts.cs | 323 ---------- examples/Tcp/Core/Tcp_001_BasicUsage.cs | 138 ++--- examples/Tcp/Core/Tcp_002_ConnectionString.cs | 163 +---- .../Tcp/Core/Tcp_003_DependencyInjection.cs | 136 +---- .../Tcp/Core/Tcp_004_MigratingFromHttp.cs | 207 ++----- examples/Tcp/Observability/Tcp_001_Logging.cs | 39 ++ .../Observability/Tcp_002_OpenTelemetry.cs | 64 ++ .../Observability/Tcp_003_MetadataBlocks.cs | 79 +++ .../Tcp/Observability/Tcp_004_HealthChecks.cs | 54 ++ .../Observability/Tcp_005_Testcontainers.cs | 78 +++ examples/Tcp/Observability/Tcp_026_Logging.cs | 347 ----------- .../Observability/Tcp_027_OpenTelemetry.cs | 395 ------------ .../Observability/Tcp_028_MetadataBlocks.cs | 336 ----------- .../Tcp/Observability/Tcp_029_HealthChecks.cs | 226 ------- .../Observability/Tcp_030_Testcontainers.cs | 144 ----- examples/Tcp/README.md | 81 +-- examples/Tcp/Read/Tcp_001_ReadTiers.cs | 80 +++ examples/Tcp/Read/Tcp_002_BlocksAndColumns.cs | 76 +++ examples/Tcp/Read/Tcp_003_Parameters.cs | 92 +++ examples/Tcp/Read/Tcp_004_Poco.cs | 81 +++ examples/Tcp/Read/Tcp_005_ReadTiers.cs | 303 ---------- examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs | 425 ------------- examples/Tcp/Read/Tcp_007_Parameters.cs | 409 ------------- examples/Tcp/Read/Tcp_008_Poco.cs | 282 --------- examples/Tcp/Types/Tcp_001_ScalarTypes.cs | 131 ++++ .../Tcp/Types/Tcp_002_DateTimeAndTimezones.cs | 112 ++++ examples/Tcp/Types/Tcp_003_CompositeRead.cs | 104 ++++ .../Tcp/Types/Tcp_004_VariantDynamicJson.cs | 137 +++++ .../Tcp/Types/Tcp_005_QBitVectorSearch.cs | 91 +++ examples/Tcp/Types/Tcp_011_ScalarTypes.cs | 472 --------------- .../Tcp/Types/Tcp_012_DateTimeAndTimezones.cs | 462 -------------- examples/Tcp/Types/Tcp_013_CompositeRead.cs | 567 ------------------ .../Tcp/Types/Tcp_014_VariantDynamicJson.cs | 431 ------------- .../Tcp/Types/Tcp_015_QBitVectorSearch.cs | 491 --------------- examples/Tcp/Write/Tcp_001_ColumnarInsert.cs | 66 ++ examples/Tcp/Write/Tcp_002_CompositeWrites.cs | 108 ++++ examples/Tcp/Write/Tcp_009_ColumnarInsert.cs | 389 ------------ examples/Tcp/Write/Tcp_010_CompositeWrites.cs | 354 ----------- 65 files changed, 2214 insertions(+), 9956 deletions(-) create mode 100644 examples/Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs create mode 100644 examples/Tcp/Advanced/Tcp_002_ProgressAndStatistics.cs create mode 100644 examples/Tcp/Advanced/Tcp_003_Cancellation.cs create mode 100644 examples/Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs create mode 100644 examples/Tcp/Advanced/Tcp_005_Compression.cs create mode 100644 examples/Tcp/Advanced/Tcp_006_ServerInfo.cs delete mode 100644 examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs delete mode 100644 examples/Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs delete mode 100644 examples/Tcp/Advanced/Tcp_022_Cancellation.cs delete mode 100644 examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs delete mode 100644 examples/Tcp/Advanced/Tcp_024_Compression.cs delete mode 100644 examples/Tcp/Advanced/Tcp_025_ServerInfo.cs create mode 100644 examples/Tcp/Connection/Tcp_001_Sessions.cs create mode 100644 examples/Tcp/Connection/Tcp_002_PoolTuning.cs create mode 100644 examples/Tcp/Connection/Tcp_003_Tls.cs create mode 100644 examples/Tcp/Connection/Tcp_004_Timeouts.cs delete mode 100644 examples/Tcp/Connection/Tcp_016_Sessions.cs delete mode 100644 examples/Tcp/Connection/Tcp_017_PoolTuning.cs delete mode 100644 examples/Tcp/Connection/Tcp_018_Tls.cs delete mode 100644 examples/Tcp/Connection/Tcp_019_Timeouts.cs create mode 100644 examples/Tcp/Observability/Tcp_001_Logging.cs create mode 100644 examples/Tcp/Observability/Tcp_002_OpenTelemetry.cs create mode 100644 examples/Tcp/Observability/Tcp_003_MetadataBlocks.cs create mode 100644 examples/Tcp/Observability/Tcp_004_HealthChecks.cs create mode 100644 examples/Tcp/Observability/Tcp_005_Testcontainers.cs delete mode 100644 examples/Tcp/Observability/Tcp_026_Logging.cs delete mode 100644 examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs delete mode 100644 examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs delete mode 100644 examples/Tcp/Observability/Tcp_029_HealthChecks.cs delete mode 100644 examples/Tcp/Observability/Tcp_030_Testcontainers.cs create mode 100644 examples/Tcp/Read/Tcp_001_ReadTiers.cs create mode 100644 examples/Tcp/Read/Tcp_002_BlocksAndColumns.cs create mode 100644 examples/Tcp/Read/Tcp_003_Parameters.cs create mode 100644 examples/Tcp/Read/Tcp_004_Poco.cs delete mode 100644 examples/Tcp/Read/Tcp_005_ReadTiers.cs delete mode 100644 examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs delete mode 100644 examples/Tcp/Read/Tcp_007_Parameters.cs delete mode 100644 examples/Tcp/Read/Tcp_008_Poco.cs create mode 100644 examples/Tcp/Types/Tcp_001_ScalarTypes.cs create mode 100644 examples/Tcp/Types/Tcp_002_DateTimeAndTimezones.cs create mode 100644 examples/Tcp/Types/Tcp_003_CompositeRead.cs create mode 100644 examples/Tcp/Types/Tcp_004_VariantDynamicJson.cs create mode 100644 examples/Tcp/Types/Tcp_005_QBitVectorSearch.cs delete mode 100644 examples/Tcp/Types/Tcp_011_ScalarTypes.cs delete mode 100644 examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs delete mode 100644 examples/Tcp/Types/Tcp_013_CompositeRead.cs delete mode 100644 examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs delete mode 100644 examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs create mode 100644 examples/Tcp/Write/Tcp_001_ColumnarInsert.cs create mode 100644 examples/Tcp/Write/Tcp_002_CompositeWrites.cs delete mode 100644 examples/Tcp/Write/Tcp_009_ColumnarInsert.cs delete mode 100644 examples/Tcp/Write/Tcp_010_CompositeWrites.cs diff --git a/ClickHouse.Driver.Tests/ADO/ClickHouseRawResultDecompressionTests.cs b/ClickHouse.Driver.Tests/ADO/ClickHouseRawResultDecompressionTests.cs index 98b2ffdda..b32118858 100644 --- a/ClickHouse.Driver.Tests/ADO/ClickHouseRawResultDecompressionTests.cs +++ b/ClickHouse.Driver.Tests/ADO/ClickHouseRawResultDecompressionTests.cs @@ -195,7 +195,7 @@ public async Task ReadDecompressedStreamAsync_WithUnsupportedCodec_LeavesTheBody /// /// Contrast case: the four original members are verbatim pass-throughs and must stay that way — - /// examples/Select/Select_005_CompressedRawExport.cs writes the compressed bytes to a file. + /// examples/Http/Select/Select_005_CompressedRawExport.cs writes the compressed bytes to a file. /// [Test] public async Task TheOriginalRawResultMembers_WithCompressedResponse_ReturnTheRawBytesVerbatim() diff --git a/docs/overview.mdx b/docs/overview.mdx index 17537a1ad..9e1591762 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -1079,7 +1079,7 @@ var json = await bodyReader.ReadToEndAsync(); Read the returned stream to completion before it goes out of scope, as above. When the response *is* compressed you get a decoder created with `leaveOpen`, so disposing it leaves the response intact; when it is **not** compressed you get the HTTP content stream itself, so disposing it ends the body. Either way the `ClickHouseRawResult` owns the response — don't call its other read members after the stream has been disposed. Disposing the `ClickHouseRawResult` is always required and on its own sufficient: it releases both the response and any decoder inserted here (decoders hold pooled buffers). The `await using` above is therefore optional, and safe to keep. Repeated sequential calls hand back the same stream; the type is not safe to use concurrently. -See [Select_007_ResponseCompression.cs](https://github.com/ClickHouse/clickhouse-cs/blob/main/examples/Select/Select_007_ResponseCompression.cs) for a runnable example. +See [Select_007_ResponseCompression.cs](https://github.com/ClickHouse/clickhouse-cs/blob/main/examples/Http/Select/Select_007_ResponseCompression.cs) for a runnable example. #### Insert (request) compression {#insert-compression} @@ -2741,7 +2741,7 @@ SqlMapper.AddTypeHandler(new BigIntegerHandler()); SqlMapper.AddTypeHandler(new IpAddressHandler()); ``` -See the [Dapper example](https://github.com/ClickHouse/clickhouse-cs/blob/main/examples/ORM/ORM_001_Dapper.cs) for an example type handler implementation. +See the [Dapper example](https://github.com/ClickHouse/clickhouse-cs/blob/main/examples/Http/ORM/ORM_001_Dapper.cs) for an example type handler implementation. #### Dapper.Contrib {#dapper-contrib} diff --git a/examples/AGENTS.md b/examples/AGENTS.md index bf62da49a..84158ff74 100644 --- a/examples/AGENTS.md +++ b/examples/AGENTS.md @@ -49,9 +49,11 @@ Take the server from `ExampleConfig`, which resolves environment variables over those would produce a duplicate. - `ExampleConfig.HttpBuilder()` to *change* one of those five. It returns a fresh builder each call. -Four examples are exempt, because configuration is their subject or they start their own server: +Seven examples are exempt, because configuration is their subject, they use a separate endpoint, or +they start their own server: `Core_002_ConnectionStringConfiguration`, `Core_003_DependencyInjection`, -`Testing_001_Testcontainers`, `Tcp_030_Testcontainers`. A literal connection string inside a comment, +`Auth_001_JwtAuthentication`, `Tables_003_CreateTableCloud`, `Testing_001_Testcontainers`, +`Tcp_003_Tls`, `Tcp_005_Testcontainers`. A literal connection string inside a comment, shown to teach the reader what one looks like, is also fine. Ask `ExampleConfig` for the endpoint with `HttpEndpoint` or `TcpEndpoint` when an example has to name diff --git a/examples/ExampleConfig.cs b/examples/ExampleConfig.cs index 2dbf0063d..f2ae2e572 100644 --- a/examples/ExampleConfig.cs +++ b/examples/ExampleConfig.cs @@ -4,40 +4,13 @@ namespace ClickHouse.Driver.Examples; -/// -/// The one place the examples get their server from, so that pointing the whole suite at a different -/// ClickHouse is a matter of environment variables rather than editing every file. -/// +/// Creates example clients from shared environment settings. /// -/// -/// Every value falls back to what a stock clickhouse/clickhouse-server container exposes on -/// localhost, so the examples run with nothing set. Override any of: -/// -/// -/// CLICKHOUSE_HOSTdefault localhost -/// CLICKHOUSE_HTTP_PORTdefault 8123 -/// CLICKHOUSE_TCP_PORTdefault 9000, the native protocol port -/// CLICKHOUSE_USERdefault default -/// CLICKHOUSE_PASSWORDdefault empty -/// CLICKHOUSE_DATABASEdefault default -/// -/// -/// CLICKHOUSE_HTTP_CONNECTION_STRING and CLICKHOUSE_TCP_CONNECTION_STRING replace the -/// whole assembled string for their transport, for a server the pieces above cannot describe — TLS, a -/// cloud endpoint, an extra setting. Either one also becomes what and -/// return, so an example that changes one key still starts from the override. -/// -/// -/// Four examples deliberately do not use this: Core_002_ConnectionStringConfiguration and -/// Core_003_DependencyInjection, whose subject is configuration itself, and -/// Testing_001_Testcontainers and Tcp_030_Testcontainers, which start their own servers. -/// +/// Component settings use local Docker defaults. A transport-specific connection string overrides +/// all components. See the examples README for the supported environment variables. /// public static class ExampleConfig { - // Private, and read only by the two component builders below. A whole-string override does not - // reach them, so an example that asked one of these for the endpoint would print, proxy or dial - // somewhere other than where it connected. HttpEndpoint and TcpEndpoint answer that question. private static readonly string Host = Env("CLICKHOUSE_HOST") ?? "localhost"; private static readonly ushort HttpPort = ushort.Parse(Env("CLICKHOUSE_HTTP_PORT") ?? "8123"); private static readonly ushort TcpPort = ushort.Parse(Env("CLICKHOUSE_TCP_PORT") ?? "9000"); @@ -53,24 +26,13 @@ public static class ExampleConfig public static string TcpConnectionString { get; } = Env("CLICKHOUSE_TCP_CONNECTION_STRING") ?? TcpFromComponents().ToString(); - /// - /// A builder pre-filled with the configured endpoint and credentials, for an example that has to - /// change one key. Each call returns a fresh builder. - /// - /// A builder describing the configured HTTP endpoint. + /// Creates an HTTP builder from the configured connection string. public static ClickHouseConnectionStringBuilder HttpBuilder() => new(HttpConnectionString); - /// - /// A builder pre-filled with the configured endpoint and credentials for the native protocol, for - /// an example that has to change one key. Each call returns a fresh builder. - /// - /// A builder describing the configured native endpoint. + /// Creates a native builder from the configured connection string. public static ClickHouseTcpConnectionStringBuilder TcpBuilder() => new(TcpConnectionString); - /// - /// The host and port the HTTP examples reach, for an example that has to name or dial the endpoint - /// itself rather than hand a connection string to a client. - /// + /// Gets the configured HTTP host and port. public static (string Host, ushort Port) HttpEndpoint { get @@ -80,32 +42,24 @@ public static (string Host, ushort Port) HttpEndpoint } } - /// The host and port the native examples dial. Not interchangeable with . + /// Gets the configured native host and port. public static (string Host, int Port) TcpEndpoint { get { var builder = TcpBuilder(); - // The builder reports no port when the connection string omits one, which is the native - // default rather than an absence. return (builder.Host, builder.Port ?? 9000); } } - /// Creates a client against the configured server. The caller disposes it. - /// A client for the configured HTTP endpoint. + /// Creates an HTTP client. The caller owns it. public static ClickHouseClient CreateHttpClient() => new(HttpConnectionString); - /// - /// Creates a native-protocol client against the configured server. The caller disposes it, - /// asynchronously where it can. - /// - /// A client for the configured native endpoint. + /// Creates a native client. The caller owns it. public static ClickHouseTcpClient CreateTcpClient() => new(TcpConnectionString); - /// Creates an ADO.NET connection against the configured server. The caller disposes it. - /// A connection for the configured HTTP endpoint. + /// Creates an ADO.NET connection. The caller owns it. public static ClickHouseConnection CreateHttpConnection() => new(HttpConnectionString); private static ClickHouseConnectionStringBuilder FromComponents() => new() diff --git a/examples/ExamplePreflight.cs b/examples/ExamplePreflight.cs index 87a37fd11..790a31dc6 100644 --- a/examples/ExamplePreflight.cs +++ b/examples/ExamplePreflight.cs @@ -3,29 +3,18 @@ namespace ClickHouse.Driver.Examples; -/// -/// Reaches the server once before any example runs, so that an unreachable or misconfigured endpoint -/// is reported as itself rather than as a failure inside whichever example happened to run first. -/// -/// -/// A failure exits non-zero instead of skipping. CI runs the whole suite with no filter, and a skip -/// would leave the run green while nothing had been exercised. -/// +/// Checks required endpoints before any example runs. public static class ExamplePreflight { private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); - /// - /// Checks the endpoints the given examples need, and reports what to fix if one is unreachable. - /// + /// Checks the endpoints required by a set of examples. /// The examples about to run. Only their transports are checked. /// True when every needed endpoint answered. public static Task CheckAsync(IEnumerable examples) => CheckAsync(examples.SelectMany(e => e.RequiredTransports).Distinct().ToArray()); - /// - /// Checks the named endpoints, and reports what to fix if one is unreachable. - /// + /// Checks the named endpoints. /// The transports to check. Duplicates are checked once. /// True when every named endpoint answered. public static async Task CheckAsync(params ExampleTransport[] transports) @@ -80,14 +69,23 @@ public static async Task CheckAsync(params ExampleTransport[] transports) private static void Report(ExampleTransport transport, string failure) { - // Reported from the effective endpoint, so that a whole-string override is described as itself - // rather than as whatever the component variables say. + // Report the effective endpoint, including whole-connection-string overrides. var http = ExampleConfig.HttpEndpoint; var tcp = ExampleConfig.TcpEndpoint; var (name, endpoint, user, port, source) = transport == ExampleTransport.Http - ? ("HTTP interface", $"{http.Host}:{http.Port}", ExampleConfig.HttpBuilder().Username, "CLICKHOUSE_HTTP_PORT", "CLICKHOUSE_HTTP_CONNECTION_STRING") - : ("native protocol", $"{tcp.Host}:{tcp.Port}", ExampleConfig.TcpBuilder().Username, "CLICKHOUSE_TCP_PORT", "CLICKHOUSE_TCP_CONNECTION_STRING"); + ? ( + "HTTP interface", + $"{http.Host}:{http.Port}", + ExampleConfig.HttpBuilder().Username, + "CLICKHOUSE_HTTP_PORT", + "CLICKHOUSE_HTTP_CONNECTION_STRING") + : ( + "native protocol", + $"{tcp.Host}:{tcp.Port}", + ExampleConfig.TcpBuilder().Username, + "CLICKHOUSE_TCP_PORT", + "CLICKHOUSE_TCP_CONNECTION_STRING"); Console.WriteLine(); Console.WriteLine($"Cannot reach ClickHouse on the {name} at {endpoint} as user '{user}'."); @@ -96,12 +94,16 @@ private static void Report(ExampleTransport transport, string failure) if (transport == ExampleTransport.Tcp) { - Console.WriteLine($" The native protocol listens on port 9000 by default, not on the HTTP port ({http.Port})."); + Console.WriteLine( + $" The native protocol listens on port 9000 by default, " + + $"not on the HTTP port ({http.Port})."); Console.WriteLine(); } Console.WriteLine(" Start a server with both ports published:"); - Console.WriteLine(" docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server"); + Console.WriteLine( + " docker run -d --name clickhouse-server " + + "-p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server"); Console.WriteLine(); Console.WriteLine(" Or point the examples somewhere else:"); Console.WriteLine($" CLICKHOUSE_HOST, {port}, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD, CLICKHOUSE_DATABASE"); diff --git a/examples/ExampleRunner.cs b/examples/ExampleRunner.cs index 7b7502f21..308873bb0 100644 --- a/examples/ExampleRunner.cs +++ b/examples/ExampleRunner.cs @@ -8,31 +8,31 @@ namespace ClickHouse.Driver.Examples; /// public static class ExampleRunner { - /// - /// Examples that talk to both interfaces, so their class-name prefix understates what they need. - /// Declared before _examples: static initializers run in order, and discovery reads this. - /// + // These sets must be initialized before _examples because discovery reads them. + // Preflight both endpoints for examples that use both clients. private static readonly HashSet _crossTransport = new(StringComparer.Ordinal) { "TcpMigratingFromHttp", - "TcpOpenTelemetry", }; - /// - /// Examples that start their own server, so the configured endpoint is not theirs and preflight must not - /// hold them up. Declared before _examples for the same reason as _crossTransport. - /// + // Self-contained examples do not need the configured server. private static readonly HashSet _selfContained = new(StringComparer.Ordinal) { "Testcontainers", "TcpTestcontainers", }; - /// - /// Examples needing infrastructure an ordinary server does not have, so neither an unfiltered run - /// nor a transport run includes them. An explicit --filter still reaches them. Keep this in - /// step with the list in AGENTS.md. - /// + // These examples configure their own endpoint instead of using ExampleConfig. + private static readonly HashSet _customEndpoint = new(StringComparer.Ordinal) + { + "ConnectionStringConfiguration", + "CreateTableCloud", + "DependencyInjection", + "JwtAuthentication", + "TcpTls", + }; + + // Run infrastructure-dependent examples only when a filter selects them explicitly. private static readonly HashSet _optIn = new(StringComparer.Ordinal) { "CreateTableCluster", @@ -52,29 +52,13 @@ public record ExampleInfo(string ClassName, Type Type, MethodInfo RunMethod) /// public string NormalizedName { get; } = Normalize(ClassName); - /// - /// Which transport the example is filed under. Read from the class name, because every - /// example shares one namespace and so a native-protocol example cannot reuse an HTTP - /// example's class name — the Tcp prefix that keeps them apart is the signal. - /// - public ExampleTransport Transport { get; } = ClassName.StartsWith("Tcp", StringComparison.Ordinal) - ? ExampleTransport.Tcp - : ExampleTransport.Http; + /// The transport indicated by the example's class-name prefix. + public ExampleTransport Transport { get; } = GetTransport(ClassName); - /// - /// Every endpoint the example needs to reach, which is not always the one it is filed under: - /// an example comparing the two transports needs both. - /// - public IReadOnlyList RequiredTransports { get; } = _selfContained.Contains(ClassName) - ? [] - : _crossTransport.Contains(ClassName) - ? [ExampleTransport.Http, ExampleTransport.Tcp] - : [ClassName.StartsWith("Tcp", StringComparison.Ordinal) ? ExampleTransport.Tcp : ExampleTransport.Http]; + /// The endpoints that preflight must check. + public IReadOnlyList RequiredTransports { get; } = GetRequiredTransports(ClassName); - /// - /// Whether a run that names no example includes it. False for one needing a cluster, Cloud - /// credentials or a token, which only an explicit filter should reach. - /// + /// Whether an unfiltered run includes this example. public bool RunsByDefault { get; } = !_optIn.Contains(ClassName); } @@ -121,8 +105,7 @@ public static async Task RunExample(ExampleInfo example) /// public static void ListExamples(ExampleTransport? transport = null) { - // Lists the opt-in examples too, marked. They are what a reader is most likely to be looking - // for by name, since no unfiltered run ever prints them. + // Keep opt-in examples discoverable even though an unfiltered run skips them. var listed = _examples.Where(e => transport is null || e.Transport == transport).ToList(); Console.WriteLine(transport is { } named ? $"Available {named} examples:\n" : "Available examples:\n"); @@ -198,6 +181,26 @@ private static string Normalize(string input) return input.Replace("_", "").Replace("-", "").ToLowerInvariant(); } + private static ExampleTransport GetTransport(string className) + => className.StartsWith("Tcp", StringComparison.Ordinal) + ? ExampleTransport.Tcp + : ExampleTransport.Http; + + private static IReadOnlyList GetRequiredTransports(string className) + { + if (_selfContained.Contains(className) || _customEndpoint.Contains(className)) + { + return []; + } + + if (_crossTransport.Contains(className)) + { + return [ExampleTransport.Http, ExampleTransport.Tcp]; + } + + return [GetTransport(className)]; + } + private static int GetSimilarityScore(string filter, string target) { int score = 0; diff --git a/examples/Program.cs b/examples/Program.cs index f15a01a9c..2073c8c94 100644 --- a/examples/Program.cs +++ b/examples/Program.cs @@ -48,8 +48,7 @@ static async Task Main(string[] args) private static async Task RunFiltered(string filter, ExampleTransport? transport, bool isInteractive) { - // A transport flag narrows the filter rather than being ignored: '--tcp --filter basicusage' - // otherwise also selects the HTTP BasicUsage and asks for an endpoint the caller did not name. + // A transport flag also limits fuzzy matches. var matches = ExampleRunner.FindMatches(filter, transport); if (matches.Count == 0) @@ -80,9 +79,7 @@ private static async Task RunTransport(ExampleTransport transport, bool isIntera Console.WriteLine($"Running {selected.Count} {transport} example(s):\n"); - // The named transport plus whatever the selection needs beyond it, so that asking for one - // with none written yet still reports whether its endpoint answers, and an example - // comparing the two transports still gets both checked. + // Include endpoints needed by cross-transport examples. var needed = selected.SelectMany(e => e.RequiredTransports).Append(transport).Distinct().ToArray(); if (!await ExamplePreflight.CheckAsync(needed)) @@ -343,160 +340,49 @@ private static async Task RunAllExamples(bool isInteractive) await Testcontainers.Run(); WaitForUser(isInteractive); - // Native Protocol: Core Usage & Configuration - Console.WriteLine("\n\n" + new string('=', 70)); - Console.WriteLine("NATIVE PROTOCOL: CORE USAGE & CONFIGURATION"); - Console.WriteLine(new string('=', 70) + "\n"); - - Console.WriteLine($"Running: {nameof(TcpBasicUsage)}"); - await TcpBasicUsage.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpConnectionString)}"); - await TcpConnectionString.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpDependencyInjection)}"); - await TcpDependencyInjection.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpMigratingFromHttp)}"); - await TcpMigratingFromHttp.Run(); - WaitForUser(isInteractive); - - // Native Protocol: Reading Data - Console.WriteLine("\n\n" + new string('=', 70)); - Console.WriteLine("NATIVE PROTOCOL: READING DATA"); - Console.WriteLine(new string('=', 70) + "\n"); - - Console.WriteLine($"Running: {nameof(TcpReadTiers)}"); - await TcpReadTiers.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpBlocksAndColumns)}"); - await TcpBlocksAndColumns.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpParameters)}"); - await TcpParameters.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpPoco)}"); - await TcpPoco.Run(); - WaitForUser(isInteractive); - - // Native Protocol: Writing Data - Console.WriteLine("\n\n" + new string('=', 70)); - Console.WriteLine("NATIVE PROTOCOL: WRITING DATA"); - Console.WriteLine(new string('=', 70) + "\n"); - - Console.WriteLine($"Running: {nameof(TcpColumnarInsert)}"); - await TcpColumnarInsert.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpCompositeWrites)}"); - await TcpCompositeWrites.Run(); - WaitForUser(isInteractive); - - // Native Protocol: Data Types - Console.WriteLine("\n\n" + new string('=', 70)); - Console.WriteLine("NATIVE PROTOCOL: DATA TYPES"); - Console.WriteLine(new string('=', 70) + "\n"); - - Console.WriteLine($"Running: {nameof(TcpScalarTypes)}"); - await TcpScalarTypes.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpDateTimeAndTimezones)}"); - await TcpDateTimeAndTimezones.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpCompositeRead)}"); - await TcpCompositeRead.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpVariantDynamicJson)}"); - await TcpVariantDynamicJson.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpQBitVectorSearch)}"); - await TcpQBitVectorSearch.Run(); - WaitForUser(isInteractive); - - // Native Protocol: Connections and Sessions - Console.WriteLine("\n\n" + new string('=', 70)); - Console.WriteLine("NATIVE PROTOCOL: CONNECTIONS AND SESSIONS"); - Console.WriteLine(new string('=', 70) + "\n"); - - Console.WriteLine($"Running: {nameof(TcpSessions)}"); - await TcpSessions.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpPoolTuning)}"); - await TcpPoolTuning.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpTls)}"); - await TcpTls.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpTimeouts)}"); - await TcpTimeouts.Run(); - WaitForUser(isInteractive); - - // Native Protocol: Advanced - Console.WriteLine("\n\n" + new string('=', 70)); - Console.WriteLine("NATIVE PROTOCOL: ADVANCED"); - Console.WriteLine(new string('=', 70) + "\n"); - - Console.WriteLine($"Running: {nameof(TcpSettingsAndQueryId)}"); - await TcpSettingsAndQueryId.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpProgressAndStatistics)}"); - await TcpProgressAndStatistics.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpCancellation)}"); - await TcpCancellation.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpErrorsAndRetries)}"); - await TcpErrorsAndRetries.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpCompression)}"); - await TcpCompression.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpServerInfo)}"); - await TcpServerInfo.Run(); - WaitForUser(isInteractive); - - // Native Protocol: Observability - Console.WriteLine("\n\n" + new string('=', 70)); - Console.WriteLine("NATIVE PROTOCOL: OBSERVABILITY"); - Console.WriteLine(new string('=', 70) + "\n"); - - Console.WriteLine($"Running: {nameof(TcpLogging)}"); - await TcpLogging.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpOpenTelemetry)}"); - await TcpOpenTelemetry.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpMetadataBlocks)}"); - await TcpMetadataBlocks.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpHealthChecks)}"); - await TcpHealthChecks.Run(); - WaitForUser(isInteractive); - - Console.WriteLine($"\n\nRunning: {nameof(TcpTestcontainers)}"); - await TcpTestcontainers.Run(); - WaitForUser(isInteractive); + await RunCategory("NATIVE PROTOCOL: CORE USAGE & CONFIGURATION", isInteractive, + (nameof(TcpBasicUsage), TcpBasicUsage.Run), + (nameof(TcpConnectionString), TcpConnectionString.Run), + (nameof(TcpDependencyInjection), TcpDependencyInjection.Run), + (nameof(TcpMigratingFromHttp), TcpMigratingFromHttp.Run)); + + await RunCategory("NATIVE PROTOCOL: READING DATA", isInteractive, + (nameof(TcpReadTiers), TcpReadTiers.Run), + (nameof(TcpBlocksAndColumns), TcpBlocksAndColumns.Run), + (nameof(TcpParameters), TcpParameters.Run), + (nameof(TcpPoco), TcpPoco.Run)); + + await RunCategory("NATIVE PROTOCOL: WRITING DATA", isInteractive, + (nameof(TcpColumnarInsert), TcpColumnarInsert.Run), + (nameof(TcpCompositeWrites), TcpCompositeWrites.Run)); + + await RunCategory("NATIVE PROTOCOL: DATA TYPES", isInteractive, + (nameof(TcpScalarTypes), TcpScalarTypes.Run), + (nameof(TcpDateTimeAndTimezones), TcpDateTimeAndTimezones.Run), + (nameof(TcpCompositeRead), TcpCompositeRead.Run), + (nameof(TcpVariantDynamicJson), TcpVariantDynamicJson.Run), + (nameof(TcpQBitVectorSearch), TcpQBitVectorSearch.Run)); + + await RunCategory("NATIVE PROTOCOL: CONNECTIONS AND SESSIONS", isInteractive, + (nameof(TcpSessions), TcpSessions.Run), + (nameof(TcpPoolTuning), TcpPoolTuning.Run), + (nameof(TcpTls), TcpTls.Run), + (nameof(TcpTimeouts), TcpTimeouts.Run)); + + await RunCategory("NATIVE PROTOCOL: ADVANCED", isInteractive, + (nameof(TcpSettingsAndQueryId), TcpSettingsAndQueryId.Run), + (nameof(TcpProgressAndStatistics), TcpProgressAndStatistics.Run), + (nameof(TcpCancellation), TcpCancellation.Run), + (nameof(TcpErrorsAndRetries), TcpErrorsAndRetries.Run), + (nameof(TcpCompression), TcpCompression.Run), + (nameof(TcpServerInfo), TcpServerInfo.Run)); + + await RunCategory("NATIVE PROTOCOL: OBSERVABILITY", isInteractive, + (nameof(TcpLogging), TcpLogging.Run), + (nameof(TcpOpenTelemetry), TcpOpenTelemetry.Run), + (nameof(TcpMetadataBlocks), TcpMetadataBlocks.Run), + (nameof(TcpHealthChecks), TcpHealthChecks.Run), + (nameof(TcpTestcontainers), TcpTestcontainers.Run)); Console.WriteLine("\n\n" + new string('=', 70)); Console.WriteLine("ALL EXAMPLES COMPLETED SUCCESSFULLY!"); @@ -566,4 +452,22 @@ private static void WaitForUser(bool isInteractive) Console.WriteLine(); // Just add a blank line in non-interactive mode } } + + private static async Task RunCategory( + string title, + bool isInteractive, + params (string Name, Func Run)[] examples) + { + Console.WriteLine("\n\n" + new string('=', 70)); + Console.WriteLine(title); + Console.WriteLine(new string('=', 70) + "\n"); + + foreach (var example in examples) + { + Console.WriteLine($"Running: {example.Name}"); + await example.Run(); + WaitForUser(isInteractive); + Console.WriteLine(); + } + } } diff --git a/examples/README.md b/examples/README.md index 39dbb8fd7..0129164bf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -101,60 +101,60 @@ Examples are grouped by transport. Everything under [Http/](Http) uses `ClickHou These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/README.md) first. -- [Tcp_001_BasicUsage.cs](Tcp/Core/Tcp_001_BasicUsage.cs) - Constructing `ClickHouseTcpClient`, DDL with `ExecuteAsync`, inserting with `InsertRowsAsync`, reading with `QueryAsync` and `ExecuteScalarAsync`, and disposal -- [Tcp_002_ConnectionString.cs](Tcp/Core/Tcp_002_ConnectionString.cs) - The native key set (compression codec, pool keys, TLS keys, no `Protocol`), `ClickHouseTcpConnectionStringBuilder`, and deriving an options variant with a `with` expression -- [Tcp_003_DependencyInjection.cs](Tcp/Core/Tcp_003_DependencyInjection.cs) - `AddClickHouseTcpDataSource`, injecting `IClickHouseTcpClient`, keyed registrations for two clusters, and who disposes the shared pool -- [Tcp_004_MigratingFromHttp.cs](Tcp/Core/Tcp_004_MigratingFromHttp.cs) - The same task over both transports, the call-for-call API mapping, the `CHTCP0001` opt-in, and what the native client cannot do +- [Tcp_001_BasicUsage.cs](Tcp/Core/Tcp_001_BasicUsage.cs) - Connect, create a table, insert rows, and query them +- [Tcp_002_ConnectionString.cs](Tcp/Core/Tcp_002_ConnectionString.cs) - Build and customize a native connection string +- [Tcp_003_DependencyInjection.cs](Tcp/Core/Tcp_003_DependencyInjection.cs) - Register shared and keyed native data sources +- [Tcp_004_MigratingFromHttp.cs](Tcp/Core/Tcp_004_MigratingFromHttp.cs) - Compare common HTTP and native client operations ### Native Protocol: Reading Data -- [Tcp_005_ReadTiers.cs](Tcp/Read/Tcp_005_ReadTiers.cs) - The three read tiers side by side — `QueryAsync` (boxed `object[]`), `QueryAsync` (POCO, converted), `StreamAsync` (columnar blocks) — what each allocates, and which to pick -- [Tcp_006_BlocksAndColumns.cs](Tcp/Read/Tcp_006_BlocksAndColumns.cs) - The block tier in depth: `Block.ColumnNames`, the indexers, `Column`, `IColumn` metadata, `ReadOnlySpan` values, `IDateTimeColumn`/`ITimeColumn`, `IArrayColumn`, and the borrowed-lifetime contract -- [Tcp_007_Parameters.cs](Tcp/Read/Tcp_007_Parameters.cs) - `ClickHouseTcpParameterCollection` and `ClickHouseTcpQueryOptions.Parameters`, plus the three traps: `{name:Type}` is required, an instant needs a declared timezone, and a parameter named after a server setting -- [Tcp_008_Poco.cs](Tcp/Read/Tcp_008_Poco.cs) - `QueryAsync` and `InsertRowsAsync` over one class, the name-matching rules, `[ClickHouseTcpColumn]`, `[ClickHouseTcpNotMapped]`, and what a mapping mismatch reports +- [Tcp_001_ReadTiers.cs](Tcp/Read/Tcp_001_ReadTiers.cs) - Read rows as arrays, POCOs, or columnar blocks +- [Tcp_002_BlocksAndColumns.cs](Tcp/Read/Tcp_002_BlocksAndColumns.cs) - Access typed block columns and copy borrowed data +- [Tcp_003_Parameters.cs](Tcp/Read/Tcp_003_Parameters.cs) - Bind typed values and identifiers +- [Tcp_004_Poco.cs](Tcp/Read/Tcp_004_Poco.cs) - Map query results and inserts to a POCO ### Native Protocol: Writing Data -- [Tcp_009_ColumnarInsert.cs](Tcp/Write/Tcp_009_ColumnarInsert.cs) - The columnar insert tier: `ClickHouseTcpColumn.Create` per target column plus `InsertAsync`, matching by name so the order is free, a named subset with the server filling the rest, why no ClickHouse type is ever stated, `MaxRowsPerBlock`, and how it differs from `InsertRowsAsync` -- [Tcp_010_CompositeWrites.cs](Tcp/Write/Tcp_010_CompositeWrites.cs) - Writing composites: the jagged and dense `Array(T)` shapes, re-inserting a column read out of a block with nothing rebuilt, the non-nullable-row rule, and `Map`, `Tuple`, `Nullable`, `LowCardinality` +- [Tcp_001_ColumnarInsert.cs](Tcp/Write/Tcp_001_ColumnarInsert.cs) - Insert typed columns and let ClickHouse fill defaults +- [Tcp_002_CompositeWrites.cs](Tcp/Write/Tcp_002_CompositeWrites.cs) - Insert composite values and reuse a column from a block ### Native Protocol: Data Types -- [Tcp_011_ScalarTypes.cs](Tcp/Types/Tcp_011_ScalarTypes.cs) - The CLR type of every scalar: the integer widths including `Int256`/`UInt256`, `BFloat16`'s lost precision, why the declared precision and not the value decides between `decimal` and `ClickHouseTcpDecimal`, `String` against `FixedString(N)`, and enums as bare ordinals -- [Tcp_012_DateTimeAndTimezones.cs](Tcp/Types/Tcp_012_DateTimeAndTimezones.cs) - `Date`, `Date32`, `DateTime`, `DateTime64(scale)`, `Time`, `Time64(scale)`: the stored count against the presented calendar value, where the presentation timezone comes from, what `DateTime.Kind` does on an insert, and why a parameter naming an instant needs a declared timezone -- [Tcp_013_CompositeRead.cs](Tcp/Types/Tcp_013_CompositeRead.cs) - Reading composites through `IArrayColumn`, `IMapColumn`, `ITupleColumn`, `INestedColumn`, `INullableColumn` and `ILowCardinalityColumn`, how they nest, and the geo aliases — which surface as `ValueTuple` where the HTTP driver builds `System.Tuple` -- [Tcp_014_VariantDynamicJson.cs](Tcp/Types/Tcp_014_VariantDynamicJson.cs) - `IVariantColumn` and `IDynamicColumn`: discriminators, local indices, the two different NULL markers, and typed dispatch without boxing — then `JSON`, which travels as text and comes back normalized, so what you write is not what you read -- [Tcp_015_QBitVectorSearch.cs](Tcp/Types/Tcp_015_QBitVectorSearch.cs) - `QBit(T, N)` and `IQBitColumn`: the transposed bit-plane layout, `GetPlane` and the bitmap byte order, rebuilding a vector from its top planes to match `L2DistanceTransposed`'s precision argument, and the padding a dimension that is not a multiple of 8 costs +- [Tcp_001_ScalarTypes.cs](Tcp/Types/Tcp_001_ScalarTypes.cs) - Inspect representative ClickHouse-to-CLR scalar mappings +- [Tcp_002_DateTimeAndTimezones.cs](Tcp/Types/Tcp_002_DateTimeAndTimezones.cs) - Read and write date, time, and timezone-aware values +- [Tcp_003_CompositeRead.cs](Tcp/Types/Tcp_003_CompositeRead.cs) - Read arrays, maps, tuples, nullable values, and nested data +- [Tcp_004_VariantDynamicJson.cs](Tcp/Types/Tcp_004_VariantDynamicJson.cs) - Read `Variant`, `Dynamic`, and `JSON` columns +- [Tcp_005_QBitVectorSearch.cs](Tcp/Types/Tcp_005_QBitVectorSearch.cs) - Inspect `QBit` planes and run approximate vector search ### Native Protocol: Connections and Sessions -- [Tcp_016_Sessions.cs](Tcp/Connection/Tcp_016_Sessions.cs) - `OpenSessionAsync`: one pinned connection, so a temporary table and a `SET` survive from one operation to the next, `IsOpen`, one operation at a time, disposal closing rather than pooling the connection, and `SET ROLE` as the native answer to HTTP's per-query `Roles` -- [Tcp_017_PoolTuning.cs](Tcp/Connection/Tcp_017_PoolTuning.cs) - `MinPoolSize`, `MaxPoolSize`, `PoolTimeout`, `IdleTimeout`, `MaxConnectionLifetime`, `SweepInterval` and `PoolReusePolicy`, measured: concurrency actually capped, `PoolTimeout` expiring, the sweep retiring and topping up, `Lifo` against `Fifo`, and what a `ClickHouseTcpDataSource` shares -- [Tcp_018_Tls.cs](Tcp/Connection/Tcp_018_Tls.cs) - `UseTls`, `TlsServerName`, `TlsCaCertificatePath` (which replaces the host trust store rather than adding to it), `TlsAllowInvalidCertificates`, `ConfigureTls`, the default port moving to 9440, and the TLS mistakes the constructor refuses before anything connects -- [Tcp_019_Timeouts.cs](Tcp/Connection/Tcp_019_Timeouts.cs) - `DialTimeout`, `ReadTimeout` as an idle deadline rather than a time limit, `PoolTimeout`, `StatementMaxLength` in the log line, `MaxSendBufferBytes`, and where a `CancellationToken` takes over +- [Tcp_001_Sessions.cs](Tcp/Connection/Tcp_001_Sessions.cs) - Keep temporary tables and settings in one session +- [Tcp_002_PoolTuning.cs](Tcp/Connection/Tcp_002_PoolTuning.cs) - Configure pool size, reuse, and checkout timeouts +- [Tcp_003_Tls.cs](Tcp/Connection/Tcp_003_Tls.cs) - Configure TLS and certificate validation +- [Tcp_004_Timeouts.cs](Tcp/Connection/Tcp_004_Timeouts.cs) - Set connection and read timeouts, then cancel a query ### Native Protocol: Advanced -- [Tcp_020_SettingsAndQueryId.cs](Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs) - Client-level `CustomSettings` against per-query `ClickHouseTcpQueryOptions.Settings` and the precedence between them, a misspelled setting name being ignored rather than refused, `QueryId` in `system.query_log` and what reusing one does, and `async_insert` as a setting that changes what an insert means -- [Tcp_021_ProgressAndStatistics.cs](Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs) - `ClickHouseTcpQueryCallbacks`: `OnProgress` interleaved with the rows as the query runs, why every counter is an increment, `OnProfileInfo`'s once-per-query summary, `OnProfileEvents`' increments and gauges, and the callback contract — synchronous, on the draining thread, and never allowed to throw -- [Tcp_022_Cancellation.cs](Tcp/Advanced/Tcp_022_Cancellation.cs) - A `CancellationToken` through `QueryAsync`, `StreamAsync` and `ExecuteAsync`: what the caller catches, the cancellation the server logs, why the connection is closed rather than pooled, why abandoning a result is the same thing, and how it differs from `ReadTimeout` and `max_execution_time` -- [Tcp_023_ErrorsAndRetries.cs](Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs) - `ClickHouseTcpServerException` (`Code`, `RawCode`, `Name`, `ServerStackTrace`), `ClickHouseTcpTransportException`, `ClickHouseTcpProtocolException`, switching on `ClickHouseErrorCode`, what `IsTransient` does and does not promise, a retry that recovers, and why retrying an insert needs `insert_deduplication_token` and a table that can deduplicate -- [Tcp_024_Compression.cs](Tcp/Advanced/Tcp_024_Compression.cs) - `Compression=lz4|zstd|none` and `ClickHouseTcpClientOptions.Compressor`, measured in bytes on the wire: what LZ4 saves by default, why the client's codec does not choose what the server sends (`network_compression_method` does), what it does choose on an insert, and why loopback cannot measure the benefit -- [Tcp_025_ServerInfo.cs](Tcp/Advanced/Tcp_025_ServerInfo.cs) - `GetServerInfoAsync` and every field of `ClickHouseTcpServerInfo`, the build number `Version` does not carry, gating on `ProtocolRevision` (query parameters need 54459) with one gate that passes and one that does not, and gating on the server version with a printed skip +- [Tcp_001_SettingsAndQueryId.cs](Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs) - Apply settings and assign a query ID +- [Tcp_002_ProgressAndStatistics.cs](Tcp/Advanced/Tcp_002_ProgressAndStatistics.cs) - Receive progress and profile callbacks +- [Tcp_003_Cancellation.cs](Tcp/Advanced/Tcp_003_Cancellation.cs) - Cancel row, block, and command operations +- [Tcp_004_ErrorsAndRetries.cs](Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs) - Handle errors and retry transient reads safely +- [Tcp_005_Compression.cs](Tcp/Advanced/Tcp_005_Compression.cs) - Select a native compression codec +- [Tcp_006_ServerInfo.cs](Tcp/Advanced/Tcp_006_ServerInfo.cs) - Read handshake metadata and gate optional features ### Native Protocol: Observability -- [Tcp_026_Logging.cs](Tcp/Observability/Tcp_026_Logging.cs) - `ClickHouseTcpClientOptions.LoggerFactory` and the three categories `ClickHouseTcpDiagnostics.ClientLogCategory`, `.ConnectionLogCategory`, `.PoolLogCategory`: what each reports, which levels they use (nothing at `Information`), why a stock `ILoggerFactory` shows one line, and two measured filter sets — production against debugging a connection problem -- [Tcp_027_OpenTelemetry.cs](Tcp/Observability/Tcp_027_OpenTelemetry.cs) - `ClickHouseTcpDiagnostics.ActivitySourceName` and `IncludeSqlInActivityTags`, with spans collected and printed: the span names and attributes, `connect` nested under the statement that dialled, the W3C trace context the client propagates so the server's own spans join the trace, and why the source is separate from the HTTP transport's -- [Tcp_028_MetadataBlocks.cs](Tcp/Observability/Tcp_028_MetadataBlocks.cs) - The three `Block`-shaped callbacks Tcp_021 does not cover — `OnLog` with `send_logs_level`, `OnTotals` for `WITH TOTALS`, `OnExtremes` with `extremes` — the borrowed-block rule of copying inside the callback, the log priority scale, and what each `send_logs_level` costs -- [Tcp_029_HealthChecks.cs](Tcp/Observability/Tcp_029_HealthChecks.cs) - `PingAsync` as a health check over an `AddClickHouseTcpDataSource` registration: a protocol ping measured against `SELECT 1`, Healthy, Degraded and Unhealthy in one report, and what a Pong does and does not prove -- [Tcp_030_Testcontainers.cs](Tcp/Observability/Tcp_030_Testcontainers.cs) - A throwaway ClickHouse over the native protocol: the mapped 9000 rather than `GetConnectionString()`'s 8123, why a native-port wait strategy alone reports ready too early, and a query against the container +- [Tcp_001_Logging.cs](Tcp/Observability/Tcp_001_Logging.cs) - Configure diagnostic log categories and levels +- [Tcp_002_OpenTelemetry.cs](Tcp/Observability/Tcp_002_OpenTelemetry.cs) - Export native client activities with OpenTelemetry +- [Tcp_003_MetadataBlocks.cs](Tcp/Observability/Tcp_003_MetadataBlocks.cs) - Receive server logs, totals, and extremes +- [Tcp_004_HealthChecks.cs](Tcp/Observability/Tcp_004_HealthChecks.cs) - Use native protocol pings in an ASP.NET Core health check +- [Tcp_005_Testcontainers.cs](Tcp/Observability/Tcp_005_Testcontainers.cs) - Run a native client test against a temporary container ## How to run ### Prerequisites -- .NET 9.0 SDK or later +- .NET 10.0 SDK or later - ClickHouse server (local or remote) - For local runs, you can use Docker: ```bash @@ -186,15 +186,16 @@ dotnet run -- --filter basicusage dotnet run -- basicusage ``` -Before running anything, the runner reaches the endpoints the selected examples need and reports what to fix if one does not answer, rather than letting the first example fail with a connection error. +Before running an example that uses `ExampleConfig`, the runner checks its endpoint and reports what +to fix if it cannot connect. Examples that configure or start their own server skip this preflight. The filter matches the example's class name, which `--list` prints. A class name is the topic without the file's category prefix: `Core_001_BasicUsage.cs` declares `class BasicUsage`. Matching ignores case and underscores and accepts any substring, so `basicusage`, `basic` and `usage` all match it. The file's `core001` prefix does not. ### Connection configuration -Every example takes its server from [ExampleConfig.cs](ExampleConfig.cs), apart from the four listed -below, so one environment variable points the whole suite somewhere else. The defaults are what a -stock server container exposes on localhost, and the examples run with nothing set. +Most examples take their live server endpoint from [ExampleConfig.cs](ExampleConfig.cs). The seven +exceptions are listed below. The defaults are what a stock server container exposes on localhost, +and the examples run with nothing set. | Variable | Default | | --- | --- | @@ -212,12 +213,15 @@ For an endpoint those pieces cannot describe — TLS, a cloud host, an extra set CLICKHOUSE_HOST=my-server CLICKHOUSE_PASSWORD=secret dotnet run -- basicusage ``` -Four examples do not read `ExampleConfig`. Two keep literal connection strings because configuration -is what they teach: +Seven examples do not rely on `ExampleConfig` for the endpoint they connect to. Two keep literal +connection strings because configuration is what they teach: [Core_002_ConnectionStringConfiguration.cs](Http/Core/Core_002_ConnectionStringConfiguration.cs) and -[Core_003_DependencyInjection.cs](Http/Core/Core_003_DependencyInjection.cs). Two start their own -server and address that: [Testing_001_Testcontainers.cs](Http/Testing/Testing_001_Testcontainers.cs) -and [Tcp_030_Testcontainers.cs](Tcp/Observability/Tcp_030_Testcontainers.cs). +[Core_003_DependencyInjection.cs](Http/Core/Core_003_DependencyInjection.cs). Three use their own +environment variables: [Auth_001_JwtAuthentication.cs](Http/Core/Auth_001_JwtAuthentication.cs) and +[Tables_003_CreateTableCloud.cs](Http/Tables/Tables_003_CreateTableCloud.cs), plus +[Tcp_003_Tls.cs](Tcp/Connection/Tcp_003_Tls.cs). Two start their own server: +[Testing_001_Testcontainers.cs](Http/Testing/Testing_001_Testcontainers.cs) and +[Tcp_005_Testcontainers.cs](Tcp/Observability/Tcp_005_Testcontainers.cs). ### ClickHouse Cloud diff --git a/examples/Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs b/examples/Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs new file mode 100644 index 000000000..a20c6bb80 --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs @@ -0,0 +1,43 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Applies client and query settings and assigns a query ID. +public static class TcpSettingsAndQueryId +{ + public static async Task Run() + { + var builder = ExampleConfig.TcpBuilder(); + + // A set_ connection-string key becomes a default setting for every query. + builder["set_max_threads"] = 2; + + await using var client = new ClickHouseTcpClient(builder.ToOptions()); + + object clientSetting = await client.ExecuteScalarAsync("SELECT getSetting('max_threads')"); + Console.WriteLine($"Client default max_threads: {clientSetting}"); + + // Query options override client defaults for this operation only. + var queryOptions = new ClickHouseTcpQueryOptions + { + QueryId = $"example-tcp-settings-{Guid.NewGuid():N}", + Settings = new Dictionary + { + ["max_threads"] = "4", + ["max_execution_time"] = "10", + }, + }; + + object querySetting = await client.ExecuteScalarAsync( + "SELECT getSetting('max_threads')", + queryOptions); + object queryId = await client.ExecuteScalarAsync("SELECT currentQueryID()", queryOptions); + + Console.WriteLine($"Per-query max_threads: {querySetting}"); + Console.WriteLine($"Requested query ID: {queryOptions.QueryId}"); + Console.WriteLine($"Server query ID: {queryId}"); + + object nextSetting = await client.ExecuteScalarAsync("SELECT getSetting('max_threads')"); + Console.WriteLine($"Next query uses the client default again: {nextSetting}"); + } +} diff --git a/examples/Tcp/Advanced/Tcp_002_ProgressAndStatistics.cs b/examples/Tcp/Advanced/Tcp_002_ProgressAndStatistics.cs new file mode 100644 index 000000000..1796a869e --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_002_ProgressAndStatistics.cs @@ -0,0 +1,53 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Receives progress and profile callbacks while a query runs. +public static class TcpProgressAndStatistics +{ + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + ClickHouseTcpProgress total = default; + ClickHouseTcpProfileInfo profile = default; + int progressUpdates = 0; + int profileEventBlocks = 0; + + var options = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary + { + ["interactive_delay"] = "30000", + ["max_block_size"] = "1", + }, + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnProgress = progress => + { + // Progress values are increments, so add them to get a query-wide total. + total += progress; + progressUpdates++; + Console.WriteLine($"Progress: +{progress.Rows} rows"); + }, + OnProfileInfo = info => profile = info, + OnProfileEvents = _ => profileEventBlocks++, + }, + }; + + int rows = 0; + await foreach (object[] _ in client.QueryAsync( + "SELECT number, sleepEachRow(0.04) FROM numbers(8)", + options)) + { + rows++; + } + + Console.WriteLine($"Rows read: {rows}"); + Console.WriteLine($"Progress updates: {progressUpdates}; reported rows: {total.Rows}"); + Console.WriteLine($"Profile rows: {profile.Rows}; blocks: {profile.Blocks}"); + Console.WriteLine($"Profile event blocks: {profileEventBlocks}"); + + // Callbacks run synchronously while the response is read. Keep them fast and do not throw. + } +} diff --git a/examples/Tcp/Advanced/Tcp_003_Cancellation.cs b/examples/Tcp/Advanced/Tcp_003_Cancellation.cs new file mode 100644 index 000000000..dcc9cc37d --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_003_Cancellation.cs @@ -0,0 +1,49 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Cancels native-protocol operations with a CancellationToken. +public static class TcpCancellation +{ + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + using var cancellation = new CancellationTokenSource(); + + int rows = 0; + try + { + await foreach (object[] _ in client.QueryAsync( + "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 1", + cancellationToken: cancellation.Token)) + { + rows++; + if (rows == 3) + { + cancellation.Cancel(); + } + } + } + catch (OperationCanceledException) + { + Console.WriteLine($"Cancelled after {rows} rows."); + } + + // Cancellation abandons the response, so its connection is not returned to the pool. + // The client remains usable and opens or reuses another connection. + object value = await client.ExecuteScalarAsync("SELECT 'still usable'"); + Console.WriteLine(value); + + using var deadline = new CancellationTokenSource(TimeSpan.FromMilliseconds(150)); + try + { + await client.ExecuteAsync( + "SELECT sleep(1)", + cancellationToken: deadline.Token); + } + catch (OperationCanceledException) + { + Console.WriteLine("ExecuteAsync reached its operation deadline."); + } + } +} diff --git a/examples/Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs b/examples/Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs new file mode 100644 index 000000000..48673f7d1 --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs @@ -0,0 +1,98 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Handles server, transport, and protocol errors and retries transient reads. +public static class TcpErrorsAndRetries +{ + private const string TableName = "example_tcp_retry_deduplication"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + string missingTable = $"example_tcp_missing_{Guid.NewGuid():N}"; + + try + { + await client.ExecuteScalarAsync($"SELECT * FROM {missingTable}"); + } + catch (ClickHouseTcpServerException ex) + { + Console.WriteLine( + $"Server error: {ex.Code} ({ex.RawCode}), transient={ex.IsTransient}"); + } + + await using var unreachable = new ClickHouseTcpClient( + ExampleConfig.TcpBuilder().ToOptions() with + { + Port = 1, + DialTimeout = TimeSpan.FromSeconds(1), + }); + + // A transient read is safe to retry. A failed write may already have reached the server. + int attempts = 0; + object result = await RetryRead(async () => + { + attempts++; + return attempts == 1 + ? await unreachable.ExecuteScalarAsync("SELECT 1") + : await client.ExecuteScalarAsync("SELECT 1"); + }); + Console.WriteLine($"Read succeeded after {attempts} attempts: {result}"); + + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} (id UInt64) + ENGINE = MergeTree + ORDER BY id + SETTINGS non_replicated_deduplication_window = 100 + """); + + object[][] batch = { new object[] { 1UL }, new object[] { 2UL } }; + // Reuse one token for retries of the same logical batch. The table must enable deduplication. + var insertOptions = new ClickHouseTcpInsertOptions + { + Settings = new Dictionary + { + ["insert_deduplication_token"] = "example-logical-batch-1", + }, + }; + + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id) VALUES", + batch, + insertOptions); + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id) VALUES", + batch, + insertOptions); + + object count = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + Console.WriteLine($"Rows after retrying the same deduplicated insert: {count}"); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } + + private static async Task RetryRead(Func> operation) + { + const int MaxAttempts = 3; + + for (int attempt = 1; ; attempt++) + { + try + { + return await operation(); + } + catch (ClickHouseTcpException ex) when (ex.IsTransient && attempt < MaxAttempts) + { + Console.WriteLine($"Transient {ex.GetType().Name}; retrying."); + await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt)); + } + } + } +} diff --git a/examples/Tcp/Advanced/Tcp_005_Compression.cs b/examples/Tcp/Advanced/Tcp_005_Compression.cs new file mode 100644 index 000000000..7a2a4e1a6 --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_005_Compression.cs @@ -0,0 +1,27 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Selects LZ4, Zstandard, or no native block compression. +public static class TcpCompression +{ + public static async Task Run() + { + // Compression applies to native data blocks; it does not change query results. + foreach (string codec in new[] { "lz4", "zstd", "none" }) + { + var builder = ExampleConfig.TcpBuilder(); + builder.Compression = codec; + ClickHouseTcpClientOptions options = builder.ToOptions(); + + await using var client = new ClickHouseTcpClient(options); + object rows = await client.ExecuteScalarAsync("SELECT count() FROM numbers(10000)"); + + string compressor = options.Compressor?.GetType().Name ?? "none"; + Console.WriteLine($"Compression={codec,-4} -> {compressor,-20}; rows={rows}"); + } + + Console.WriteLine("LZ4 is the default. Zstandard usually trades more CPU for smaller payloads."); + Console.WriteLine("Choose a codec with measurements from your workload and network."); + } +} diff --git a/examples/Tcp/Advanced/Tcp_006_ServerInfo.cs b/examples/Tcp/Advanced/Tcp_006_ServerInfo.cs new file mode 100644 index 000000000..95d928c7a --- /dev/null +++ b/examples/Tcp/Advanced/Tcp_006_ServerInfo.cs @@ -0,0 +1,48 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Reads server identity and negotiated protocol details from the handshake. +public static class TcpServerInfo +{ + private const int ParametersRevision = 54459; + private static readonly Version QBitFrom = new(25, 11); + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + + Console.WriteLine($"Name: {server.Name}"); + Console.WriteLine($"Version: {server.Version}"); + Console.WriteLine($"Protocol revision: {server.ProtocolRevision}"); + Console.WriteLine($"Timezone: {server.Timezone}"); + Console.WriteLine($"Display name: {server.DisplayName}"); + + // Gate wire-level features on the negotiated protocol revision. + if (server.ProtocolRevision >= ParametersRevision) + { + var options = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "minimum", 90UL } }, + }; + object count = await client.ExecuteScalarAsync( + "SELECT count() FROM numbers(100) WHERE number >= {minimum:UInt64}", + options); + Console.WriteLine($"Parameterized query result: {count}"); + } + else + { + Console.WriteLine("This connection does not support query parameters."); + } + + // Gate SQL features, such as data types and functions, on the server version. + Console.WriteLine( + server.Version >= QBitFrom + ? "QBit is available on this server." + : $"QBit requires ClickHouse {QBitFrom} or newer."); + + object exactVersion = await client.ExecuteScalarAsync("SELECT version()"); + Console.WriteLine($"Exact server build: {exactVersion}"); + } +} diff --git a/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs b/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs deleted file mode 100644 index 7107d3fea..000000000 --- a/examples/Tcp/Advanced/Tcp_020_SettingsAndQueryId.cs +++ /dev/null @@ -1,344 +0,0 @@ -using System.Diagnostics; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// The two places a ClickHouse setting can be set — for -/// every operation the client runs, for one — and -/// , which is how a query is found again in -/// system.query_log or stopped with KILL QUERY. -/// -/// -/// Tcp_002_ConnectionString sets a client-level setting from a connection string key and reads it back; -/// this example is about what happens when both levels name the same setting, and about the settings that change -/// how an operation behaves rather than only what it reports. async_insert is the worked example. -/// -/// -public static class TcpSettingsAndQueryId -{ - private const string TableName = "example_tcp_async_insert"; - - public static async Task Run() - { - // Two client-level settings, from the set_ keys of the connection string. Tcp_002 covers the - // spelling; what matters here is that they are the client's defaults for every operation. - var builder = ExampleConfig.TcpBuilder(); - builder["set_max_threads"] = 2; - builder["set_max_block_size"] = 4096; - - await using var client = new ClickHouseTcpClient(builder.ToOptions()); - - await TwoLevels(client); - await AMisspelledNameIsIgnored(client); - await QueryIdInTheLog(client); - await ReusingAQueryId(client); - - try - { - await AsyncInsert(client); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped {TableName}."); - } - } - - private static async Task TwoLevels(ClickHouseTcpClient client) - { - Console.WriteLine("1. Client-level settings against per-query settings\n"); - - // getSetting reports the value in force for the query asking, which makes the precedence observable. - const string sql = "SELECT getSetting('max_threads')::String, getSetting('max_block_size')::String"; - - Console.WriteLine($" Options.CustomSettings {string.Join(", ", client.Options.CustomSettings.Select(s => $"{s.Key}={s.Value}"))}"); - Console.WriteLine($" no per-query options {await Pair(client, sql, null)}"); - - // Only max_threads is named twice, and only max_threads changes. - var oneKey = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["max_threads"] = "7" }, - }; - Console.WriteLine($" Settings max_threads=7 {await Pair(client, sql, oneKey)}"); - Console.WriteLine($" the next query {await Pair(client, sql, null)}"); - Console.WriteLine(); - Console.WriteLine(" A per-query value replaces the client-level one for that key alone: max_block_size"); - Console.WriteLine(" kept the client's 4096. And it applies to one operation — nothing is left behind on"); - Console.WriteLine(" the connection, because the settings travel in the query packet rather than as a SET."); - Console.WriteLine(); - Console.WriteLine(" To carry a setting across operations, put it on the client, or run SET inside a"); - Console.WriteLine(" session (Tcp_016), which pins one connection and so can hold session state."); - Console.WriteLine(); - Console.WriteLine(" Settings is IReadOnlyDictionary: every value is text, so a number is"); - Console.WriteLine(" spelled \"7\". HTTP's QueryOptions.CustomSettings takes object instead."); - } - - private static async Task Pair(ClickHouseTcpClient client, string sql, ClickHouseTcpQueryOptions? options) - { - await foreach (object[] row in client.QueryAsync(sql, options)) - { - return $"max_threads={row[0],-3} max_block_size={row[1]}"; - } - - return "(no row)"; - } - - private static async Task AMisspelledNameIsIgnored(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. A name the server does not know is ignored, not refused\n"); - - // The settings list is not validated against the server's setting names, so a typo costs nothing and - // does nothing. There is no client-side check either: the name is whatever string you passed. - object value = await client.ExecuteScalarAsync("SELECT 1", new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["maxx_threads"] = "7" }, - }); - - Console.WriteLine($" Settings[\"maxx_threads\"] = \"7\", then SELECT 1 -> {value}, no error at all."); - - // A value that cannot be parsed as the setting's type does fail, which is the only feedback there is. - // - // On its own throwaway client, deliberately. The server raises this error while it is still reading the - // settings list — before it has accepted the query — and closes the socket, which the pool does not - // notice, so the connection goes back into the pool dead and the *next* operation on this client fails - // with a ClickHouseTcpTransportException. Scoping it to a client that is disposed here disposes the dead - // connection with it. Tcp_007 does the same for the same reason. - await using (var throwaway = new ClickHouseTcpClient(client.Options)) - { - try - { - await throwaway.ExecuteScalarAsync("SELECT 1", new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["max_threads"] = "lots" }, - }); - Console.WriteLine(" max_threads = \"lots\" was accepted, which is not what this example expected"); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" Settings[\"max_threads\"] = \"lots\" -> {ex.Code} ({ex.RawCode}): {FirstLine(ex.Message)}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" So a wrong name is silent and a wrong value is loud. Read a setting back with"); - Console.WriteLine(" getSetting('name') when it matters that it arrived."); - Console.WriteLine(); - Console.WriteLine(" That second query ran on a client of its own, because a bad setting value is refused"); - Console.WriteLine(" before the query is accepted and the server closes the connection on its way out. An"); - Console.WriteLine(" ordinary query error — a syntax error, an unknown table — leaves the connection usable,"); - Console.WriteLine(" and Tcp_023 shows that; this one does not."); - } - - private static async Task QueryIdInTheLog(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. QueryId, and finding the query again\n"); - - // Unique per run: a query id is the key of a system.query_log row, and two runs of this example against - // one server must not collide. - string queryId = $"example-tcp-020-{Guid.NewGuid():N}"; - var options = new ClickHouseTcpQueryOptions { QueryId = queryId }; - - object rows = await client.ExecuteScalarAsync("SELECT count() FROM numbers(100000)", options); - Console.WriteLine($" Ran SELECT count() FROM numbers(100000) as query_id = {queryId}"); - Console.WriteLine($" result {rows}"); - - // The QueryFinish record is queued independently of the response reaching the client, so a flush issued - // straight after the query can miss it. Retry the flush and the read rather than sleeping. - string found = await ReadLog( - client, - "SELECT type::String || ' read_rows=' || toString(read_rows) || ' threads=' || Settings['max_threads'] " + - "FROM system.query_log WHERE query_id = {id:String} AND type = 'QueryFinish'", - queryId); - - Console.WriteLine($" system.query_log by query_id: {found}"); - Console.WriteLine(); - Console.WriteLine(" The Settings column holds the settings the query ran with, client-level ones"); - Console.WriteLine(" included, which is the other reason to set a query id: it is the only handle that"); - Console.WriteLine(" ties an application's own request to a server-side row. It is also what"); - Console.WriteLine(" KILL QUERY WHERE query_id = '...' takes."); - } - - private static async Task ReusingAQueryId(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. Reusing one\n"); - - string queryId = $"example-tcp-020-reuse-{Guid.NewGuid():N}"; - var options = new ClickHouseTcpQueryOptions { QueryId = queryId }; - - await client.ExecuteScalarAsync("SELECT 1", options); - await client.ExecuteScalarAsync("SELECT 2", options); - Console.WriteLine(" Two queries, one after the other, under the same id: both accepted. The id is not"); - Console.WriteLine(" unique — the log now holds two rows for it, and telling them apart means reading"); - Console.WriteLine(" event_time_microseconds."); - - // While one is still running, the server refuses the second. Its own client, so that the two queries are - // genuinely concurrent rather than queued behind one connection. - await using var second = new ClickHouseTcpClient(client.Options); - string busyId = $"example-tcp-020-busy-{Guid.NewGuid():N}"; - var busy = new ClickHouseTcpQueryOptions { QueryId = busyId }; - - // Started without Task.Run, so the query packet goes out on this thread rather than whenever the thread - // pool gets to it. That ordering is what decides which of the two the server refuses. - Task slow = client - .ExecuteScalarAsync("SELECT sleepEachRow(0.05) FROM numbers(6)", busy) - .AsTask(); - - // Waits until the server really is running it, rather than guessing with a delay. Until this returns, the - // id is not yet claimed and it is undecided which query would be the duplicate. - await WaitUntilRunning(second, busyId); - - try - { - await second.ExecuteScalarAsync("SELECT 1", busy); - Console.WriteLine(" A concurrent reuse was accepted, which is not what this example expected"); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($"\n The same id while the first is still running -> {ex.Code} (RawCode {ex.RawCode})"); - Console.WriteLine($" {FirstLine(ex.Message)}"); - Console.WriteLine(" Code reads Unknown because ClickHouseErrorCode does not name 216; RawCode"); - Console.WriteLine(" always carries the server's number. Tcp_023 is about that pair."); - } - - await slow; - Console.WriteLine("\n Use a fresh id per attempt (a Guid, or your own request id) unless you want the"); - Console.WriteLine(" server to reject a duplicate submission for you — which, with a retry, is the one"); - Console.WriteLine(" case where reusing an id is the point rather than a mistake."); - } - - private static async Task AsyncInsert(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. async_insert: a setting that changes what an insert means\n"); - - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($"CREATE TABLE {TableName} (id UInt64, note String) ENGINE = MergeTree ORDER BY id"); - - object[][] first = [[1UL, "a"], [2UL, "b"]]; - object[][] second = [[3UL, "c"], [4UL, "d"]]; - - // Settings live on ClickHouseTcpInsertOptions too: it derives from ClickHouseTcpQueryOptions and adds - // MaxRowsPerBlock (Tcp_009). - var waits = new ClickHouseTcpInsertOptions - { - Settings = new Dictionary { ["async_insert"] = "1", ["wait_for_async_insert"] = "1" }, - }; - var doesNotWait = new ClickHouseTcpInsertOptions - { - Settings = new Dictionary { ["async_insert"] = "1", ["wait_for_async_insert"] = "0" }, - }; - - var clock = Stopwatch.StartNew(); - await client.InsertRowsAsync($"INSERT INTO {TableName} (id, note) VALUES", first, waits); - long waited = clock.ElapsedMilliseconds; - object afterWaiting = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); - - Console.WriteLine($" async_insert=1, wait_for_async_insert=1: returned after {waited} ms, and the rows are"); - Console.WriteLine($" already queryable — count() = {afterWaiting}. The rows went into a server-side buffer"); - Console.WriteLine(" shared with other clients' inserts, and the call waited for that buffer to be written."); - - clock.Restart(); - await client.InsertRowsAsync($"INSERT INTO {TableName} (id, note) VALUES", second, doesNotWait); - long notWaited = clock.ElapsedMilliseconds; - object immediately = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); - - // Polling this one table, not SYSTEM FLUSH ASYNC INSERT QUEUE: that command flushes every client's - // pending async inserts on the server, so an example must not issue it on a shared one. - long afterFlush = await CountWhenItReaches(client, 4); - - Console.WriteLine($"\n async_insert=1, wait_for_async_insert=0: returned after {notWaited} ms."); - Console.WriteLine($" count() straight afterwards = {immediately}. That number is 2 on one run and 4 on the next:"); - Console.WriteLine(" the buffer flushes on its own schedule and the call no longer waits for it."); - Console.WriteLine($" Counting again until the buffer has been written: count() = {afterFlush}."); - Console.WriteLine(); - Console.WriteLine(" Two things the pair changes, neither of which is visible in the API:"); - Console.WriteLine(" - a returned InsertRowsAsync no longer means the rows are stored, so a failure"); - Console.WriteLine(" after the return is reported to nobody;"); - Console.WriteLine(" - read-after-write stops holding, so a test that inserts and counts fails."); - Console.WriteLine(); - Console.WriteLine(" It is worth it for many small inserts from many clients, which is what the server-side"); - Console.WriteLine(" buffer is for. For one large insert, MaxRowsPerBlock and a plain insert are better."); - } - - /// - /// Reads one scalar out of system.query_log, retrying the flush and the read. Pick an expression that - /// is never NULL for a row that exists, so that "no row yet" and "row with an empty value" cannot be confused. - /// - private static async Task ReadLog(ClickHouseTcpClient client, string sql, string queryId) - { - var options = new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "id", queryId } }, - }; - - for (int attempt = 1; attempt <= 5; attempt++) - { - await client.ExecuteAsync("SYSTEM FLUSH LOGS"); - await foreach (object[] row in client.QueryAsync(sql, options)) - { - return $"{row[0]} (attempt {attempt})"; - } - - await Task.Delay(50); - } - - return "no row appeared in system.query_log after 5 attempts"; - } - - /// Counts the table until it holds rows, or gives up. - private static async Task CountWhenItReaches(ClickHouseTcpClient client, long expected) - { - long count = 0; - - for (int attempt = 0; attempt < 300; attempt++) - { - count = Convert.ToInt64(await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}")); - if (count >= expected) - { - return count; - } - - await Task.Delay(10); - } - - return count; - } - - /// Waits until system.processes shows the query, so a race cannot decide the next assertion. - private static async Task WaitUntilRunning(ClickHouseTcpClient client, string queryId) - { - var options = new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "id", queryId } }, - }; - - for (int attempt = 0; attempt < 300; attempt++) - { - object running = await client.ExecuteScalarAsync( - "SELECT count() FROM system.processes WHERE query_id = {id:String}", - options); - if (Convert.ToUInt64(running) > 0) - { - return; - } - - await Task.Delay(10); - } - - throw new TimeoutException( - $"query {queryId} did not appear in system.processes, so the next step's precondition does not hold"); - } - - /// The server's message is one long line with its own detail appended; the first line is the fact. - private static string FirstLine(string message) - { - string text = message.Replace("DB::Exception: ", string.Empty); - int newline = text.IndexOf('\n'); - if (newline >= 0) - { - text = text[..newline]; - } - - return text.Length <= 110 ? text : text[..110] + "..."; - } -} diff --git a/examples/Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs b/examples/Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs deleted file mode 100644 index fbb6bef1f..000000000 --- a/examples/Tcp/Advanced/Tcp_021_ProgressAndStatistics.cs +++ /dev/null @@ -1,247 +0,0 @@ -using System.Diagnostics; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// : the metadata the server interleaves into a response — -/// while the query is still running, -/// once with the execution summary, and -/// with the server's own performance counters. -/// -/// -/// This is what the native protocol has that HTTP does not. HTTP reports the same numbers in a trailing header, -/// after the response; here they arrive as packets between the data blocks, so a long query can drive a progress -/// bar while it runs. -/// -/// -/// -/// The contract matters more than the numbers. A callback runs synchronously on the thread draining the -/// response, in packet order, so anything slow in one stalls the read. A callback that throws propagates out of -/// the operation and terminates the connection — this example does not demonstrate that, because there is nothing -/// to see: the result is simply gone. Keep them to counters and a log line, and never let one throw. -/// -/// -public static class TcpProgressAndStatistics -{ - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - await ProgressArrivesDuringTheQuery(client); - IncrementsNotTotals(); - await ProfileInfoOnce(client); - await ProfileEvents(client); - WhatElseIsThere(); - } - - private static async Task ProgressArrivesDuringTheQuery(ClickHouseTcpClient client) - { - Console.WriteLine("1. OnProgress arrives while the query runs\n"); - - // The record of what happened, in the order it happened. Appending from the callback is safe without a - // lock precisely because callbacks run on the thread draining the response — the same thread this loop - // body runs on. - var timeline = new List(); - var clock = Stopwatch.StartNew(); - ClickHouseTcpProgress total = default; - int packets = 0; - int rowsSoFar = 0; - int beforeTheLastRow = 0; - - var options = new ClickHouseTcpQueryOptions - { - Callbacks = new ClickHouseTcpQueryCallbacks - { - OnProgress = progress => - { - packets++; - total += progress; - if (rowsSoFar < 8) - { - beforeTheLastRow++; - } - - timeline.Add($"progress +{progress.Rows} rows at {clock.ElapsedMilliseconds,4} ms"); - }, - }, - - // interactive_delay is how often the server reports progress, in microseconds. The default is 100 ms; - // 30 ms makes the interleaving obvious in an example short enough to run in CI. - Settings = new Dictionary - { - ["interactive_delay"] = "30000", - ["max_block_size"] = "1", - }, - }; - - int rows = 0; - await foreach (object[] row in client.QueryAsync( - "SELECT number, sleepEachRow(0.04) FROM numbers(8)", options)) - { - rows++; - rowsSoFar = rows; - timeline.Add($"row {rows} at {clock.ElapsedMilliseconds,4} ms"); - } - - Console.WriteLine(" 8 rows, each taking the server 40 ms, one row per block:\n"); - foreach (string line in timeline) - { - Console.WriteLine($" {line}"); - } - - Console.WriteLine(); - Console.WriteLine($" {packets} progress packets and {rows} rows, interleaved — {beforeTheLastRow} of the packets arrived before the"); - Console.WriteLine(" last row, which is the whole point. On HTTP every one of those numbers arrives after"); - Console.WriteLine($" the response. Summed: {total.Rows} rows, {total.Bytes} bytes, {total.ElapsedNs / 1_000_000} ms of server-side time."); - } - - private static void IncrementsNotTotals() - { - Console.WriteLine("\n2. Every counter is an increment\n"); - - // Two packets, added rather than replaced. Keeping the last one reports the most recent step, not the run. - var first = new ClickHouseTcpProgress(rows: 100, bytes: 800, totalRows: 1000, wroteRows: 0, wroteBytes: 0, elapsedNs: 5_000_000); - var next = new ClickHouseTcpProgress(rows: 250, bytes: 2000, totalRows: 500, wroteRows: 0, wroteBytes: 0, elapsedNs: 7_000_000); - - Console.WriteLine($" packet 1 Rows={first.Rows,4} Bytes={first.Bytes,5} TotalRows={first.TotalRows}"); - Console.WriteLine($" packet 2 Rows={next.Rows,4} Bytes={next.Bytes,5} TotalRows={next.TotalRows}"); - Console.WriteLine($" first + next Rows={(first + next).Rows,4} Bytes={(first + next).Bytes,5} TotalRows={(first + next).TotalRows}"); - Console.WriteLine(); - Console.WriteLine(" TotalRows is an increment too: it is the rise in the server's estimate of the rows"); - Console.WriteLine(" this query has to read, so a progress bar's denominator is the running sum of it and"); - Console.WriteLine(" can grow as the server learns more. Use operator + or ClickHouseTcpProgress.Add."); - Console.WriteLine(); - Console.WriteLine(" WroteRows and WroteBytes are the insert side of the same packet. On 26.6 an insert"); - Console.WriteLine(" through this client produces no progress packets at all, so they read zero — a large"); - Console.WriteLine(" insert has no progress to report yet."); - } - - private static async Task ProfileInfoOnce(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. OnProfileInfo, once, with totals rather than increments\n"); - - ClickHouseTcpProfileInfo info = default; - int calls = 0; - - var options = new ClickHouseTcpQueryOptions - { - Callbacks = new ClickHouseTcpQueryCallbacks - { - OnProfileInfo = summary => - { - info = summary; - calls++; - }, - }, - }; - - // A LIMIT, so that AppliedLimit and RowsBeforeLimit have something to say. - int rows = 0; - await foreach (object[] row in client.QueryAsync( - "SELECT number FROM numbers(1000) ORDER BY number DESC LIMIT 5", options)) - { - rows++; - } - - Console.WriteLine($" SELECT number FROM numbers(1000) ORDER BY number DESC LIMIT 5 ({rows} rows read)\n"); - Console.WriteLine($" called {calls} time"); - Console.WriteLine($" Rows {info.Rows}"); - Console.WriteLine($" Blocks {info.Blocks}"); - Console.WriteLine($" Bytes {info.Bytes}"); - Console.WriteLine($" AppliedLimit {info.AppliedLimit}"); - Console.WriteLine($" RowsBeforeLimit {info.RowsBeforeLimit}"); - Console.WriteLine($" CalculatedRowsBeforeLimit {info.CalculatedRowsBeforeLimit}"); - Console.WriteLine(); - Console.WriteLine(" RowsBeforeLimit is what a paging UI wants for its 'of N' — but only when"); - Console.WriteLine(" CalculatedRowsBeforeLimit is true. The server does not always work it out, and the"); - Console.WriteLine(" field is then zero rather than absent, so the flag is the one to read first."); - Console.WriteLine(); - Console.WriteLine(" Bytes counts the result as the server measured it in memory, not the bytes that"); - Console.WriteLine(" crossed the socket. Tcp_024 measures those."); - } - - private static async Task ProfileEvents(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. OnProfileEvents: the server's own counters, as it goes\n"); - - // Two dictionaries, because the block carries two kinds of row. type 1 is an increment to add up; type 2 - // is a gauge reading that replaces the last one. - var increments = new Dictionary(StringComparer.Ordinal); - var gauges = new Dictionary(StringComparer.Ordinal); - var threadIds = new HashSet(); - int blocks = 0; - - var options = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["interactive_delay"] = "30000", ["max_block_size"] = "1" }, - Callbacks = new ClickHouseTcpQueryCallbacks - { - OnProfileEvents = block => - { - blocks++; - - // The block is borrowed: valid until the callback returns. Names have to be copied out (they - // are already strings); spans must not outlive it. - IColumn name = block.Column("name"); - ReadOnlySpan value = block.Column("value").Values; - ReadOnlySpan type = block.Column("type").Values; - ReadOnlySpan thread = block.Column("thread_id").Values; - - for (int row = 0; row < block.RowCount; row++) - { - threadIds.Add(thread[row]); - if (type[row] == 1) - { - increments.TryGetValue(name[row], out long soFar); - increments[name[row]] = soFar + value[row]; - } - else - { - gauges[name[row]] = value[row]; - } - } - }, - }, - }; - - await foreach (object[] row in client.QueryAsync("SELECT number, sleepEachRow(0.03) FROM numbers(8)", options)) - { - } - - Console.WriteLine($" {blocks} blocks of counters arrived during the query, {increments.Count} distinct increments and"); - Console.WriteLine($" {gauges.Count} gauges. thread_id values seen: {string.Join(", ", threadIds.Order())} — 0 is the query-wide total.\n"); - - foreach (string counter in new[] { "SelectedRows", "SelectedBytes", "SleepFunctionMicroseconds", "NetworkSendBytes", "RealTimeMicroseconds" }) - { - string reading = increments.TryGetValue(counter, out long sum) ? sum.ToString("N0") : "(not reported)"; - Console.WriteLine($" increment {counter,-26} {reading,12}"); - } - - foreach (string gauge in gauges.Keys.Order()) - { - Console.WriteLine($" gauge {gauge,-26} {gauges[gauge],12:N0}"); - } - - Console.WriteLine(); - Console.WriteLine(" Every counter in system.events and system.metrics can appear here, so this is the"); - Console.WriteLine(" whole of what the server knows about its own work on this query. Reading `name`"); - Console.WriteLine(" allocates a string per row and the same counter arrives on every packet, so pick the"); - Console.WriteLine(" handful you care about rather than keeping them all."); - } - - private static void WhatElseIsThere() - { - Console.WriteLine("\n5. The rest of the record\n"); - Console.WriteLine(" OnLog the server's own log lines, when the query sets send_logs_level."); - Console.WriteLine(" priority is a Poco severity, so a lower number is more severe."); - Console.WriteLine(" OnTotals the WITH TOTALS row, in the query's own result shape."); - Console.WriteLine(" OnExtremes two rows, the minimum and the maximum, when the extremes setting is on."); - Console.WriteLine(); - Console.WriteLine(" All three hand over a borrowed Block on the same contract as StreamAsync: copy out"); - Console.WriteLine(" what must outlive the callback, and retain neither the block nor a span over it."); - Console.WriteLine(); - Console.WriteLine(" An unset callback costs nothing beyond the discarded result. The packets are decoded"); - Console.WriteLine(" either way, because skipping one would leave the connection misaligned."); - } -} diff --git a/examples/Tcp/Advanced/Tcp_022_Cancellation.cs b/examples/Tcp/Advanced/Tcp_022_Cancellation.cs deleted file mode 100644 index 7a2627dc1..000000000 --- a/examples/Tcp/Advanced/Tcp_022_Cancellation.cs +++ /dev/null @@ -1,303 +0,0 @@ -using System.Diagnostics; -using ClickHouse.Driver.Tcp; -using Microsoft.Extensions.Logging; - -namespace ClickHouse.Driver.Examples; - -/// -/// Cancelling a native-protocol operation: what the caller sees, what the server is told, and what it costs the -/// connection pool. -/// -/// -/// Every method takes a , and it is the only bound on a whole operation — the three -/// deadlines in Tcp_019_Timeouts each cover one phase. Cancelling is not free, though: the client tells the -/// server the result is abandoned and then closes the connection, because a socket part-way through a response -/// nobody will read is of no use to the next caller. The client itself stays usable; its pool opens another. -/// -/// -public static class TcpCancellation -{ - public static async Task Run() - { - await CancellingMidResult(); - await WhatTheServerWasTold(); - int abandonedAfter = await ThePoolDiscardsIt(); - WhatThePoolLinesSay(abandonedAfter); - await ExecuteAndStream(); - TheOtherWaysAnOperationEnds(); - } - - private static async Task CancellingMidResult() - { - Console.WriteLine("1. Cancelling part-way through a result\n"); - - await using var client = ExampleConfig.CreateTcpClient(); - using var cancellation = new CancellationTokenSource(); - - int rows = 0; - var clock = Stopwatch.StartNew(); - try - { - // 40 rows at 50 ms each, one row per block, so the loop body really does run between rows. - await foreach (object[] row in client.QueryAsync( - "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 1", - cancellationToken: cancellation.Token)) - { - rows++; - if (rows == 3) - { - cancellation.Cancel(); - } - } - - Console.WriteLine(" The loop finished, which is not what this example expected"); - } - catch (OperationCanceledException ex) - { - Console.WriteLine($" Cancelled after {rows} of 40 rows, {clock.ElapsedMilliseconds} ms in."); - Console.WriteLine($" Caught {ex.GetType().Name}: {ex.Message}"); - Console.WriteLine(); - Console.WriteLine(" The runtime raises TaskCanceledException here, which derives from"); - Console.WriteLine(" OperationCanceledException — catch the base one. It is not a"); - Console.WriteLine(" ClickHouseTcpException: nothing went wrong between the client and the server,"); - Console.WriteLine(" the caller asked to stop. Tcp_023 covers the exceptions that are."); - } - - // The same client, straight afterwards. Cancelling costs a connection, not the client. - object still = await client.ExecuteScalarAsync("SELECT 'the client is still usable'"); - Console.WriteLine($"\n Next operation on the same client: {still}"); - } - - private static async Task WhatTheServerWasTold() - { - Console.WriteLine("\n2. What the server was told\n"); - - await using var client = ExampleConfig.CreateTcpClient(); - string queryId = $"example-tcp-022-{Guid.NewGuid():N}"; - using var cancellation = new CancellationTokenSource(); - - try - { - await foreach (object[] row in client.QueryAsync( - "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 1", - new ClickHouseTcpQueryOptions { QueryId = queryId }, - cancellation.Token)) - { - cancellation.Cancel(); - } - } - catch (OperationCanceledException) - { - } - - // The QueryFinish/ExceptionWhileProcessing record is queued independently of the response reaching the - // client, so the flush and the read are retried rather than delayed. - string logged = await ReadLog( - client, - "SELECT type::String || ' exception_code=' || toString(exception_code) || ' ' || splitByChar('(', exception)[1] " + - "FROM system.query_log WHERE query_id = {id:String} AND type != 'QueryStart'", - queryId); - - Console.WriteLine($" system.query_log for that query_id:\n {logged}"); - Console.WriteLine(); - Console.WriteLine(" 735 is QUERY_WAS_CANCELLED_BY_CLIENT. The client sent a Cancel packet before closing"); - Console.WriteLine(" the connection, so the server stopped the query rather than finishing it into a socket"); - Console.WriteLine(" nobody was reading. That is the difference between cancelling and hanging up: the"); - Console.WriteLine(" work stops, and the reason is in the log."); - } - - /// - /// Six operations on a one-connection pool, with the pool's own log lines: two ordinary ones, a cancelled one, - /// an abandoned one, and an ordinary one after each. Its own method so that the logger factory is disposed — - /// and its lines flushed to the console — before the interpretation prints. - /// - /// How many rows the abandoned enumeration read before breaking out. - private static async Task ThePoolDiscardsIt() - { - Console.WriteLine("\n3. The connection is closed, not pooled\n"); - Console.WriteLine(" MaxPoolSize = 1, and the pool's own log lines. Had a connection gone back into the"); - Console.WriteLine(" pool, the operation after it would be reusing it — the pool holds only one.\n"); - - using ILoggerFactory poolLog = LoggerFactory.Create(builder => builder - .AddFilter((category, _) => category == "ClickHouse.Driver.Tcp.Pool") - .AddSimpleConsole(console => console.SingleLine = true) - .SetMinimumLevel(LogLevel.Trace)); - - await using var client = new ClickHouseTcpClient(ExampleConfig.TcpBuilder().ToOptions() with - { - MaxPoolSize = 1, - LoggerFactory = poolLog, - }); - - // Two ordinary operations first, so that a reuse line is in the output to compare against. - _ = await client.ExecuteScalarAsync("SELECT 1"); - _ = await client.ExecuteScalarAsync("SELECT 2"); - - using var cancellation = new CancellationTokenSource(); - try - { - await foreach (object[] row in client.QueryAsync( - "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 1", - cancellationToken: cancellation.Token)) - { - cancellation.Cancel(); - } - } - catch (OperationCanceledException) - { - } - - _ = await client.ExecuteScalarAsync("SELECT 3"); - - // No token this time: the loop simply stops reading, which is abandonment rather than cancellation. - int rows = 0; - await foreach (object[] row in client.QueryAsync( - "SELECT number FROM numbers(10000000) SETTINGS max_block_size = 100")) - { - if (++rows == 5) - { - break; - } - } - - _ = await client.ExecuteScalarAsync("SELECT 4"); - return rows; - } - - private static void WhatThePoolLinesSay(int abandonedAfter) - { - Console.WriteLine("\n Read that as four pairs. SELECT 1 opened a connection and SELECT 2 reused it — 'its 2"); - Console.WriteLine(" operation'. The cancelled query got 'its 3 operation' and then ended it: 'Closing a"); - Console.WriteLine(" returned connection rather than pooling it', so SELECT 3 had to open another."); - Console.WriteLine(); - Console.WriteLine($" Then the same thing with no token at all: a loop that read {abandonedAfter} rows of ten million"); - Console.WriteLine(" and broke out. The same two lines follow it, so abandoning a result is treated exactly"); - Console.WriteLine(" as cancelling one — the connection is closed, and SELECT 4 opened a fresh one."); - Console.WriteLine(); - Console.WriteLine(" So a cancellation costs a dial, and a loop that cancels every query keeps the pool"); - Console.WriteLine(" empty. It does not cost the client: every operation after one of these succeeded."); - Console.WriteLine(); - Console.WriteLine(" `break` inside an `await foreach` disposes the enumerator, which is what sends the"); - Console.WriteLine(" Cancel packet and returns the connection. So does `return`, and so does an exception"); - Console.WriteLine(" thrown from the loop body."); - Console.WriteLine(); - Console.WriteLine(" The one shape that does not is a hand-rolled enumerator that is never disposed:"); - Console.WriteLine(" var e = client.QueryAsync(sql).GetAsyncEnumerator(); // no await using"); - Console.WriteLine(" Its connection is neither returned nor closed, and nothing reclaims it — there is no"); - Console.WriteLine(" finalizer to free the pool slot, so it is gone for as long as the client lives. Use"); - Console.WriteLine(" `await foreach`, or `await using` on the enumerator."); - } - - private static async Task ExecuteAndStream() - { - Console.WriteLine("\n4. The same token on ExecuteAsync and StreamAsync\n"); - - await using var client = ExampleConfig.CreateTcpClient(); - - // ExecuteAsync drains the whole response before returning, so there is no loop to break out of and the - // token is the only way to stop waiting. - using (var deadline = new CancellationTokenSource(150)) - { - var clock = Stopwatch.StartNew(); - try - { - await client.ExecuteAsync("SELECT sleepEachRow(0.2) FROM numbers(5)", cancellationToken: deadline.Token); - Console.WriteLine(" ExecuteAsync returned, which is not what this example expected"); - } - catch (OperationCanceledException ex) - { - Console.WriteLine($" ExecuteAsync, token cancelled after 150 ms: {ex.GetType().Name} at {clock.ElapsedMilliseconds} ms"); - } - } - - // StreamAsync is the same contract one level down: the block being iterated is released, the enumerator - // is disposed by the loop, and the connection is closed. - using (var cancellation = new CancellationTokenSource()) - { - int blocks = 0; - try - { - await foreach (Block block in client.StreamAsync( - "SELECT number, sleepEachRow(0.05) FROM numbers(40) SETTINGS max_block_size = 4", - cancellationToken: cancellation.Token)) - { - blocks++; - cancellation.Cancel(); - } - } - catch (OperationCanceledException ex) - { - Console.WriteLine($" StreamAsync, cancelled after {blocks} block: {ex.GetType().Name}"); - } - } - - Console.WriteLine($" And afterwards: {await client.ExecuteScalarAsync("SELECT 'still usable'")}"); - Console.WriteLine(); - Console.WriteLine(" A token already cancelled when the call is made throws before the pool is touched, so"); - Console.WriteLine(" nothing is dialled and nothing is closed — the pool's log stays silent, and the next"); - Console.WriteLine(" operation reuses whatever was idle:"); - - using (var alreadyDone = new CancellationTokenSource()) - { - await alreadyDone.CancelAsync(); - try - { - _ = await client.ExecuteScalarAsync("SELECT 1", cancellationToken: alreadyDone.Token); - Console.WriteLine(" it ran anyway, which is not what this example expected"); - } - catch (OperationCanceledException ex) - { - Console.WriteLine($" {ex.GetType().Name} straight away"); - } - } - } - - private static void TheOtherWaysAnOperationEnds() - { - Console.WriteLine("\n5. Three ways to stop a query, and which one to reach for\n"); - Console.WriteLine(" CancellationToken The caller changed its mind: a request was abandoned, a"); - Console.WriteLine(" timeout of your own elapsed, the process is shutting down."); - Console.WriteLine(" Throws OperationCanceledException, costs the connection,"); - Console.WriteLine(" and the server is told (section 2)."); - Console.WriteLine(); - Console.WriteLine(" ReadTimeout The server went quiet. An idle deadline, not a time limit —"); - Console.WriteLine(" Tcp_019 measures it. Throws TimeoutException and also costs"); - Console.WriteLine(" the connection, because a socket that stopped answering"); - Console.WriteLine(" mid-response cannot be reused either."); - Console.WriteLine(); - Console.WriteLine(" max_execution_time The server gives up, as a per-query setting (Tcp_020). The"); - Console.WriteLine(" query stops server-side, the client gets an ordinary"); - Console.WriteLine(" ClickHouseTcpServerException with code 159, and the"); - Console.WriteLine(" connection survives, because the response completed — with"); - Console.WriteLine(" an error rather than rows. Tcp_023 covers it."); - Console.WriteLine(); - Console.WriteLine(" So the cheapest of the three is the server-side one: it is the only one that does not"); - Console.WriteLine(" end a connection. Prefer max_execution_time for 'this query must not run longer than N"); - Console.WriteLine(" seconds', and keep the token for 'this caller no longer wants the answer'."); - } - - /// - /// Reads one row out of system.query_log, retrying the flush and the read. A query's record is queued - /// independently of its response, so one flush straight after the query can miss it. - /// - private static async Task ReadLog(ClickHouseTcpClient client, string sql, string queryId) - { - var options = new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "id", queryId } }, - }; - - for (int attempt = 1; attempt <= 5; attempt++) - { - await client.ExecuteAsync("SYSTEM FLUSH LOGS"); - await foreach (object[] row in client.QueryAsync(sql, options)) - { - return (string)row[0]; - } - - await Task.Delay(50); - } - - return "no row appeared in system.query_log after 5 attempts"; - } -} diff --git a/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs b/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs deleted file mode 100644 index 8f733774f..000000000 --- a/examples/Tcp/Advanced/Tcp_023_ErrorsAndRetries.cs +++ /dev/null @@ -1,411 +0,0 @@ -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// The native client's exception hierarchy — , -/// and under -/// — how to branch on , and which failures -/// are worth retrying. -/// -/// -/// Retrying is where the two halves meet. A read is idempotent, so a retry costs a round trip and nothing else. An -/// insert is not: the same batch sent twice lands twice, unless the target table can deduplicate it and the insert -/// carries a token that says which insert it is. This example measures both. -/// -/// -public static class TcpErrorsAndRetries -{ - private const string PlainTable = "example_tcp_retry_plain"; - private const string DedupTable = "example_tcp_retry_dedup"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - TheHierarchy(); - await ServerErrors(client); - await NotServerErrors(); - await Transient(client); - await RetryingARead(client); - - try - { - await RetryingAnInsert(client); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {PlainTable}"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {DedupTable}"); - Console.WriteLine($"\nDropped {PlainTable} and {DedupTable}."); - } - } - - private static void TheHierarchy() - { - Console.WriteLine("1. Three exception types, one base, and what is deliberately not in it\n"); - Console.WriteLine(" ClickHouseTcpException : DbException catch this for 'anything between the"); - Console.WriteLine(" client and the server went wrong'"); - Console.WriteLine(" ClickHouseTcpServerException the server reported an error for a query,"); - Console.WriteLine(" a handshake or a ping"); - Console.WriteLine(" ClickHouseTcpTransportException the socket failed: refused, dropped, TLS"); - Console.WriteLine(" ClickHouseTcpProtocolException the bytes did not match the protocol"); - Console.WriteLine(); - Console.WriteLine(" The hierarchy is closed — the constructors are not visible outside the assembly — so a"); - Console.WriteLine(" caught ClickHouseTcpException is always one of the three."); - Console.WriteLine(); - Console.WriteLine(" Mistakes in the calling code keep the usual framework types, on purpose:"); - Console.WriteLine(" ArgumentException a bad option or a null argument (Tcp_019 has nine)"); - Console.WriteLine(" InvalidOperationException a misused object — a session running two operations"); - Console.WriteLine(" ObjectDisposedException use after disposal"); - Console.WriteLine(" OperationCanceledException the caller cancelled (Tcp_022)"); - Console.WriteLine(" TimeoutException a deadline elapsed: PoolTimeout, DialTimeout,"); - Console.WriteLine(" ReadTimeout (Tcp_019)"); - Console.WriteLine(); - Console.WriteLine(" So `catch (ClickHouseTcpException)` never swallows a bug in your own code, and never"); - Console.WriteLine(" swallows a cancellation."); - } - - private static async Task ServerErrors(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. ClickHouseTcpServerException: Code, RawCode, Name, ServerStackTrace\n"); - - (string What, string Sql)[] cases = - [ - ("a syntax error", "SELECT FROM WHERE"), - ("an unknown table", "SELECT * FROM does_not_exist_example_tcp_023"), - ("an unknown function", "SELECT no_such_function(1)"), - ("an unparseable value", "SELECT toUInt8('abc')"), - ("a division by zero", "SELECT intDiv(1, 0)"), - ]; - - Console.WriteLine($" {"",-22} {"Code",-28} {"RawCode",7} {"transient",9} message"); - foreach ((string what, string sql) in cases) - { - try - { - _ = await client.ExecuteScalarAsync(sql); - Console.WriteLine($" {what,-22} succeeded, which is not what this example expected"); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" {what,-22} {ex.Code,-28} {ex.RawCode,7} {ex.IsTransient,9} {FirstLine(ex.Message, 40)}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" RawCode is always the number the server sent. Code is that number as a named"); - Console.WriteLine(" constant, or Unknown when this client does not name it — the enum carries the codes"); - Console.WriteLine(" worth branching on, not all ~660 of them. Division by zero (153) is one it does not"); - Console.WriteLine(" name, so branch on RawCode for anything outside the list."); - Console.WriteLine(); - Console.WriteLine(" Every one of those left the connection usable: the server reported an error as part of"); - Console.WriteLine($" a complete response, so the pool keeps it. Proof — {await client.ExecuteScalarAsync("SELECT 'still here'")}."); - - // The two fields an operator asks for, on one error. - try - { - _ = await client.ExecuteScalarAsync("SELECT * FROM does_not_exist_example_tcp_023"); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine("\n The whole of one error:"); - Console.WriteLine($" Code {ex.Code}"); - Console.WriteLine($" RawCode {ex.RawCode}"); - Console.WriteLine($" Name {ex.Name}"); - Console.WriteLine($" IsTransient {ex.IsTransient}"); - Console.WriteLine($" ErrorCode {ex.ErrorCode} (DbException's, the same number)"); - Console.WriteLine($" ServerStackTrace {ex.ServerStackTrace?.Length ?? 0} characters of the server's own C++ frames"); - Console.WriteLine($" Message {FirstLine(ex.Message, 90)}"); - Console.WriteLine(); - Console.WriteLine(" Message repeats Name, because the server puts its exception class in both."); - Console.WriteLine(" ServerStackTrace is for a bug report, not for a log line."); - - // Branching. A switch on Code is the readable form; the default arm has to exist, because a code the - // enum does not name arrives as Unknown. - string advice = ex.Code switch - { - ClickHouseErrorCode.UnknownTable or ClickHouseErrorCode.UnknownDatabase => "check the name and the database the client is pointed at", - ClickHouseErrorCode.SyntaxError => "the query text is wrong; do not retry it", - ClickHouseErrorCode.AccessDenied or ClickHouseErrorCode.AuthenticationFailed => "a grant or a credential problem", - ClickHouseErrorCode.TooManyParts or ClickHouseErrorCode.ServerOverloaded => "back off and try again", - _ => $"unrecognized; RawCode {ex.RawCode}", - }; - Console.WriteLine($"\n switch (ex.Code) -> {advice}"); - } - } - - private static async Task NotServerErrors() - { - Console.WriteLine("\n3. The other two: transport and protocol\n"); - - ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); - - // Nothing listening: a socket failure, and a fresh connection may well work, so IsTransient is true. - try - { - await using var refused = new ClickHouseTcpClient(options with { Port = 1, DialTimeout = TimeSpan.FromSeconds(2) }); - await refused.PingAsync(); - Console.WriteLine(" Port 1 answered, which is not what this example expected"); - } - catch (ClickHouseTcpException ex) - { - Console.WriteLine($" nothing listening on the port {ex.GetType().Name}"); - Console.WriteLine($" IsTransient={ex.IsTransient}, InnerException={ex.InnerException?.GetType().Name}"); - Console.WriteLine(" Match the inner exception when the distinction"); - Console.WriteLine(" matters: SocketException, IOException,"); - Console.WriteLine(" EndOfStreamException, AuthenticationException."); - } - - // The HTTP port: a peer that answers, but not in this protocol. Not transient — retrying a - // misconfiguration just fails again. - try - { - await using var wrongPort = new ClickHouseTcpClient(options with { Port = ExampleConfig.HttpEndpoint.Port }); - await wrongPort.PingAsync(); - Console.WriteLine(" The HTTP port spoke the native protocol, which is not what this example expected"); - } - catch (ClickHouseTcpException ex) - { - Console.WriteLine($"\n the HTTP port ({ExampleConfig.HttpEndpoint.Port}) {ex.GetType().Name}"); - Console.WriteLine($" IsTransient={ex.IsTransient}"); - Console.WriteLine($" {ex.Message}"); - Console.WriteLine(" 72 is 'H', the first byte of an HTTP response."); - } - - Console.WriteLine(); - Console.WriteLine(" Both terminate the connection and it is never reused, which is why neither needs a"); - Console.WriteLine(" 'is the client still usable' check: the pool simply dials again."); - } - - private static async Task Transient(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. IsTransient, and what it does and does not promise\n"); - - // A real timeout: a scan far too large for the deadline. Transient by code, and completely deterministic. - await Report( - client, - "max_execution_time = 0.2 over a huge scan", - "SELECT count() FROM numbers(50000000000)", - new Dictionary { ["max_execution_time"] = "0.2" }); - - // Looks temporary, is not: the same query at the same size needs the same memory every time. - await Report( - client, - "max_memory_usage = 1 MB", - "SELECT groupArray(number) FROM numbers(10000000)", - new Dictionary { ["max_memory_usage"] = "1000000" }); - - Console.WriteLine(); - Console.WriteLine(" IsTransient reads the code, and it means 'retrying could plausibly succeed' — not"); - Console.WriteLine(" 'will'. TimeoutExceeded is transient because the server may be less busy next time,"); - Console.WriteLine(" and yet the query above will time out on every attempt, because the cause is its own"); - Console.WriteLine(" size. MemoryLimitExceeded is the opposite reading: it looks temporary and is judged"); - Console.WriteLine(" not transient, because the same query at the same size repeats it."); - Console.WriteLine(); - Console.WriteLine(" So cap the attempts, and prefer a failure whose cause is outside your query:"); - Console.WriteLine(" transient TimeoutExceeded(159) TooManySimultaneousQueries(202) NoFreeConnection(203)"); - Console.WriteLine(" SocketTimeout(209) NetworkError(210) TooManyParts(252)"); - Console.WriteLine(" AllConnectionTriesFailed(279) ServerOverloaded(745) KeeperException(999)"); - Console.WriteLine(" and every ClickHouseTcpTransportException"); - Console.WriteLine(" not syntax, unknown table or column, type mismatch, access denied,"); - Console.WriteLine(" memory limit, and every ClickHouseTcpProtocolException"); - } - - private static async Task Report( - ClickHouseTcpClient client, - string label, - string sql, - Dictionary settings) - { - try - { - _ = await client.ExecuteScalarAsync(sql, new ClickHouseTcpQueryOptions { Settings = settings }); - Console.WriteLine($" {label,-42} succeeded, which is not what this example expected"); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" {label,-42} {ex.Code} ({ex.RawCode}), IsTransient={ex.IsTransient}"); - } - } - - private static async Task RetryingARead(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. Retrying a read, which is free\n"); - - // A real transient failure with nothing injected. max_concurrent_queries_for_user is checked when a query - // starts, and only against the query that declares it: the slow query below declares nothing, so it holds - // a slot and can never itself be refused, and only the retry loop can lose. That one-sidedness is what - // makes the outcome determined rather than raced. - // Started without Task.Run, so the query packet goes out on this thread before the poll below rather than - // whenever the thread pool gets to it. AsTask only wraps the operation already in flight. - string holderId = $"example-tcp-023-holder-{Guid.NewGuid():N}"; - Task holder = client - .ExecuteScalarAsync("SELECT sleepEachRow(0.08) FROM numbers(6)", new ClickHouseTcpQueryOptions { QueryId = holderId }) - .AsTask(); - - // A separate client, so that the two queries are really concurrent rather than queued behind one - // connection. - await using var second = new ClickHouseTcpClient(client.Options); - - // Waits until the server is really running it, so the first attempt below is refused rather than usually - // refused. Polling rather than a delay: a delay is the same race with a longer window. - await WaitUntilRunning(second, holderId); - - Console.WriteLine(" A slow query is running. A second one asks with max_concurrent_queries_for_user = 1,"); - Console.WriteLine(" so the server refuses it until the first has finished.\n"); - - var oneAtATime = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["max_concurrent_queries_for_user"] = "1" }, - }; - - const int attempts = 8; - - for (int attempt = 1; attempt <= attempts; attempt++) - { - try - { - object counted = await second.ExecuteScalarAsync("SELECT count() FROM numbers(1000)", oneAtATime); - Console.WriteLine($" attempt {attempt}: {counted}"); - break; - } - catch (ClickHouseTcpException ex) when (ex.IsTransient) - { - string reason = ex is ClickHouseTcpServerException server ? $"{server.Code} ({server.RawCode})" : ex.GetType().Name; - Console.WriteLine($" attempt {attempt}: {reason} — transient, so try again"); - - // The last attempt is caught too. The limit counts every query this user is running, so anything - // else on the server under the same user can keep refusing this one, and an example must report - // that rather than throw out of the demonstration. - if (attempt == attempts) - { - Console.WriteLine($" gave up after {attempts}: the cap is what stops a retry becoming a loop."); - break; - } - - await Task.Delay(100 * attempt); - } - } - - await holder; - - Console.WriteLine(); - Console.WriteLine(" `catch (ClickHouseTcpException ex) when (ex.IsTransient)` is the whole filter, and the"); - Console.WriteLine(" attempt cap is what keeps a deterministic failure from becoming a loop. The read is"); - Console.WriteLine(" idempotent, so nothing had to be checked before trying again — which is the only"); - Console.WriteLine(" reason this retry is safe to write in three lines."); - Console.WriteLine(); - Console.WriteLine(" The limit travelled in the query packet, so it bounded those attempts and nothing"); - Console.WriteLine(" else — not the query it was waiting for, and not whatever runs next. Tcp_020 is about"); - Console.WriteLine(" that."); - } - - /// - /// Waits until system.processes shows the query. Deliberately sets nothing of its own, so this poll can - /// never be the query a concurrency limit refuses. - /// - private static async Task WaitUntilRunning(ClickHouseTcpClient client, string queryId) - { - var options = new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "id", queryId } }, - }; - - for (int attempt = 0; attempt < 200; attempt++) - { - object running = await client.ExecuteScalarAsync( - "SELECT count() FROM system.processes WHERE query_id = {id:String}", - options); - if (Convert.ToUInt64(running) > 0) - { - return; - } - - await Task.Delay(10); - } - - throw new TimeoutException( - $"query {queryId} did not appear in system.processes, so the next step's precondition does not hold"); - } - - private static async Task RetryingAnInsert(ClickHouseTcpClient client) - { - Console.WriteLine("\n6. Retrying an insert, which is not\n"); - - await client.ExecuteAsync($"DROP TABLE IF EXISTS {PlainTable}"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {DedupTable}"); - await client.ExecuteAsync($"CREATE TABLE {PlainTable} (id UInt64) ENGINE = MergeTree ORDER BY id"); - - // A non-replicated MergeTree deduplicates nothing unless it is told to keep a window of insert hashes. - await client.ExecuteAsync( - $"CREATE TABLE {DedupTable} (id UInt64) ENGINE = MergeTree ORDER BY id " + - "SETTINGS non_replicated_deduplication_window = 100"); - - object[][] batch = [[1UL], [2UL], [3UL]]; - - // The failure mode: a retry after a transport error that in fact delivered the rows. - await client.InsertRowsAsync($"INSERT INTO {PlainTable} (id) VALUES", batch); - await client.InsertRowsAsync($"INSERT INTO {PlainTable} (id) VALUES", batch); - Console.WriteLine($" plain MergeTree, the same 3 rows sent twice count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {PlainTable}")}"); - - var token = new ClickHouseTcpInsertOptions - { - Settings = new Dictionary { ["insert_deduplication_token"] = "example-tcp-023-batch-1" }, - }; - - await client.InsertRowsAsync($"INSERT INTO {PlainTable} (id) VALUES", batch, token); - await client.InsertRowsAsync($"INSERT INTO {PlainTable} (id) VALUES", batch, token); - Console.WriteLine($" ... twice more with one insert_deduplication_token count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {PlainTable}")}"); - Console.WriteLine(" The token did nothing: this table keeps no window of insert hashes to compare it"); - Console.WriteLine(" against, so there is nothing for it to match."); - - await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", batch, token); - await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", batch, token); - Console.WriteLine($"\n non_replicated_deduplication_window = 100, same token count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {DedupTable}")}"); - - // The token identifies the insert, not the bytes: a second attempt that produced different rows is still - // dropped, which is what makes a retry safe even when the data was rebuilt. - object[][] different = [[4UL], [5UL], [6UL]]; - await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", different, token); - Console.WriteLine($" ... and again with different rows, same token count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {DedupTable}")}"); - - // No token at all on the same table: the block's own hash is still compared, so a byte-identical retry is - // dropped too. Worth knowing, and not worth relying on. - await client.ExecuteAsync($"TRUNCATE TABLE {DedupTable}"); - await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", batch); - await client.InsertRowsAsync($"INSERT INTO {DedupTable} (id) VALUES", batch); - Console.WriteLine($" ... and the same batch twice with no token at all count() = {await client.ExecuteScalarAsync($"SELECT count() FROM {DedupTable}")}"); - - Console.WriteLine(); - Console.WriteLine(" So a safe insert retry needs two things, and one of them is not in the client:"); - Console.WriteLine(" - the table must deduplicate — a Replicated engine, or"); - Console.WriteLine(" non_replicated_deduplication_window on a plain MergeTree;"); - Console.WriteLine(" - the insert must carry insert_deduplication_token, one value per logical batch,"); - Console.WriteLine(" reused by every retry of it."); - Console.WriteLine(); - Console.WriteLine(" The last line is why the token matters even though a window alone dropped the"); - Console.WriteLine(" duplicate: that was the block's own hash matching, and a retry that rebuilt the batch"); - Console.WriteLine(" in a different order, or split it differently, hashes differently and lands twice."); - Console.WriteLine(); - Console.WriteLine(" And an insert that fails with a ClickHouseTcpTransportException may or may not have"); - Console.WriteLine(" been applied — the client cannot tell which side of the socket the failure was. That"); - Console.WriteLine(" is the case the token exists for."); - } - - /// - /// The server's message is one long line with its own multi-line detail appended, and it starts with the class - /// name that already carries. - /// - private static string FirstLine(string message, int width) - { - string text = message.Replace("DB::Exception: ", string.Empty); - int newline = text.IndexOf('\n'); - if (newline >= 0) - { - text = text[..newline]; - } - - return text.Length <= width ? text : text[..width] + "..."; - } -} diff --git a/examples/Tcp/Advanced/Tcp_024_Compression.cs b/examples/Tcp/Advanced/Tcp_024_Compression.cs deleted file mode 100644 index 4dec13c93..000000000 --- a/examples/Tcp/Advanced/Tcp_024_Compression.cs +++ /dev/null @@ -1,350 +0,0 @@ -using System.Net; -using System.Net.Sockets; -using ClickHouse.Driver.Compression; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// Block compression on the native protocol: the Compression=lz4|zstd|none connection-string key, -/// , and what each setting is worth in bytes on the wire. -/// -/// -/// LZ4 is the default, so blocks are compressed in both directions unless you say otherwise. The HTTP transport's -/// Compression key is a boolean; this one names a codec. -/// -/// -/// -/// How this example measures. It forwards the connection through a local socket that counts the bytes each -/// way, which is the only thing a client can observe honestly. Wall-clock time is not measured: everything here -/// runs over loopback, where there is no bandwidth to save, so a timing comparison would report the CPU cost of -/// compressing and none of the benefit. Wire size is deterministic and is the thing compression actually buys. -/// -/// -public static class TcpCompression -{ - private const string TableName = "example_tcp_compression"; - - // Big enough that the codec dominates the fixed cost of a handshake, small enough to stay quick. - private const int SelectRows = 200_000; - private const int InsertRows = 100_000; - - // Two columns, one of them compressible text, so the ratio is representative rather than a best case. - private static readonly string Query = $"SELECT number, toString(number % 97) AS text FROM numbers({SelectRows})"; - - public static async Task Run() - { - WhatIsInForce(); - - await using var client = ExampleConfig.CreateTcpClient(); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($"CREATE TABLE {TableName} (id UInt64, text String) ENGINE = MergeTree ORDER BY id"); - - try - { - await ReadingDirection(); - await WhoChoosesTheServersCodec(); - await WritingDirection(); - WhatItBuys(); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped {TableName}."); - } - } - - private static void WhatIsInForce() - { - Console.WriteLine("1. Which codec a client is using\n"); - - // The assembled connection string carries no Compression key, so this is the default. - ClickHouseTcpClientOptions fromDefaults = ClickHouseTcpClientOptions.FromConnectionString(ExampleConfig.TcpConnectionString); - - Console.WriteLine($" no Compression key Compressor = {Describe(fromDefaults.Compressor)}"); - foreach (string codec in new[] { "lz4", "zstd", "none" }) - { - ClickHouseTcpClientOptions options = Options(codec); - Console.WriteLine($" Compression={codec,-16}Compressor = {Describe(options.Compressor)}"); - } - - Console.WriteLine(); - Console.WriteLine(" 'none' leaves Compressor null, and null means the query asks the server for no"); - Console.WriteLine(" compression at all — which is not the same as a frame whose method byte says NONE."); - Console.WriteLine(); - Console.WriteLine(" Setting it in code instead of in a connection string takes the codec object:"); - Console.WriteLine(" options with { Compressor = ZstdCompressor.Default }"); - Console.WriteLine(" options with { Compressor = new ZstdCompressor(level: 9) }"); - Console.WriteLine(" options with { Compressor = null } // off"); - Console.WriteLine(); - - // A codec that only implements the HTTP body path cannot frame a block, and the client says so at - // construction rather than mid-query. - try - { - using var refused = new ClickHouseTcpClient(Options("lz4") with { Compressor = GZipCompressor.Default }); - Console.WriteLine(" A GZip codec was accepted, which is not what this example expected"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" Not every IClickHouseCompressor will do: {ex.Message.Split(" (Parameter")[0]}"); - } - } - - private static async Task ReadingDirection() - { - Console.WriteLine($"\n2. Server to client: {SelectRows:N0} rows, measured on the wire\n"); - Console.WriteLine($" {Query}\n"); - Console.WriteLine($" {"client codec",-14}{"bytes from the server",22} {"vs none",8}"); - - long baseline = 0; - foreach (string codec in new[] { "none", "lz4", "zstd" }) - { - (long fromServer, _, long rows) = await Measure(Options(codec), null); - if (codec == "none") - { - baseline = fromServer; - } - - Console.WriteLine($" {codec,-14}{fromServer,22:N0} {(double)baseline / fromServer,7:0.00}x ({rows:N0} rows)"); - } - - Console.WriteLine(); - Console.WriteLine(" LZ4 cut it to about a third. ZSTD produced the same count as LZ4, to within the few"); - Console.WriteLine(" bytes of progress packets that vary between runs — which is the thing to understand"); - Console.WriteLine(" about this key."); - } - - private static async Task WhoChoosesTheServersCodec() - { - Console.WriteLine("\n3. The client's codec does not choose what the server sends\n"); - Console.WriteLine(" The query packet carries one flag: compressed, or not. Which codec the server then"); - Console.WriteLine(" frames its blocks with is the server's own choice, from its network_compression_method"); - Console.WriteLine(" setting — LZ4 by default. So asking for ZSTD on the client changed nothing above."); - Console.WriteLine(" Set it as a per-query setting to change it:\n"); - - Console.WriteLine($" {"network_compression_method",-30}{"bytes from the server",22}"); - foreach (string method in new[] { "LZ4", "ZSTD" }) - { - var options = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["network_compression_method"] = method }, - }; - (long fromServer, _, _) = await Measure(Options("lz4"), options); - Console.WriteLine($" {method,-30}{fromServer,22:N0}"); - } - - Console.WriteLine(); - Console.WriteLine(" The client decodes whatever arrives, whichever codec it asked for, so this is safe to"); - Console.WriteLine(" set per query. What the client's own Compressor decides is the direction the client"); - Console.WriteLine(" writes — which is an insert."); - } - - private static async Task WritingDirection() - { - Console.WriteLine($"\n4. Client to server: an insert of {InsertRows:N0} rows\n"); - Console.WriteLine($" {"client codec",-14}{"bytes to the server",22} {"vs none",8}"); - - long baseline = 0; - foreach (string codec in new[] { "none", "lz4", "zstd" }) - { - long toServer = await MeasureInsert(Options(codec)); - if (codec == "none") - { - baseline = toServer; - } - - Console.WriteLine($" {codec,-14}{toServer,22:N0} {(double)baseline / toServer,7:0.00}x"); - } - - Console.WriteLine(); - Console.WriteLine(" Here the key does what its name suggests, because these are the client's own frames."); - Console.WriteLine(" ZSTD is the smaller of the two and costs more CPU on the client to produce; LZ4 is the"); - Console.WriteLine(" cheaper one and is what the default gives you."); - } - - private static void WhatItBuys() - { - Console.WriteLine("\n5. What the numbers above do and do not tell you\n"); - Console.WriteLine(" They are bytes, and bytes are the honest measurement: run this example twice and the"); - Console.WriteLine(" counts differ only by the handful of progress packets the server chose to send."); - Console.WriteLine(); - Console.WriteLine(" There is deliberately no timing here. Every connection in this example is loopback,"); - Console.WriteLine(" where a saved byte saves nothing, so a wall-clock ranking of none/lz4/zstd would"); - Console.WriteLine(" measure the cost of compressing and none of the benefit — and would then read as an"); - Console.WriteLine(" argument for turning compression off. Where it pays is where the bytes have somewhere"); - Console.WriteLine(" to go: a link between availability zones, a metered egress bill, a saturated uplink,"); - Console.WriteLine(" or a server whose network is busier than its CPU."); - Console.WriteLine(); - Console.WriteLine(" Reasonable defaults, then:"); - Console.WriteLine(" lz4 leave it alone. Cheapest in CPU, lightest on the server, ~3x here."); - Console.WriteLine(" zstd a slow or metered link, and inserts large enough for the ratio to matter."); - Console.WriteLine(" none a client and a server on the same host, where the CPU is the scarce thing."); - Console.WriteLine(); - Console.WriteLine(" Compression is per query, not per connection, so nothing has to be restarted to change"); - Console.WriteLine(" it — but the codec lives on the client, so it takes a second client to run two."); - } - - /// Options for the configured server with one Compression value. - private static ClickHouseTcpClientOptions Options(string codec) - { - var builder = ExampleConfig.TcpBuilder(); - builder.Compression = codec; - return builder.ToOptions(); - } - - private static string Describe(IClickHouseCompressor compressor) - => compressor is null ? "null (no compression)" : compressor.GetType().Name; - - /// Runs the query through a counting proxy and reports the bytes each way. - private static async Task<(long FromServer, long ToServer, long Rows)> Measure( - ClickHouseTcpClientOptions options, - ClickHouseTcpQueryOptions? queryOptions) - { - await using var proxy = new CountingProxy(ExampleConfig.TcpEndpoint.Host, ExampleConfig.TcpEndpoint.Port); - - long rows = 0; - - // The client is disposed before the counters are read, so every byte of the handshake, the query and the - // close is included. The handshake is a few hundred bytes and identical between the runs. - await using (var client = new ClickHouseTcpClient(options with { Host = "127.0.0.1", Port = proxy.Port })) - { - await foreach (Block block in client.StreamAsync(Query, queryOptions)) - { - rows += block.RowCount; - } - } - - return (proxy.FromServer, proxy.ToServer, rows); - } - - private static async Task MeasureInsert(ClickHouseTcpClientOptions options) - { - await using var proxy = new CountingProxy(ExampleConfig.TcpEndpoint.Host, ExampleConfig.TcpEndpoint.Port); - - var ids = new ulong[InsertRows]; - var text = new string[InsertRows]; - for (int i = 0; i < InsertRows; i++) - { - ids[i] = (ulong)i; - text[i] = (i % 97).ToString(); - } - - await using (var client = new ClickHouseTcpClient(options with { Host = "127.0.0.1", Port = proxy.Port })) - { - await client.InsertAsync( - $"INSERT INTO {TableName} (id, text) VALUES", - [ClickHouseTcpColumn.Create("id", ids), ClickHouseTcpColumn.Create("text", text)]); - } - - return proxy.ToServer; - } - - /// - /// A local socket that forwards to the real server and counts the bytes each way. Nothing an application needs - /// — it is here because wire size is not otherwise observable from the client, and a byte count is the only - /// claim about compression that loopback can support. - /// - private sealed class CountingProxy : IAsyncDisposable - { - private readonly TcpListener listener; - private readonly CancellationTokenSource stopping = new(); - private readonly Task accepting; - private long fromServer; - private long toServer; - - public CountingProxy(string host, int port) - { - listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - Port = ((IPEndPoint)listener.LocalEndpoint).Port; - accepting = AcceptLoop(host, port); - } - - /// The loopback port to point a client at. - public int Port { get; } - - public long FromServer => Interlocked.Read(ref fromServer); - - public long ToServer => Interlocked.Read(ref toServer); - - public async ValueTask DisposeAsync() - { - await stopping.CancelAsync(); - listener.Stop(); - try - { - await accepting; - } - catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) - { - } - - stopping.Dispose(); - } - - private async Task AcceptLoop(string host, int port) - { - var sessions = new List(); - try - { - while (!stopping.IsCancellationRequested) - { - TcpClient accepted = await listener.AcceptTcpClientAsync(stopping.Token); - sessions.Add(Forward(accepted, host, port)); - } - } - catch (Exception ex) when (ex is OperationCanceledException or SocketException or ObjectDisposedException) - { - // The listener was stopped, which is the normal ending here. - } - - foreach (Task session in sessions) - { - try - { - await session; - } - catch (Exception ex) when (ex is OperationCanceledException or IOException or SocketException or ObjectDisposedException) - { - } - } - } - - private async Task Forward(TcpClient downstream, string host, int port) - { - using TcpClient upstream = new(); - using (downstream) - { - await upstream.ConnectAsync(host, port, stopping.Token); - await Task.WhenAll( - Copy(downstream.GetStream(), upstream.GetStream(), towardsServer: true), - Copy(upstream.GetStream(), downstream.GetStream(), towardsServer: false)); - } - } - - private async Task Copy(Stream from, Stream to, bool towardsServer) - { - byte[] buffer = new byte[64 * 1024]; - try - { - while (true) - { - int read = await from.ReadAsync(buffer, stopping.Token); - if (read == 0) - { - break; - } - - Interlocked.Add(ref towardsServer ? ref toServer : ref fromServer, read); - await to.WriteAsync(buffer.AsMemory(0, read), stopping.Token); - await to.FlushAsync(stopping.Token); - } - } - catch (Exception ex) when (ex is OperationCanceledException or IOException or SocketException or ObjectDisposedException) - { - // Either side closing ends the copy; the counts up to that point are what matters. - } - } - } -} diff --git a/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs b/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs deleted file mode 100644 index e0940d777..000000000 --- a/examples/Tcp/Advanced/Tcp_025_ServerInfo.cs +++ /dev/null @@ -1,192 +0,0 @@ -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// and -/// : what the server said about itself during the handshake, and the two -/// different things to gate a feature on — the protocol revision for anything the wire has to carry, and -/// the server version for anything SQL has to name. -/// -/// -/// Code that runs against one server you control needs none of this. Code that ships — a library, a migration -/// tool, an agent deployed across a fleet — meets 25.8 and 26.7 on the same afternoon, and the choice is between -/// asking first and catching an error afterwards. Asking is cheaper and says why in the log. -/// -/// -public static class TcpServerInfo -{ - /// The protocol revision that added the query-parameters list to the Query packet. - private const int ParametersRevision = 54459; - - /// The revision that introduced per-packet chunk framing, used here as a gate that does not pass. - private const int ChunkedFramingRevision = 54470; - - /// The oldest server this driver is tested against. - private static readonly Version SupportedFloor = new(25, 8); - - /// QBit(Int8, N) needs a newer server; QBit itself does not. Tcp_015 is about the type. - private static readonly Version QBitInt8From = new(26, 7); - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - // One call, and the answer came from the handshake rather than from a query — there is no round trip - // beyond opening a connection, so this is cheap enough to do at startup. - // - // It describes the connection this call borrowed, not the client. Every query below borrows its own, so - // behind a load balancer over mixed versions a gate decided here can be wrong for the query it gates. - // Open a session when that matters: one session is one connection for its whole life. - ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); - - EveryField(server); - await VersionAgainstSqlVersion(client, server); - await GatingOnTheRevision(client, server); - await GatingOnTheVersion(client, server); - WhichGateToUse(); - } - - private static void EveryField(ClickHouseTcpServerInfo server) - { - Console.WriteLine("1. What the handshake carried\n"); - Console.WriteLine($" ToString() {server}"); - Console.WriteLine($" Name {server.Name}"); - Console.WriteLine($" Version {server.Version}"); - Console.WriteLine($" VersionMajor {server.VersionMajor}"); - Console.WriteLine($" VersionMinor {server.VersionMinor}"); - Console.WriteLine($" VersionPatch {server.VersionPatch}"); - Console.WriteLine($" ProtocolRevision {server.ProtocolRevision}"); - Console.WriteLine($" Timezone {Quote(server.Timezone)}"); - Console.WriteLine($" DisplayName {Quote(server.DisplayName)}"); - Console.WriteLine(); - Console.WriteLine(" Timezone is the server's own, and it is what a bare DateTime column means — Tcp_012 is"); - Console.WriteLine(" about that. DisplayName is whatever display_name the server was configured with, which"); - Console.WriteLine(" is often the container's host name and is empty when nothing set it."); - Console.WriteLine(); - Console.WriteLine(" ClickHouseTcpServerInfo is a record, so two readings compare equal by value and it is"); - Console.WriteLine(" safe to cache. None of it changes for the life of a connection."); - } - - private static async Task VersionAgainstSqlVersion(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) - { - Console.WriteLine("\n2. Version, and the fourth number that is not in it\n"); - - object sqlVersion = await client.ExecuteScalarAsync("SELECT version()"); - object sqlTimezone = await client.ExecuteScalarAsync("SELECT timezone()"); - - Console.WriteLine($" server.Version {server.Version}"); - Console.WriteLine($" SELECT version() {sqlVersion}"); - Console.WriteLine($" server.Timezone {server.Timezone}"); - Console.WriteLine($" SELECT timezone() {sqlTimezone}"); - Console.WriteLine(); - Console.WriteLine(" The handshake carries three numbers, so Version is major.minor.patch and the build"); - Console.WriteLine(" number that version() shows has nowhere to go. Compare against Version for a feature"); - Console.WriteLine(" gate — the three numbers are what a release note names — and read version() only when"); - Console.WriteLine(" you want the exact build for a bug report."); - } - - private static async Task GatingOnTheRevision(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) - { - Console.WriteLine("\n3. Gating on ProtocolRevision, for what the wire has to carry\n"); - Console.WriteLine(" The revision is the lower of what the client and the server support, so it can be"); - Console.WriteLine(" below what either alone offers. Everything the protocol grew — a field, a packet, a"); - Console.WriteLine(" framing — is switched on by a number like these.\n"); - - Console.WriteLine($" negotiated {server.ProtocolRevision}"); - Console.WriteLine($" query parameters need {ParametersRevision} -> {Available(server.ProtocolRevision >= ParametersRevision)}"); - Console.WriteLine($" per-packet chunk framing needs {ChunkedFramingRevision} -> {Available(server.ProtocolRevision >= ChunkedFramingRevision)}"); - Console.WriteLine(); - - if (server.ProtocolRevision >= ParametersRevision) - { - var options = new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "floor", 90UL } }, - }; - object count = await client.ExecuteScalarAsync( - "SELECT count() FROM numbers(100) WHERE number >= {floor:UInt64}", - options); - - Console.WriteLine($" So the parameterized query ran: count() = {count}."); - Console.WriteLine($" Below {ParametersRevision} the query packet has no field for the parameters list, so the client"); - Console.WriteLine(" throws NotSupportedException before it sends anything, rather than sending the query"); - Console.WriteLine(" unparameterized. The connection stays usable. Tcp_007 covers parameters themselves."); - } - else - { - Console.WriteLine($" Skipped the parameterized query: this connection negotiated {server.ProtocolRevision}, and query"); - Console.WriteLine($" parameters need {ParametersRevision}. Interpolate the value into the SQL text instead, and quote it."); - } - - Console.WriteLine(); - Console.WriteLine(" The chunk-framing row is a real gate rather than a hypothetical one: it is above the"); - Console.WriteLine(" revision this connection negotiated, so nothing on this connection uses it. That is the"); - Console.WriteLine(" asymmetry to remember — a newer server alone does not raise the number, because the"); - Console.WriteLine(" client has to offer the revision too."); - } - - private static async Task GatingOnTheVersion(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) - { - Console.WriteLine("\n4. Gating on Version, for what SQL has to name\n"); - - // The passing direction: the floor the driver is tested against. - bool supported = server.Version >= SupportedFloor; - Console.WriteLine($" this driver is tested from {SupportedFloor} upwards, and the server is {server.Version} -> {Available(supported)}"); - - // The failing direction, with the same shape Tcp_015 uses. A type the server refuses outright cannot be - // caught cheaply: the CREATE TABLE fails, so ask first. - if (server.Version >= QBitInt8From) - { - const string table = "example_tcp_serverinfo_qbit"; - await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); - try - { - await client.ExecuteAsync($"CREATE TABLE {table} (v QBit(Int8, 8)) ENGINE = MergeTree ORDER BY tuple()"); - // Qualified by database as well as by name: system.columns spans every database on the server. - object declared = await client.ExecuteScalarAsync( - $"SELECT type FROM system.columns WHERE database = currentDatabase() AND table = '{table}' AND name = 'v'"); - Console.WriteLine($" QBit(Int8, 8) declared as {declared}"); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); - } - } - else - { - Console.WriteLine($" QBit(Int8, N) needs ClickHouse {QBitInt8From} or newer, and this server is {server.Version}:"); - Console.WriteLine(" skipped. On 26.6 the server refuses the type outright — 'QBit data type only"); - Console.WriteLine(" supports BFloat16, Float32, or Float64 as element type' — so a client that"); - Console.WriteLine(" offers Int8 vectors has to know before it writes the DDL. Tcp_015 is about QBit."); - } - - Console.WriteLine(); - Console.WriteLine(" A skip that prints why is worth more than a caught exception: the reason survives into"); - Console.WriteLine(" the log, and nothing had to be attempted against a server that would refuse it."); - } - - private static void WhichGateToUse() - { - Console.WriteLine("\n5. Which of the two to read\n"); - Console.WriteLine(" ProtocolRevision anything the wire carries: query parameters, the fields of a"); - Console.WriteLine(" progress packet, chunk framing, custom serialization. The client"); - Console.WriteLine(" already gates its own reads and writes on it, so this matters when"); - Console.WriteLine(" your own code depends on a protocol-level capability."); - Console.WriteLine(); - Console.WriteLine(" Version anything SQL names: a data type, a function, a table setting, a"); - Console.WriteLine(" SETTINGS key. None of it is visible in the revision, because the"); - Console.WriteLine(" protocol did not change to carry it."); - Console.WriteLine(); - Console.WriteLine(" Two things this record does not tell you, and where to get them:"); - Console.WriteLine(" the cluster SELECT * FROM system.clusters"); - Console.WriteLine(" the build SELECT * FROM system.build_options"); - Console.WriteLine(); - Console.WriteLine(" For a health check, prefer PingAsync: it is a protocol ping rather than a SELECT, so it"); - Console.WriteLine(" proves the connection without asking the server to plan anything."); - } - - private static string Available(bool yes) => yes ? "available" : "not on this connection"; - - private static string Quote(string value) => value.Length == 0 ? "(empty)" : $"'{value}'"; -} diff --git a/examples/Tcp/Connection/Tcp_001_Sessions.cs b/examples/Tcp/Connection/Tcp_001_Sessions.cs new file mode 100644 index 000000000..d6d55007f --- /dev/null +++ b/examples/Tcp/Connection/Tcp_001_Sessions.cs @@ -0,0 +1,43 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Keeps temporary tables and settings on one pinned connection. +public static class TcpSessions +{ + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + object clientDefault = await client.ExecuteScalarAsync("SELECT getSetting('max_threads')"); + + // A session pins one connection, so temporary state survives between its operations. + await using IClickHouseTcpSession session = await client.OpenSessionAsync(); + + Console.WriteLine($"Session open: {session.IsOpen}"); + + await session.ExecuteAsync(""" + CREATE TEMPORARY TABLE example_tcp_session_values + (id UInt64, note String) + ENGINE = Memory + """); + await session.InsertRowsAsync( + "INSERT INTO example_tcp_session_values (id, note) VALUES", + new[] + { + new object[] { 1UL, "first" }, + new object[] { 2UL, "second" }, + }); + + object count = await session.ExecuteScalarAsync( + "SELECT count() FROM example_tcp_session_values"); + Console.WriteLine($"Temporary table rows: {count}"); + + await session.ExecuteAsync("SET max_threads = 2"); + object sessionValue = await session.ExecuteScalarAsync("SELECT getSetting('max_threads')"); + + Console.WriteLine($"max_threads in session: {sessionValue}"); + Console.WriteLine($"max_threads before the session: {clientDefault}"); + + // Sessions accept one operation at a time. Disposal closes the pinned connection and its state. + } +} diff --git a/examples/Tcp/Connection/Tcp_002_PoolTuning.cs b/examples/Tcp/Connection/Tcp_002_PoolTuning.cs new file mode 100644 index 000000000..20e19b95c --- /dev/null +++ b/examples/Tcp/Connection/Tcp_002_PoolTuning.cs @@ -0,0 +1,49 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Configures pool size, checkout timeout, lifetime, and reuse policy. +public static class TcpPoolTuning +{ + public static async Task Run() + { + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions() with + { + MinPoolSize = 0, + MaxPoolSize = 2, + PoolTimeout = TimeSpan.FromMilliseconds(250), + IdleTimeout = TimeSpan.FromMinutes(2), + MaxConnectionLifetime = TimeSpan.FromMinutes(30), + PoolReusePolicy = ClickHouseTcpPoolReusePolicy.Lifo, + }; + + await using var client = new ClickHouseTcpClient(options); + + Console.WriteLine( + $"Pool: min={options.MinPoolSize}, max={options.MaxPoolSize}, " + + $"checkout timeout={options.PoolTimeout}"); + + var stopwatch = Stopwatch.StartNew(); + Task[] queries = Enumerable.Range(0, 4) + .Select(_ => client.ExecuteScalarAsync("SELECT sleep(0.15)").AsTask()) + .ToArray(); + await Task.WhenAll(queries); + Console.WriteLine($"Four queries through a two-connection pool: {stopwatch.ElapsedMilliseconds} ms"); + + await using IClickHouseTcpSession first = await client.OpenSessionAsync(); + await using IClickHouseTcpSession second = await client.OpenSessionAsync(); + + // Both pool slots are pinned by sessions, so the next checkout reaches PoolTimeout. + stopwatch.Restart(); + try + { + await client.PingAsync(); + } + catch (TimeoutException ex) + { + Console.WriteLine( + $"Pool checkout timed out after {stopwatch.ElapsedMilliseconds} ms: {ex.Message}"); + } + } +} diff --git a/examples/Tcp/Connection/Tcp_003_Tls.cs b/examples/Tcp/Connection/Tcp_003_Tls.cs new file mode 100644 index 000000000..4a78280d2 --- /dev/null +++ b/examples/Tcp/Connection/Tcp_003_Tls.cs @@ -0,0 +1,39 @@ +using System.Security.Authentication; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Configures TLS for the native protocol and connects to an optional secure endpoint. +public static class TcpTls +{ + private const string TlsConnectionStringVariable = "CLICKHOUSE_TCP_TLS_CONNECTION_STRING"; + + public static async Task Run() + { + ClickHouseTcpClientOptions plain = ExampleConfig.TcpBuilder().ToOptions() with { Port = null }; + ClickHouseTcpClientOptions secure = plain with + { + UseTls = true, + TlsServerName = plain.Host, + ConfigureTls = tls => tls.EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + }; + + Console.WriteLine($"Plain endpoint: {plain.Host}:{plain.Port ?? 9000}"); + Console.WriteLine($"Secure endpoint: {secure.Host}:{secure.Port ?? 9440}"); + Console.WriteLine("With Port unset, TLS uses the native secure port 9440."); + + // Use TlsCaCertificatePath for a private CA. It replaces the host trust store. + // TlsAllowInvalidCertificates is intended only for local development. + + string? connectionString = Environment.GetEnvironmentVariable(TlsConnectionStringVariable); + if (string.IsNullOrWhiteSpace(connectionString)) + { + Console.WriteLine($"Set {TlsConnectionStringVariable} to test a TLS endpoint."); + return; + } + + await using var client = new ClickHouseTcpClient(connectionString); + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + Console.WriteLine($"Connected securely to {server}."); + } +} diff --git a/examples/Tcp/Connection/Tcp_004_Timeouts.cs b/examples/Tcp/Connection/Tcp_004_Timeouts.cs new file mode 100644 index 000000000..932ebd121 --- /dev/null +++ b/examples/Tcp/Connection/Tcp_004_Timeouts.cs @@ -0,0 +1,55 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Configures connection, pool, read, and operation timeouts. +public static class TcpTimeouts +{ + public static async Task Run() + { + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions() with + { + DialTimeout = TimeSpan.FromSeconds(5), + PoolTimeout = TimeSpan.FromSeconds(2), + ReadTimeout = TimeSpan.FromMilliseconds(150), + }; + + Console.WriteLine($"Dial timeout: {options.DialTimeout}"); + Console.WriteLine($"Pool timeout: {options.PoolTimeout}"); + Console.WriteLine($"Read timeout: {options.ReadTimeout}"); + + await using var client = new ClickHouseTcpClient(options); + + // ReadTimeout limits server silence, not total query duration. + var stopwatch = Stopwatch.StartNew(); + int rows = 0; + await foreach (object[] _ in client.QueryAsync( + "SELECT number, sleepEachRow(0.05) FROM numbers(6) SETTINGS max_block_size = 1")) + { + rows++; + } + + Console.WriteLine($"Read {rows} active rows in {stopwatch.ElapsedMilliseconds} ms."); + + try + { + await client.ExecuteScalarAsync("SELECT sleep(0.4)"); + } + catch (TimeoutException ex) + { + Console.WriteLine($"Read timeout during server silence: {ex.Message}"); + } + + // A cancellation token is the deadline for the complete operation. + using var cancellation = new CancellationTokenSource(TimeSpan.FromMilliseconds(150)); + try + { + await client.ExecuteScalarAsync("SELECT sleep(1)", cancellationToken: cancellation.Token); + } + catch (OperationCanceledException) + { + Console.WriteLine("The operation-wide cancellation deadline expired."); + } + } +} diff --git a/examples/Tcp/Connection/Tcp_016_Sessions.cs b/examples/Tcp/Connection/Tcp_016_Sessions.cs deleted file mode 100644 index f71152f11..000000000 --- a/examples/Tcp/Connection/Tcp_016_Sessions.cs +++ /dev/null @@ -1,209 +0,0 @@ -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// OpenSessionAsync: one connection out of the pool, pinned until the session is disposed, so state a -/// single connection holds — a temporary table, what a SET changed, which roles are active — survives from -/// one operation to the next. -/// -/// -/// Three rules come with that. One operation runs at a time, because the protocol carries one query per -/// connection. Disposal closes the connection instead of returning it to the pool, so no unrelated caller -/// can inherit the session's state. And a session holds one of the pool's slots for its whole lifetime, so keep it -/// short — Tcp_017_PoolTuning shows what happens when sessions outnumber the pool. -/// -/// -/// -/// A session is also the native answer to HTTP's per-query Roles, which this transport does not have: -/// SET ROLE inside a session applies to every operation that follows it and to nothing else. -/// -/// -public static class TcpSessions -{ - private const string RoleTable = "example_tcp_sessions_orders"; - private const string RoleName = "example_tcp_sessions_reader"; - private const string RoleUser = "example_tcp_sessions_user"; - - // The password of the user this example creates to demonstrate SET ROLE. Not a connection string: the - // endpoint still comes from ExampleConfig. - private const string RoleUserPassword = "example_tcp_sessions_pw"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - await StateThatSurvives(client); - await OneOperationAtATime(client); - await DisposalClosesTheConnection(); - await SetRoleInASession(client); - WhatToRemember(); - } - - private static async Task StateThatSurvives(ClickHouseTcpClient client) - { - Console.WriteLine("1. One pinned connection, so connection-local state survives\n"); - - await using IClickHouseTcpSession session = await client.OpenSessionAsync(); - Console.WriteLine($" Opened a session. IsOpen = {session.IsOpen}"); - - // A temporary table belongs to the connection that created it, which is why it needs a session at all. - // It also needs no cleanup: the server drops it when the connection closes, and disposing the session - // closes the connection. - await session.ExecuteAsync("CREATE TEMPORARY TABLE example_tcp_sessions_scratch (id UInt64, note String) ENGINE = Memory"); - await session.InsertRowsAsync( - "INSERT INTO example_tcp_sessions_scratch (id, note) VALUES", - new[] - { - new object[] { 1UL, "first" }, - new object[] { 2UL, "second" }, - }); - - object rows = await session.ExecuteScalarAsync("SELECT count() FROM example_tcp_sessions_scratch"); - Console.WriteLine($" Created a TEMPORARY TABLE, inserted 2 rows, read back {rows} — three operations, one connection"); - - const string visible = "SELECT count() FROM system.tables WHERE is_temporary AND name = 'example_tcp_sessions_scratch'"; - Console.WriteLine($" system.tables sees it inside the session: {await session.ExecuteScalarAsync(visible)}"); - - // The same client, but this operation takes whatever connection the pool hands out, so it is a different - // session on the server and the table is not there. - Console.WriteLine($" ... and not from the client's pool: {await client.ExecuteScalarAsync(visible)}"); - - // A SET is connection state too, so it lasts exactly as long as the session. - await session.ExecuteAsync("SET max_threads = 7"); - Console.WriteLine("\n After SET max_threads = 7 in the session:"); - Console.WriteLine($" getSetting('max_threads') in the session = {await session.ExecuteScalarAsync("SELECT getSetting('max_threads')")}"); - Console.WriteLine($" getSetting('max_threads') on the client = {await client.ExecuteScalarAsync("SELECT getSetting('max_threads')")}"); - Console.WriteLine(" A client-level setting reaches every operation instead; see Tcp_002's set_ keys."); - } - - private static async Task OneOperationAtATime(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. One operation at a time\n"); - - await using IClickHouseTcpSession session = await client.OpenSessionAsync(); - - // Not awaited yet: an async method body starts on the calling thread, so by the time this returns the - // session has already claimed its connection for this query. - ValueTask running = session.ExecuteScalarAsync("SELECT sleep(0.2), 'slow'"); - - Console.WriteLine($" A query is in flight. IsOpen = {session.IsOpen} (busy is not closed)"); - - try - { - _ = await session.ExecuteScalarAsync("SELECT 'me too'"); - } - catch (InvalidOperationException ex) - { - Console.WriteLine($" A second operation on the same session throws {ex.GetType().Name}:"); - Console.WriteLine($" {ex.Message}"); - } - - _ = await running; - Console.WriteLine($"\n The first one finished, so the session is free again: {await session.ExecuteScalarAsync("SELECT 'next'")}"); - Console.WriteLine(" To run operations at once, run them on the client: each takes its own connection."); - } - - private static async Task DisposalClosesTheConnection() - { - Console.WriteLine("\n3. Disposal closes the connection, it does not pool it\n"); - - // MaxPoolSize = 1 makes the experiment decisive. If disposal returned the connection to the pool, the - // second session would be handed the same one and would find the first session's temporary table. - await using var client = new ClickHouseTcpClient(ExampleConfig.TcpBuilder().ToOptions() with { MaxPoolSize = 1 }); - - const string visible = "SELECT count() FROM system.tables WHERE is_temporary AND name = 'example_tcp_sessions_handover'"; - - IClickHouseTcpSession first = await client.OpenSessionAsync(); - await using (first) - { - await first.ExecuteAsync("CREATE TEMPORARY TABLE example_tcp_sessions_handover (id UInt64) ENGINE = Memory"); - Console.WriteLine($" First session created a temporary table and sees it: {await first.ExecuteScalarAsync(visible)}"); - } - - Console.WriteLine($" Disposed it. IsOpen = {first.IsOpen}"); - - await using IClickHouseTcpSession second = await client.OpenSessionAsync(); - Console.WriteLine($" Second session, from a pool of exactly one connection, sees: {await second.ExecuteScalarAsync(visible)}"); - Console.WriteLine(" Zero, so the first connection was closed rather than reused. That is the point: a"); - Console.WriteLine(" caller must never inherit another caller's temporary tables and settings."); - Console.WriteLine(" It also means nothing has to be dropped — and that a session costs a reconnect."); - } - - private static async Task SetRoleInASession(ClickHouseTcpClient admin) - { - Console.WriteLine("\n4. SET ROLE, which is what this transport has instead of a per-query role\n"); - Console.WriteLine(" ClickHouseTcpQueryOptions carries no Roles: there is nowhere on the wire to put one per"); - Console.WriteLine(" query. A session is the equivalent, and it is per connection rather than per query.\n"); - - try - { - // Inside the try, so that a failure part way through this sequence still reaches the finally and - // drops whatever it did create. - await admin.ExecuteAsync($"CREATE OR REPLACE TABLE {RoleTable} (id UInt64) ENGINE = MergeTree ORDER BY id"); - await admin.ExecuteAsync($"CREATE ROLE OR REPLACE {RoleName}"); - await admin.ExecuteAsync($"GRANT SELECT ON {RoleTable} TO {RoleName}"); - await admin.ExecuteAsync($"CREATE USER OR REPLACE {RoleUser} IDENTIFIED WITH plaintext_password BY '{RoleUserPassword}'"); - await admin.ExecuteAsync($"GRANT {RoleName} TO {RoleUser}"); - Console.WriteLine($" Created user '{RoleUser}', role '{RoleName}' holding SELECT on '{RoleTable}'"); - - // Same server as everything else here; only the credentials differ. - var asUser = ExampleConfig.TcpBuilder(); - asUser.Username = RoleUser; - asUser.Password = RoleUserPassword; - - await using var client = new ClickHouseTcpClient(asUser.ToOptions()); - await using IClickHouseTcpSession session = await client.OpenSessionAsync(); - - Console.WriteLine($"\n Fresh session, granted roles active by default: {await session.ExecuteScalarAsync("SELECT toString(currentRoles())")}"); - - await session.ExecuteAsync("SET ROLE NONE"); - Console.WriteLine($" After SET ROLE NONE: {await session.ExecuteScalarAsync("SELECT toString(currentRoles())")}"); - - try - { - _ = await session.ExecuteScalarAsync($"SELECT count() FROM {RoleTable}"); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" Reading the table now fails with {ex.Code} ({ex.RawCode}), so the grant really came from the role."); - - // A server-side error in a query the server accepted does not end the session: the connection is - // still good, only the query failed. - Console.WriteLine($" IsOpen after that error = {session.IsOpen}, so the session carries on"); - } - - await session.ExecuteAsync($"SET ROLE {RoleName}"); - Console.WriteLine($" After SET ROLE {RoleName}, the read works again: {await session.ExecuteScalarAsync($"SELECT count() FROM {RoleTable}")} rows"); - - // The client's own operations run over other connections, which never saw either SET ROLE. - Console.WriteLine($"\n Meanwhile an operation on the client, over a pooled connection: {await client.ExecuteScalarAsync("SELECT toString(currentRoles())")}"); - Console.WriteLine(" Untouched. A SET ROLE reaches exactly the connection it ran on."); - } - finally - { - await admin.ExecuteAsync($"DROP USER IF EXISTS {RoleUser}"); - await admin.ExecuteAsync($"DROP ROLE IF EXISTS {RoleName}"); - await admin.ExecuteAsync($"DROP TABLE IF EXISTS {RoleTable}"); - Console.WriteLine($"\n Dropped the user, the role and '{RoleTable}'. The temporary tables above needed no cleanup."); - } - } - - private static void WhatToRemember() - { - Console.WriteLine("\n5. Worth knowing before you open one\n"); - Console.WriteLine(" Keep it short. A session holds one of MaxPoolSize connections from OpenSessionAsync"); - Console.WriteLine(" until disposal, so as many sessions as the pool is wide leaves nothing for anything"); - Console.WriteLine(" else, and the next caller waits out PoolTimeout and then fails. See Tcp_017."); - Console.WriteLine(); - Console.WriteLine(" Finish what you stream. A StreamAsync or QueryAsync result holds the session until it"); - Console.WriteLine(" is read to the end or its enumerator is disposed — 'await foreach' does that for you."); - Console.WriteLine(" One left suspended mid-enumeration and never disposed cannot give its slot back at all,"); - Console.WriteLine(" which is the one thing here not demonstrated: showing it means leaking a connection."); - Console.WriteLine(); - Console.WriteLine(" Read IsOpen as a floor, not a promise. False is certain: the session is finished, its"); - Console.WriteLine(" server-side state is gone, and the answer is a new session rather than a retry. True"); - Console.WriteLine(" only means nothing is known to be wrong. A failed transport, a cancellation, or a"); - Console.WriteLine(" half-read stream ends a session; a server error in a query the server accepted does not."); - } -} diff --git a/examples/Tcp/Connection/Tcp_017_PoolTuning.cs b/examples/Tcp/Connection/Tcp_017_PoolTuning.cs deleted file mode 100644 index 020ee92dc..000000000 --- a/examples/Tcp/Connection/Tcp_017_PoolTuning.cs +++ /dev/null @@ -1,375 +0,0 @@ -using System.Diagnostics; -using ClickHouse.Driver.Tcp; -using Microsoft.Extensions.Logging; - -namespace ClickHouse.Driver.Examples; - -/// -/// Sizing the native client's connection pool: MinPoolSize, MaxPoolSize, PoolTimeout, -/// IdleTimeout, MaxConnectionLifetime, SweepInterval and PoolReusePolicy — what each -/// one does, and what it looks like when it acts. -/// -/// -/// One connection carries one query, so MaxPoolSize is the client's concurrency limit, for the whole -/// process rather than per caller. Everything below is measured: the counts come from the pool's own log and from -/// system.processes on the server, because nothing on the client reports how many connections are open, -/// idle or in use. -/// -/// -/// -/// The lifetime limits default to minutes, which an example cannot wait out, so the sections that show them set -/// them to milliseconds. The code path is the same one a 30-minute limit takes. -/// -/// -public static class TcpPoolTuning -{ - public static async Task Run() - { - TheKnobs(); - await ReuseAndTheOnlyWindowIntoThePool(); - await MaxPoolSizeCapsConcurrency(); - await PoolTimeoutExpires(); - await RetirementAndTheSweep(); - await LifoAgainstFifo(); - await WhatADataSourceShares(); - } - - private static void TheKnobs() - { - ClickHouseTcpClientOptions defaults = new(); - - Console.WriteLine("1. The pool keys and their defaults\n"); - Console.WriteLine($" MinPoolSize {defaults.MinPoolSize,-8} connections kept open when the pool can"); - Console.WriteLine($" MaxPoolSize {defaults.MaxPoolSize,-8} hard cap, and so the concurrency limit"); - Console.WriteLine($" PoolTimeout {defaults.PoolTimeout.TotalSeconds + "s",-8} wait for a slot before TimeoutException"); - Console.WriteLine($" IdleTimeout {defaults.IdleTimeout.TotalMinutes + "m",-8} unused for this long, and it is retired"); - Console.WriteLine($" MaxConnectionLifetime {defaults.MaxConnectionLifetime.TotalMinutes + "m",-8} open for this long, and it is retired"); - Console.WriteLine($" SweepInterval {"derived",-8} how often the pool looks for work to do"); - Console.WriteLine($" PoolReusePolicy {defaults.PoolReusePolicy,-8} which idle connection is handed out next"); - Console.WriteLine(); - Console.WriteLine(" TimeSpan.Zero opts out of IdleTimeout and MaxConnectionLifetime; PoolTimeout has to be"); - Console.WriteLine(" positive. A null SweepInterval derives the period as a quarter of the shorter of the two"); - Console.WriteLine(" limits, held between 1 and 30 seconds — 30 seconds at these defaults. The derived value"); - Console.WriteLine(" is not exposed, so the rule is the only way to know it."); - - // The same keys exist on the connection string, so a deployment can size the pool without a rebuild. - var builder = ExampleConfig.TcpBuilder(); - builder.MinPoolSize = 2; - builder.MaxPoolSize = 8; - builder.PoolTimeout = TimeSpan.FromSeconds(5); - builder.IdleTimeout = TimeSpan.FromSeconds(45); - builder.MaxConnectionLifetime = TimeSpan.FromMinutes(10); - builder.PoolReusePolicy = ClickHouseTcpPoolReusePolicy.Fifo; - - ClickHouseTcpClientOptions tuned = builder.ToOptions(); - Console.WriteLine("\n The same thing through the connection string (MinPoolSize=2;MaxPoolSize=8;...):"); - Console.WriteLine($" Min {tuned.MinPoolSize}, Max {tuned.MaxPoolSize}, PoolTimeout {tuned.PoolTimeout}, IdleTimeout {tuned.IdleTimeout}, Lifetime {tuned.MaxConnectionLifetime}, {tuned.PoolReusePolicy}"); - } - - private static async Task ReuseAndTheOnlyWindowIntoThePool() - { - Console.WriteLine("\n2. What the pool will tell you\n"); - Console.WriteLine(" There are no counters to read, so the pool's log is the window into it. These lines come"); - Console.WriteLine(" from the ClickHouse.Driver.Tcp.Pool and .Connection categories, at Debug and Trace.\n"); - - var capture = new LogCapture(); - await using (var client = new ClickHouseTcpClient(Options() with { LoggerFactory = capture })) - { - _ = await client.ExecuteScalarAsync("SELECT 1"); - _ = await client.ExecuteScalarAsync("SELECT 2"); - } - - Print(capture.Lines.Where(l => !l.StartsWith("Client", StringComparison.Ordinal))); - Console.WriteLine("\n Two queries, one connection: the first opened it, the second reused it, and the drain"); - Console.WriteLine(" at disposal closed it. 'its 2 operation' is that connection's use count."); - } - - private static async Task MaxPoolSizeCapsConcurrency() - { - Console.WriteLine("\n3. MaxPoolSize caps how many operations run at once\n"); - Console.WriteLine(" Four queries, each sleeping 150 ms, started together. A second client watches"); - Console.WriteLine(" system.processes to see how many of them the server is really running.\n"); - - await Measure(maxPoolSize: 2); - await Measure(maxPoolSize: 4); - - Console.WriteLine("\n The queries are not lost when the pool is full, only queued: each waits for a slot for"); - Console.WriteLine(" up to PoolTimeout. So MaxPoolSize is a throughput knob, and PoolTimeout is the deadline"); - Console.WriteLine(" on getting one of its slots."); - } - - private static async Task Measure(int maxPoolSize) - { - // Unique per call: the count below matches on the marker, so a second run of this example would - // otherwise be counted too and could report more running queries than this client's pool allows. - string marker = $"example_tcp_pool_cap_{maxPoolSize}_{Guid.NewGuid():N}"; - var capture = new LogCapture(); - - await using var observer = ExampleConfig.CreateTcpClient(); - await using var client = new ClickHouseTcpClient(Options() with { MaxPoolSize = maxPoolSize, LoggerFactory = capture }); - - var clock = Stopwatch.StartNew(); - Task work = Task.WhenAll(Enumerable.Range(0, 4).Select(_ => Task.Run(async () => - await client.ExecuteScalarAsync($"SELECT sleep(0.15) /* {marker} */")))); - - // The marker is a comment, so it appears in the query text the server reports. The observer's own query - // carries it too, hence the second condition. - string count = $"SELECT count() FROM system.processes WHERE query LIKE '%{marker}%' AND query NOT LIKE '%system.processes%'"; - - int mostSeen = 0; - while (!work.IsCompleted && clock.ElapsedMilliseconds < 5000) - { - mostSeen = Math.Max(mostSeen, Convert.ToInt32(await observer.ExecuteScalarAsync(count))); - await Task.Delay(20); - } - - await work; - long elapsed = clock.ElapsedMilliseconds; - - Console.WriteLine($" MaxPoolSize = {maxPoolSize}"); - Console.WriteLine($" connections opened, from the pool log : {capture.Count("opening one")}"); - Console.WriteLine($" most running at once, from the server : {mostSeen}"); - Console.WriteLine($" wall clock for all four : {elapsed} ms"); - } - - private static async Task PoolTimeoutExpires() - { - Console.WriteLine("\n4. PoolTimeout, when there is nothing left to hand out\n"); - - var capture = new LogCapture(); - await using var client = new ClickHouseTcpClient(Options() with - { - MaxPoolSize = 1, - PoolTimeout = TimeSpan.FromMilliseconds(250), - LoggerFactory = capture, - }); - - // A session pins its connection for its whole lifetime, so one session against a pool of one is an - // exhausted pool — no sleeping query needed. - await using IClickHouseTcpSession session = await client.OpenSessionAsync(); - Console.WriteLine(" MaxPoolSize = 1, PoolTimeout = 250 ms, and a session holds the only connection."); - - var clock = Stopwatch.StartNew(); - try - { - _ = await client.ExecuteScalarAsync("SELECT 1"); - } - catch (TimeoutException ex) - { - Console.WriteLine($"\n A query on the client threw TimeoutException after {clock.ElapsedMilliseconds} ms:"); - Console.WriteLine($" {ex.Message}"); - } - - Console.WriteLine("\n The pool logged it too:"); - Print(capture.Lines.Where(l => l.Contains("PoolTimeout", StringComparison.Ordinal))); - Console.WriteLine(); - Console.WriteLine(" Raising PoolTimeout only makes the caller wait longer for a pool that is too small."); - Console.WriteLine(" The message also names the other cause: a streamed result nobody finished still holds"); - Console.WriteLine(" its connection."); - } - - private static async Task RetirementAndTheSweep() - { - Console.WriteLine("\n5. Retiring connections: MaxConnectionLifetime, IdleTimeout, SweepInterval, MinPoolSize\n"); - - // Age is read at checkout and at return, so a 1 ms limit means no connection is ever reused. - var byAge = new LogCapture(); - await using (var client = new ClickHouseTcpClient(Options() with - { - MaxConnectionLifetime = TimeSpan.FromMilliseconds(1), - LoggerFactory = byAge, - })) - { - _ = await client.ExecuteScalarAsync("SELECT 1"); - _ = await client.ExecuteScalarAsync("SELECT 2"); - } - - Console.WriteLine(" MaxConnectionLifetime = 1 ms, two queries:"); - Print(byAge.Lines.Where(l => l.StartsWith("Pool", StringComparison.Ordinal))); - Console.WriteLine(" No reuse at all: the connection is over age by the time it comes back, so it is closed"); - Console.WriteLine(" on return and the next query opens another. That check is between operations, never"); - Console.WriteLine(" inside one, so no query is ever cut short by it.\n"); - - // Idle retirement is the sweep's work, so it happens without any operation to trigger it. - var byIdle = new LogCapture(); - await using (var client = new ClickHouseTcpClient(Options() with - { - IdleTimeout = TimeSpan.FromMilliseconds(150), - SweepInterval = TimeSpan.FromMilliseconds(100), - LoggerFactory = byIdle, - })) - { - _ = await client.ExecuteScalarAsync("SELECT 1"); - long waited = await WaitFor(byIdle, "Retired"); - Console.WriteLine(" IdleTimeout = 150 ms, SweepInterval = 100 ms, one query then nothing:"); - Print(byIdle.Lines.Where(l => l.Contains("Retired", StringComparison.Ordinal))); - Console.WriteLine($" The sweep retired it {waited} ms after the query, with no operation involved."); - } - - Console.WriteLine(); - - // The same sweep restores the floor, which is why MinPoolSize needs no traffic to take effect. - var byFloor = new LogCapture(); - await using (var client = new ClickHouseTcpClient(Options() with - { - MinPoolSize = 3, - MaxPoolSize = 5, - SweepInterval = TimeSpan.FromMilliseconds(100), - LoggerFactory = byFloor, - })) - { - long waited = await WaitFor(byFloor, "Connected to ClickHouse", occurrences: 3); - Console.WriteLine(" MinPoolSize = 3, SweepInterval = 100 ms, and not one query run:"); - Console.WriteLine($" connections opened by the sweep: {byFloor.Count("Connected to ClickHouse")} after {waited} ms"); - } - - Console.WriteLine(); - Console.WriteLine(" The floor and IdleTimeout multiply: neither limit respects MinPoolSize, so a quiet pool"); - Console.WriteLine(" retires its connections and the sweep opens replacements. A floor of 10 against a"); - Console.WriteLine(" 5-second idle limit is 10 handshakes every 5 seconds from an idle application. Size the"); - Console.WriteLine(" two together. Set IdleTimeout below the shortest idle timeout on the path to the server:"); - Console.WriteLine(" a proxy that drops an idle connection without a FIN leaves one that only looks alive."); - } - - private static async Task LifoAgainstFifo() - { - Console.WriteLine("\n6. PoolReusePolicy: which idle connection comes back out\n"); - Console.WriteLine(" Three queries at once fill a pool of three, then three run one after another. The use"); - Console.WriteLine(" count in the reuse line says whether they landed on one connection or on all three.\n"); - - foreach (ClickHouseTcpPoolReusePolicy policy in new[] { ClickHouseTcpPoolReusePolicy.Lifo, ClickHouseTcpPoolReusePolicy.Fifo }) - { - var capture = new LogCapture(); - await using var client = new ClickHouseTcpClient(Options() with - { - MaxPoolSize = 3, - PoolReusePolicy = policy, - LoggerFactory = capture, - }); - - await Task.WhenAll(Enumerable.Range(0, 3).Select(_ => Task.Run(async () => - await client.ExecuteScalarAsync("SELECT sleep(0.15)")))); - - for (int i = 0; i < 3; i++) - { - _ = await client.ExecuteScalarAsync("SELECT 1"); - } - - // "Reusing a pooled connection, its N operation, ..." — N is that connection's use count, which is - // what tells one policy from the other. - IEnumerable counts = capture.Lines - .Where(l => l.Contains("Reusing", StringComparison.Ordinal)) - .Select(l => l[(l.IndexOf("its ", StringComparison.Ordinal) + 4)..].Split(' ')[0]); - - string shape = policy == ClickHouseTcpPoolReusePolicy.Lifo - ? "one connection, used again and again" - : "each of the three in turn"; - - Console.WriteLine($" {policy,-4} use count of the connection each sequential query got: {string.Join(", ", counts)} ({shape})"); - } - - Console.WriteLine(); - Console.WriteLine(" Lifo keeps returning to the connection that came back last, so traffic concentrates on a"); - Console.WriteLine(" hot few and the rest go idle and close — a pool sized for peak load costs little"); - Console.WriteLine(" off-peak. Fifo spreads the work, so under steady load every connection is used again"); - Console.WriteLine(" inside its idle window and the whole pool stays warm. Both are equally correct: age,"); - Console.WriteLine(" idleness and liveness are checked whichever end the connection comes from."); - } - - private static async Task WhatADataSourceShares() - { - Console.WriteLine("\n7. What a ClickHouseTcpDataSource shares\n"); - - await using var dataSource = new ClickHouseTcpDataSource(Options() with { MaxPoolSize = 8 }); - - Console.WriteLine($" One data source owns one client, and that client owns one pool: {dataSource.Options.MaxPoolSize} connections"); - Console.WriteLine(" for every consumer that is injected with it (Tcp_003 registers one). So MaxPoolSize is"); - Console.WriteLine(" the whole application's concurrency budget, not each service's."); - Console.WriteLine(); - Console.WriteLine(" Two data sources, or two clients built with 'new', are two pools that share nothing but"); - Console.WriteLine(" the server. That is what a keyed registration per endpoint buys, and it is also the"); - Console.WriteLine(" accident behind a client built per request: every one pays a handshake and none of them"); - Console.WriteLine(" reuses anything."); - Console.WriteLine(); - Console.WriteLine(" Sizing, in short: MaxPoolSize at or a little above the number of operations you want in"); - Console.WriteLine(" flight, remembering each inserting connection can buffer MaxSendBufferBytes (Tcp_019);"); - Console.WriteLine(" MinPoolSize only where a cold first query matters; and one slot per session you hold."); - } - - private static ClickHouseTcpClientOptions Options() => ExampleConfig.TcpBuilder().ToOptions(); - - private static void Print(IEnumerable lines) - { - foreach (string line in lines) - { - Console.WriteLine($" {line}"); - } - } - - /// Waits for the pool to log something, so the example never sleeps longer than it must. - private static async Task WaitFor(LogCapture capture, string contains, int occurrences = 1) - { - var clock = Stopwatch.StartNew(); - while (clock.ElapsedMilliseconds < 3000 && capture.Count(contains) < occurrences) - { - await Task.Delay(25); - } - - return clock.ElapsedMilliseconds; - } - - /// - /// An that keeps the lines instead of printing them, so the example can show only - /// the ones under discussion. A real application passes the container's factory; see Tcp_003. - /// - private sealed class LogCapture : ILoggerFactory - { - private readonly List lines = []; - - public IReadOnlyList Lines - { - get - { - lock (lines) - { - return lines.ToArray(); - } - } - } - - public int Count(string contains) - => Lines.Count(l => l.Contains(contains, StringComparison.Ordinal)); - - public ILogger CreateLogger(string categoryName) => new Sink(categoryName, lines); - - public void AddProvider(ILoggerProvider provider) - { - } - - public void Dispose() - { - } - - private sealed class Sink(string category, List lines) : ILogger - { - // The client asks before formatting, so answering true is what makes Trace-level lines appear. - public bool IsEnabled(LogLevel logLevel) => true; - - public IDisposable? BeginScope(TState state) - where TState : notnull => null; - - public void Log( - LogLevel logLevel, - EventId eventId, - TState state, - Exception? exception, - Func formatter) - { - lock (lines) - { - lines.Add($"{category[(category.LastIndexOf('.') + 1)..]}: {formatter(state, exception)}"); - } - } - } - } -} diff --git a/examples/Tcp/Connection/Tcp_018_Tls.cs b/examples/Tcp/Connection/Tcp_018_Tls.cs deleted file mode 100644 index 782eaa1ec..000000000 --- a/examples/Tcp/Connection/Tcp_018_Tls.cs +++ /dev/null @@ -1,259 +0,0 @@ -using System.Net.Security; -using System.Security.Authentication; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// Encrypting the native transport: UseTls, TlsServerName, TlsCaCertificatePath, -/// TlsAllowInvalidCertificates and the ConfigureTls hook — plus the port, which follows -/// UseTls to 9440 unless one is given. -/// -/// -/// The native handshake sends the password as plaintext, so on any untrusted network TLS is the only thing -/// protecting the credentials. Nothing above the transport changes: the protocol sends the same bytes either way. -/// -/// -/// -/// The examples' server is plaintext, so the connecting part of this example is opt-in: set -/// CLICKHOUSE_TCP_TLS_CONNECTION_STRING to a secure endpoint and it runs. Everything else here needs no -/// server at all, because the client checks its TLS configuration when it is constructed — including reading the -/// certificate authority file. -/// -/// -public static class TcpTls -{ - private const string TlsConnectionStringVariable = "CLICKHOUSE_TCP_TLS_CONNECTION_STRING"; - - public static async Task Run() - { - WhatTlsIsFor(); - ThePortFollowsUseTls(); - BothWaysToSetIt(); - CheckedAtConstruction(); - PinningAnAuthorityReplacesTheTrustStore(); - TheEscapeHatch(); - await ConnectIfAnEndpointWasGiven(); - } - - private static void WhatTlsIsFor() - { - Console.WriteLine("1. Why TLS, on this transport\n"); - Console.WriteLine(" The native protocol's first packet carries the username and password in the clear, and"); - Console.WriteLine(" then every block of data. UseTls encrypts the socket underneath all of it. The protocol"); - Console.WriteLine(" bytes are identical either way, so nothing above the transport changes."); - Console.WriteLine(); - Console.WriteLine(" TLS is not negotiated in band. The server has to be listening for secure native"); - Console.WriteLine(" connections (tcp_port_secure, conventionally 9440), and a TLS client pointed at a"); - Console.WriteLine(" plaintext port fails its handshake rather than falling back to plaintext."); - } - - private static void ThePortFollowsUseTls() - { - Console.WriteLine("\n2. The port comes from UseTls when you do not give one\n"); - - // Port is int?, and null is not "0" but "derive it". ToString shows the port a connection would dial. - ClickHouseTcpClientOptions plain = ExampleConfig.TcpBuilder().ToOptions() with { Port = null }; - ClickHouseTcpClientOptions secure = plain with { UseTls = true }; - ClickHouseTcpClientOptions explicitPort = secure with { Port = 19440 }; - - Console.WriteLine($" UseTls = false, Port unset : {plain}"); - Console.WriteLine($" UseTls = true, Port unset : {secure}"); - Console.WriteLine($" UseTls = true, Port 19440 : {explicitPort}"); - Console.WriteLine(); - Console.WriteLine(" So switching a deployment to TLS is one key, as long as the server uses the conventional"); - Console.WriteLine(" port. An explicit Port is always used as given."); - } - - private static void BothWaysToSetIt() - { - Console.WriteLine("\n3. The same four keys, in a connection string and on the options record\n"); - - var builder = ExampleConfig.TcpBuilder(); - builder.Port = null; - builder.UseTls = true; - builder.TlsServerName = "clickhouse.internal"; - - // Naming an authority file here touches nothing: it is read when a client is constructed, and this - // example never constructs one from these options. - builder.TlsCaCertificatePath = "/etc/ssl/ca.pem"; - - ClickHouseTcpClientOptions fromBuilder = builder.ToOptions(); - - Console.WriteLine(" UseTls=true;TlsServerName=clickhouse.internal;TlsCaCertificatePath=/etc/ssl/ca.pem"); - Console.WriteLine(" TlsAllowInvalidCertificates=false"); - Console.WriteLine(); - Console.WriteLine($" builder.ToOptions() : {fromBuilder}"); - Console.WriteLine($" TlsServerName {fromBuilder.TlsServerName}"); - Console.WriteLine($" TlsCaCertificatePath {fromBuilder.TlsCaCertificatePath ?? "(null: the host trust store)"}"); - Console.WriteLine($" TlsAllowInvalidCertificates {fromBuilder.TlsAllowInvalidCertificates}"); - Console.WriteLine(" Port left unset, so the rendered options above show the 9440 it resolved to"); - Console.WriteLine(); - Console.WriteLine(" TlsServerName is the name presented as SNI and matched against the certificate; it"); - Console.WriteLine(" defaults to Host, so set it only when Host is an address or an alias the certificate"); - Console.WriteLine(" does not name. ConfigureTls is the one TLS setting with no connection-string key: it is"); - Console.WriteLine(" a delegate, so it can only be set in code."); - } - - private static void CheckedAtConstruction() - { - Console.WriteLine("\n4. What is refused before anything connects\n"); - Console.WriteLine(" Every line below comes from constructing a client, with no server involved.\n"); - - ClickHouseTcpClientOptions plaintext = ExampleConfig.TcpBuilder().ToOptions(); - - // A TLS setting on a client that does not use TLS is refused rather than ignored. Ignoring it is how a - // connection meant to be encrypted ends up in the clear with a configured authority as the only evidence. - Refused("TlsServerName, UseTls left false", plaintext with { TlsServerName = "clickhouse.internal" }); - Refused("TlsAllowInvalidCertificates, UseTls left false", plaintext with { TlsAllowInvalidCertificates = true }); - Refused("TlsCaCertificatePath, UseTls left false", plaintext with { TlsCaCertificatePath = "/etc/ssl/ca.pem" }); - Refused("ConfigureTls, UseTls left false", plaintext with { ConfigureTls = _ => { } }); - - ClickHouseTcpClientOptions tls = plaintext with { UseTls = true }; - - // Contradictory rather than merely redundant: with validation off, the authority would be read and never - // consulted. - Refused("TlsAllowInvalidCertificates and TlsCaCertificatePath together", tls with - { - TlsAllowInvalidCertificates = true, - TlsCaCertificatePath = "/etc/ssl/ca.pem", - }); - - Refused("A blank TlsCaCertificatePath", tls with { TlsCaCertificatePath = " " }); - - // The authority file is read once, when the client is constructed, so a wrong path or an unparseable file - // fails here instead of on the first connection — or worse, on the first reconnect at 3am. - string missing = Path.Combine(Path.GetTempPath(), "example-tcp-tls-no-such-ca.pem"); - Refused("A TlsCaCertificatePath that does not exist", tls with { TlsCaCertificatePath = missing }); - - string notACertificate = Path.Combine(Path.GetTempPath(), "example-tcp-tls-not-a-ca.pem"); - try - { - File.WriteAllText(notACertificate, "these are not the certificates you are looking for\n"); - Refused("A TlsCaCertificatePath that is not a PEM certificate", tls with { TlsCaCertificatePath = notACertificate }); - } - finally - { - File.Delete(notACertificate); - } - - // The connection-string parser is strict about these two keys for the same reason: a value it cannot read - // must not quietly become the plaintext default. - try - { - _ = ClickHouseTcpClientOptions.FromConnectionString("Host=clickhouse.example;UseTls=perhaps"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" UseTls=perhaps in a connection string -> {ex.GetType().Name}"); - Console.WriteLine($" {ex.Message}"); - } - } - - private static void Refused(string what, ClickHouseTcpClientOptions options) - { - try - { - // Constructed only to be refused: nothing here reaches a socket. - using var client = new ClickHouseTcpClient(options); - Console.WriteLine($" {what} -> accepted, which is not what this example expected"); - } - catch (Exception ex) when (ex is ArgumentException or IOException) - { - Console.WriteLine($" {what} -> {ex.GetType().Name}"); - Console.WriteLine($" {ex.Message.Split(" (Parameter")[0]}"); - } - } - - private static void PinningAnAuthorityReplacesTheTrustStore() - { - Console.WriteLine("\n5. TlsCaCertificatePath replaces the host's trust store — it does not add to it\n"); - Console.WriteLine(" Set it and the server must chain to one of the authorities in that file. A certificate"); - Console.WriteLine(" the host would have accepted on its own is then refused. That is the point of naming an"); - Console.WriteLine(" authority: an additive check would still accept a certificate mis-issued by any of the"); - Console.WriteLine(" hundred-odd public authorities the host trusts."); - Console.WriteLine(); - Console.WriteLine(" So a private authority is the case it serves. Pointing it at a public root to 'also"); - Console.WriteLine(" allow' a private one does not work, and pinning it in front of a server whose"); - Console.WriteLine(" certificate is publicly issued breaks that server."); - Console.WriteLine(); - Console.WriteLine(" The file must hold at least one self-issued root, which is what the chain is anchored"); - Console.WriteLine(" to; it may also hold intermediates, which are used only to build a chain to an anchor."); - Console.WriteLine(" Host name matching still happens either way — pinning roots does not replace it."); - Console.WriteLine(); - Console.WriteLine(" TlsAllowInvalidCertificates is the other thing entirely: it stops the client checking"); - Console.WriteLine(" that the peer is the server it asked for, so anyone who can intercept the connection can"); - Console.WriteLine(" present any certificate and read the handshake password. For a private authority, pin"); - Console.WriteLine(" the root and keep the check."); - } - - private static void TheEscapeHatch() - { - Console.WriteLine("\n6. ConfigureTls, for what the four keys do not cover\n"); - - ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions() with - { - UseTls = true, - ConfigureTls = tls => - { - tls.EnabledSslProtocols = SslProtocols.Tls13; - - // Where a client certificate would go: tls.ClientCertificates = new X509Certificate2Collection(cert); - }, - }; - - // The client calls this once per connection, just before the handshake. Calling it here, on a fresh - // options object, is only to show what it sets. - var authentication = new SslClientAuthenticationOptions(); - options.ConfigureTls(authentication); - - Console.WriteLine($" The hook set EnabledSslProtocols = {authentication.EnabledSslProtocols}"); - Console.WriteLine(); - Console.WriteLine(" It runs last, after everything the four keys set, which is what makes it an escape"); - Console.WriteLine(" hatch — client certificates, a protocol floor, cipher suites, a validation callback of"); - Console.WriteLine(" your own — and also what lets it weaken the transport in two ways that are easy to miss:"); - Console.WriteLine(); - Console.WriteLine(" replacing RemoteCertificateValidationCallback drops the check the keys configured;"); - Console.WriteLine(" clearing TargetHost stops the server name being matched at all, while chain"); - Console.WriteLine(" validation still appears to run."); - Console.WriteLine(); - Console.WriteLine(" With TlsCaCertificatePath set, a CertificateChainPolicy is already in place and the hook"); - Console.WriteLine(" receives it and may edit it. .NET then ignores CertificateRevocationCheckMode, so"); - Console.WriteLine(" revocation goes through that policy's own RevocationMode."); - } - - private static async Task ConnectIfAnEndpointWasGiven() - { - Console.WriteLine("\n7. Connecting over TLS\n"); - - string? connectionString = Environment.GetEnvironmentVariable(TlsConnectionStringVariable); - if (string.IsNullOrWhiteSpace(connectionString)) - { - Console.WriteLine($" Skipped: {TlsConnectionStringVariable} is not set."); - Console.WriteLine(); - Console.WriteLine(" The server these examples run against speaks plaintext on 9000, and the one CI"); - Console.WriteLine(" starts publishes 8123 and 9000 only, so there is no secure port to dial. Nothing"); - Console.WriteLine(" here fakes one: a certificate invented to make a connection succeed would teach the"); - Console.WriteLine(" wrong thing, and turning validation off to get past it would teach something worse."); - Console.WriteLine(); - Console.WriteLine(" To run this section, point it at a server listening on tcp_port_secure:"); - Console.WriteLine($" export {TlsConnectionStringVariable}=\"Host=my-host;UseTls=true;Username=default;Password=...\""); - Console.WriteLine(" A ClickHouse Cloud service is the easy case: its native endpoint is TLS on 9440 with"); - Console.WriteLine(" a publicly issued certificate, so UseTls=true and no other TLS key is needed."); - return; - } - - Console.WriteLine($" {TlsConnectionStringVariable} is set, so connecting over TLS."); - - await using var client = new ClickHouseTcpClient(connectionString); - ClickHouseTcpClientOptions options = client.Options; - - Console.WriteLine($" {options}"); - Console.WriteLine($" UseTls {options.UseTls}, TlsServerName {options.TlsServerName ?? "(Host)"}, " + - $"CA {options.TlsCaCertificatePath ?? "(host trust store)"}, AllowInvalid {options.TlsAllowInvalidCertificates}"); - - ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); - object now = await client.ExecuteScalarAsync("SELECT 'encrypted'"); - Console.WriteLine($" Connected to {server}: SELECT returned {now}"); - } -} diff --git a/examples/Tcp/Connection/Tcp_019_Timeouts.cs b/examples/Tcp/Connection/Tcp_019_Timeouts.cs deleted file mode 100644 index d98b4ae58..000000000 --- a/examples/Tcp/Connection/Tcp_019_Timeouts.cs +++ /dev/null @@ -1,323 +0,0 @@ -using System.Diagnostics; -using System.Net; -using System.Net.Sockets; -using ClickHouse.Driver.Tcp; -using Microsoft.Extensions.Logging; - -namespace ClickHouse.Driver.Examples; - -/// -/// The native client's deadlines and limits: DialTimeout, ReadTimeout, PoolTimeout, -/// StatementMaxLength and MaxSendBufferBytes. -/// -/// -/// The three deadlines cover three different phases and never overlap: PoolTimeout bounds the wait for a -/// pool slot, DialTimeout bounds the connect and handshake that may follow it, and ReadTimeout bounds -/// how long the server may stay silent while a response is being read. That last one is the one to -/// understand — it measures silence, not duration, so a query that streams for an hour never trips it. -/// -/// -/// -/// None of them bounds a whole operation. That is what a CancellationToken is for, and every method takes -/// one; Tcp_022_Cancellation is about what cancelling does to the connection. -/// -/// -public static class TcpTimeouts -{ - public static async Task Run() - { - WhichDeadlineCoversWhat(); - await DialingTheWrongThing(); - await ReadTimeoutMeasuresSilence(); - PoolTimeoutInOneLine(); - await StatementMaxLengthCapsWhatIsLogged(); - MaxSendBufferBytesAndTheValues(); - } - - private static void WhichDeadlineCoversWhat() - { - ClickHouseTcpClientOptions defaults = new(); - - Console.WriteLine("1. Four bounds, four different phases\n"); - Console.WriteLine($" PoolTimeout {defaults.PoolTimeout.TotalSeconds,5}s waiting for one of MaxPoolSize connections"); - Console.WriteLine($" DialTimeout {defaults.DialTimeout.TotalSeconds,5}s socket connect plus the protocol handshake"); - Console.WriteLine($" ReadTimeout {defaults.ReadTimeout.TotalSeconds,5}s the longest silence allowed while reading a response"); - Console.WriteLine(" CancellationToken the whole operation, and the only one that bounds it"); - Console.WriteLine(); - Console.WriteLine(" A checkout that has to open a connection can therefore take up to PoolTimeout plus"); - Console.WriteLine(" DialTimeout: the two apply to different phases, so they add rather than overlap."); - Console.WriteLine(); - Console.WriteLine(" ReadTimeout = TimeSpan.Zero removes the deadline and leaves the caller's token as the"); - Console.WriteLine(" only bound. PoolTimeout and DialTimeout must be positive — there is no opting out of"); - Console.WriteLine(" those, because a wait with no bound at all is how a request hangs forever."); - } - - private static async Task DialingTheWrongThing() - { - Console.WriteLine("\n2. DialTimeout, and the two dial failures that are not timeouts\n"); - - ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); - - // Refused: the port answers at once with a reset, so the deadline is never involved. - var clock = Stopwatch.StartNew(); - Exception? refused = await Failing(options with { Port = 1, DialTimeout = TimeSpan.FromSeconds(2) }); - Console.WriteLine($" Nothing listening on the port, after {clock.ElapsedMilliseconds} ms:"); - Console.WriteLine($" {Describe(refused)}"); - Console.WriteLine($" inner: {refused?.InnerException?.GetType().Name} — a refusal is instant, so DialTimeout never came up"); - - // The HTTP port. Both interfaces are ClickHouse, but they speak different protocols, and the native client - // reads the HTTP server's reply as a protocol packet. - clock.Restart(); - Exception? wrongPort = await Failing(options with { Port = ExampleConfig.HttpEndpoint.Port, DialTimeout = TimeSpan.FromSeconds(2) }); - Console.WriteLine($"\n The HTTP port ({ExampleConfig.HttpEndpoint.Port}) instead of the native one, after {clock.ElapsedMilliseconds} ms:"); - Console.WriteLine($" {Describe(wrongPort)}"); - Console.WriteLine(" Packet type 72 is 'H', the first byte of the HTTP response. Not a timeout either."); - - // What DialTimeout is actually for: a peer that accepts the connection and then says nothing. A firewall, - // or a load balancer with no healthy backend behind it, looks exactly like this local listener. - using var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - int silentPort = ((IPEndPoint)listener.LocalEndpoint).Port; - Task accepted = AcceptOneAndSayNothing(listener); - - try - { - clock.Restart(); - Exception? silent = await Failing(options with - { - Host = "127.0.0.1", - Port = silentPort, - DialTimeout = TimeSpan.FromMilliseconds(300), - }); - - Console.WriteLine("\n A socket that accepts and never answers, with DialTimeout = 300 ms:"); - Console.WriteLine($" {Describe(silent)}"); - Console.WriteLine($" ... after {clock.ElapsedMilliseconds} ms. The connect succeeded; it is the handshake that never"); - Console.WriteLine(" finished. DialTimeout covers both, which is why an endpoint that answers the socket"); - Console.WriteLine(" and nothing else is bounded at all."); - } - finally - { - listener.Stop(); - (await accepted)?.Dispose(); - } - } - - /// Pings a server that is expected to be unreachable, and reports why it was. - private static async Task Failing(ClickHouseTcpClientOptions options) - { - try - { - await using var client = new ClickHouseTcpClient(options); - await client.PingAsync(); - return null; - } - catch (Exception ex) - { - return ex; - } - } - - private static string Describe(Exception? failure) - => failure is null ? "it answered, which is not what this example expected" : $"{failure.GetType().Name}: {failure.Message}"; - - private static async Task AcceptOneAndSayNothing(TcpListener listener) - { - try - { - return await listener.AcceptTcpClientAsync(); - } - catch (Exception ex) when (ex is SocketException or ObjectDisposedException) - { - // The listener was stopped first, which is the normal ending here. - return null; - } - } - - private static async Task ReadTimeoutMeasuresSilence() - { - Console.WriteLine("\n3. ReadTimeout is an idle deadline, not a time limit on the query\n"); - - ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); - - // sleepEachRow with max_block_size = 1 makes the server send one row at a time with a gap between them, - // which is what a slow-but-alive server looks like from here. - await using (var chatty = new ClickHouseTcpClient(options with { ReadTimeout = TimeSpan.FromMilliseconds(250) })) - { - var clock = Stopwatch.StartNew(); - int rows = 0; - await foreach (object[] row in chatty.QueryAsync( - "SELECT number, sleepEachRow(0.05) FROM numbers(8) SETTINGS max_block_size = 1")) - { - rows++; - } - - Console.WriteLine(" ReadTimeout = 250 ms, 8 rows arriving 50 ms apart:"); - Console.WriteLine($" read {rows} rows in {clock.ElapsedMilliseconds} ms — longer than the deadline, and it never fired."); - Console.WriteLine(" Every byte that arrives resets the clock, so total duration is not what it bounds."); - } - - await OneGapTooWide(options); - - // The other half of "silence": a slow consumer is not a silent server. - await using (var pausing = new ClickHouseTcpClient(options with { ReadTimeout = TimeSpan.FromMilliseconds(150) })) - { - var clock = Stopwatch.StartNew(); - int rows = 0; - await foreach (object[] row in pausing.QueryAsync("SELECT number FROM numbers(4) SETTINGS max_block_size = 1")) - { - rows++; - - // Holding each row for longer than the deadline before asking for the next one. - await Task.Delay(200); - } - - Console.WriteLine("\n ReadTimeout = 150 ms, and a consumer that sits on each row for 200 ms:"); - Console.WriteLine($" read {rows} rows in {clock.ElapsedMilliseconds} ms, no timeout. The clock runs only while"); - Console.WriteLine(" the client is waiting on the transport, so your own processing time is never on it."); - } - - // The opt-out, for a stream that is legitimately silent for a long time. - await using (var unbounded = new ClickHouseTcpClient(options with { ReadTimeout = TimeSpan.Zero })) - { - var clock = Stopwatch.StartNew(); - _ = await unbounded.ExecuteScalarAsync("SELECT sleepEachRow(0.4) FROM numbers(1)"); - Console.WriteLine($"\n ReadTimeout = TimeSpan.Zero, a 400 ms silence: completed in {clock.ElapsedMilliseconds} ms."); - Console.WriteLine(" With no deadline the caller's CancellationToken is the only bound left. Prefer a"); - Console.WriteLine(" generous ReadTimeout to none: what it catches is a connection dropped without a"); - Console.WriteLine(" FIN, which nothing else notices and TCP alone takes about fifteen minutes to give up on."); - } - } - - /// - /// One silence wider than the deadline, and what the pool then does with the connection. Its own method so - /// that the logger factory below is disposed — and its lines flushed to the console — before the next - /// section prints. - /// - private static async Task OneGapTooWide(ClickHouseTcpClientOptions options) - { - // The pool's own lines, at Trace, because the reuse line a healthy connection produces is a Trace line - // (Tcp_017 shows what one looks like). Its absence below is the evidence. - using ILoggerFactory poolLog = LoggerFactory.Create(builder => builder - .AddFilter((category, _) => category == "ClickHouse.Driver.Tcp.Pool") - .AddSimpleConsole(console => console.SingleLine = true) - .SetMinimumLevel(LogLevel.Trace)); - - await using var strict = new ClickHouseTcpClient(options with - { - ReadTimeout = TimeSpan.FromMilliseconds(150), - MaxPoolSize = 1, - LoggerFactory = poolLog, - }); - - Console.WriteLine("\n ReadTimeout = 150 ms, the same query with 500 ms between rows, and the pool's own lines:"); - - var clock = Stopwatch.StartNew(); - try - { - await foreach (object[] row in strict.QueryAsync( - "SELECT number, sleepEachRow(0.5) FROM numbers(3) SETTINGS max_block_size = 1")) - { - } - } - catch (TimeoutException ex) - { - Console.WriteLine($" TimeoutException after {clock.ElapsedMilliseconds} ms: {ex.Message}"); - } - - // A second query on the same client, whose pool holds exactly one connection. Had the timed-out - // connection gone back into the pool, this is the one that would have got it. - _ = await strict.ExecuteScalarAsync("SELECT 1"); - - Console.WriteLine(" The pool closed that connection instead of pooling it — 'no longer reusable' — and"); - Console.WriteLine(" the query after it opened another rather than reusing one. A socket that stopped"); - Console.WriteLine(" answering mid-response is of no use to the next caller."); - } - - private static void PoolTimeoutInOneLine() - { - Console.WriteLine("\n4. PoolTimeout\n"); - Console.WriteLine(" The third deadline belongs to the pool, so Tcp_017_PoolTuning demonstrates it: with"); - Console.WriteLine(" MaxPoolSize connections in use, the next operation waits PoolTimeout for a free one and"); - Console.WriteLine(" then throws TimeoutException. Two things hold a connection longer than a caller expects —"); - Console.WriteLine(" a session, for its whole lifetime, and a streamed result nobody finished reading."); - } - - private static async Task StatementMaxLengthCapsWhatIsLogged() - { - Console.WriteLine("\n5. StatementMaxLength, which caps the query text that leaves the client\n"); - Console.WriteLine(" It bounds two channels: the Debug log line below, and the db.query.text span attribute"); - Console.WriteLine(" that IncludeSqlInActivityTags turns on. The default is 5 — a stub, not a statement — so"); - Console.WriteLine(" recording query text is something you ask for.\n"); - - const string sql = "SELECT 'a statement long enough to show the cut'"; - - foreach (int max in new[] { 5, 60 }) - { - // Only the client category, so the pool and connection lines stay out of the way. A real application - // configures this through the container; see Tcp_003. - using ILoggerFactory factory = LoggerFactory.Create(builder => builder - .AddFilter((category, level) => category == "ClickHouse.Driver.Tcp.Client" && level >= LogLevel.Debug) - .AddSimpleConsole(console => console.SingleLine = true) - .SetMinimumLevel(LogLevel.Debug)); - - Console.WriteLine($" StatementMaxLength = {max}, and the client's own log lines that follow:"); - - await using (var client = new ClickHouseTcpClient(ExampleConfig.TcpBuilder().ToOptions() with - { - LoggerFactory = factory, - StatementMaxLength = max, - })) - { - _ = await client.ExecuteScalarAsync(sql); - } - - // The factory is disposed at the end of this iteration, which drains the console logger, so its lines - // land before the next heading prints. - } - - Console.WriteLine(); - Console.WriteLine($" The statement was {sql.Length} characters, so at 5 the log line carries a stub of it and"); - Console.WriteLine(" at 60 the whole thing. Zero or less keeps the text out even where the span attribute is on."); - } - - private static void MaxSendBufferBytesAndTheValues() - { - ClickHouseTcpClientOptions defaults = new(); - - Console.WriteLine("\n6. MaxSendBufferBytes, and what the constructor refuses\n"); - Console.WriteLine($" MaxSendBufferBytes defaults to {defaults.MaxSendBufferBytes / (1024 * 1024)} MiB. It is a soft cap on the client's send"); - Console.WriteLine(" buffer during an insert: while a wire block is written, buffered bytes are flushed to the"); - Console.WriteLine(" socket whenever they exceed it. Soft, because a single column larger than the cap still"); - Console.WriteLine(" buffers in full."); - Console.WriteLine(); - Console.WriteLine(" It is independent of MaxRowsPerBlock (Tcp_009), which decides how large a block is; this"); - Console.WriteLine(" decides how much of one is held in memory on the way out. Peak send-buffer memory is"); - Console.WriteLine($" about MaxSendBufferBytes × MaxPoolSize — {defaults.MaxSendBufferBytes / (1024 * 1024)} MiB × {defaults.MaxPoolSize} at the defaults — when every"); - Console.WriteLine(" connection is inserting at once. Nothing reports how much is buffered, so this one is a"); - Console.WriteLine(" sizing decision rather than something to watch."); - Console.WriteLine(); - Console.WriteLine(" Every value here is checked when the client is constructed, not on first use:\n"); - - ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); - - Refused("MaxSendBufferBytes = 0", options with { MaxSendBufferBytes = 0 }); - Refused("ReadTimeout = -1s", options with { ReadTimeout = TimeSpan.FromSeconds(-1) }); - Refused("PoolTimeout = TimeSpan.Zero", options with { PoolTimeout = TimeSpan.Zero }); - Refused("DialTimeout = 30 days", options with { DialTimeout = TimeSpan.FromDays(30) }); - } - - private static void Refused(string what, ClickHouseTcpClientOptions options) - { - try - { - // Never reaches a socket, so a synchronous Dispose is all this needs. - using var client = new ClickHouseTcpClient(options); - Console.WriteLine($" {what} -> accepted, which is not what this example expected"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" {what,-28} -> {ex.GetType().Name}: {ex.Message.Split(" (Parameter")[0]}"); - } - } -} diff --git a/examples/Tcp/Core/Tcp_001_BasicUsage.cs b/examples/Tcp/Core/Tcp_001_BasicUsage.cs index fb568f604..cedc16ecb 100644 --- a/examples/Tcp/Core/Tcp_001_BasicUsage.cs +++ b/examples/Tcp/Core/Tcp_001_BasicUsage.cs @@ -2,124 +2,58 @@ namespace ClickHouse.Driver.Examples; -/// -/// The native-protocol client from end to end: construct a , run DDL with -/// ExecuteAsync, insert rows with InsertRowsAsync, read them back with QueryAsync, read one -/// value with ExecuteScalarAsync, and dispose it. -/// -/// -/// This client speaks ClickHouse's own TCP protocol on port 9000, and it is not an ADO.NET provider. See -/// Tcp_004_MigratingFromHttp for how each HTTP-client call maps onto it, and for what it cannot do. -/// -/// +/// Connects to ClickHouse, creates a table, inserts rows, and reads them back. public static class TcpBasicUsage { - // Fixed, like every example_* table in this project: the suite runs one at a time against a server, so - // it does not need the unique names the test suites do. It is dropped even if a step throws. private const string TableName = "example_tcp_basic_usage"; public static async Task Run() { - // One client per endpoint, kept for the life of the application: it owns a connection pool, is safe to - // share across threads, and runs as many operations at once as the pool is wide. Building one per - // operation would pay for a connect and a handshake every time. - // - // 'await using', not 'using': the client is IAsyncDisposable, and disposal closes sockets. + // Reuse one client in your application. It is thread-safe and owns a connection pool. await using var client = ExampleConfig.CreateTcpClient(); - var endpoint = ExampleConfig.TcpEndpoint; - Console.WriteLine($"Native protocol endpoint: {endpoint.Host}:{endpoint.Port}, user '{ExampleConfig.TcpBuilder().Username}'"); + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + Console.WriteLine($"Connected to {server} through {ExampleConfig.TcpEndpoint}"); - // Read out of the handshake rather than from a query. A newly built client holds no connection, so - // the first call here dials and handshakes; later ones read what that handshake recorded. - var server = await client.GetServerInfoAsync(); - Console.WriteLine($"Server: {server} (protocol revision {server.ProtocolRevision}, timezone {server.Timezone})"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); try { - await CreateTable(client); - await InsertRows(client); - await ReadRows(client); - await ReadOneValue(client); - ShowTheReadTiers(); + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + id UInt64, + name String, + score Float64 + ) + ENGINE = MergeTree + ORDER BY id + """); + + var rows = new List + { + new object[] { 1UL, "Ada", 99.5 }, + new object[] { 2UL, "Grace", 97.25 }, + new object[] { 3UL, "Alan", 91.0 }, + }; + + // End the statement at VALUES. The rows are encoded as native columnar blocks. + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, name, score) VALUES", + rows); + + await foreach (object[] row in client.QueryAsync( + $"SELECT id, name, score FROM {TableName} ORDER BY id")) + { + Console.WriteLine($"{row[0]}: {row[1]} ({row[2]})"); + } + + object count = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + Console.WriteLine($"Row count: {count}"); } finally { await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped '{TableName}'. Disposing the client closes its pooled connections."); } } - - private static async Task CreateTable(ClickHouseTcpClient client) - { - // ExecuteAsync is for anything that returns no rows: DDL, and DML other than INSERT ... VALUES. - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - id UInt64, - name String, - score Float64 - ) - ENGINE = MergeTree() - ORDER BY id"); - - Console.WriteLine($"\nCreated '{TableName}' (id UInt64, name String, score Float64)"); - } - - private static async Task InsertRows(ClickHouseTcpClient client) - { - // The statement ends at VALUES: the rows travel after it as native blocks, never as SQL text. Each - // object[] is matched to the column list by position. - // - // A column takes the CLR type of its first non-null value, so keep one type per column: ulong for - // UInt64, string for String, double for Float64. - var rows = new List - { - new object[] { 1UL, "Ada", 99.5 }, - new object[] { 2UL, "Grace", 97.25 }, - new object[] { 3UL, "Alan", 91.0 }, - }; - - await client.InsertRowsAsync($"INSERT INTO {TableName} (id, name, score) VALUES", rows); - - Console.WriteLine($"Inserted {rows.Count} rows with InsertRowsAsync"); - } - - private static async Task ReadRows(ClickHouseTcpClient client) - { - Console.WriteLine("\nQueryAsync yields one object[] per row, values in the order the SELECT names them:"); - Console.WriteLine(" ID Name Score"); - Console.WriteLine(" -- ----- -----"); - - // Rows arrive as they are read off the connection rather than after the whole result is buffered. Each - // object[] is yours to keep; the enumeration holds a connection until it ends, so read it to the end. - await foreach (object[] row in client.QueryAsync($"SELECT id, name, score FROM {TableName} ORDER BY id")) - { - Console.WriteLine($" {(ulong)row[0],2} {(string)row[1],-5} {(double)row[2],5}"); - } - } - - private static async Task ReadOneValue(ClickHouseTcpClient client) - { - // ExecuteScalarAsync returns the first column of the first row, boxed. It reads the whole result before - // returning, so write a query that produces one row. - object count = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); - - // count() is UInt64, so the box holds a ulong: a value's CLR type follows the column's ClickHouse type. - Console.WriteLine($"\nExecuteScalarAsync(\"SELECT count() ...\") = {count} (boxed {count.GetType().Name})"); - } - - private static void ShowTheReadTiers() - { - Console.WriteLine("\nThree read tiers, all on this client:"); - Console.WriteLine(" QueryAsync one object[] per row, value-type columns boxed"); - Console.WriteLine(" QueryAsync one POCO per row, filled by column name"); - Console.WriteLine(" StreamAsync whole Blocks, typed columns, no per-row boxing"); - Console.WriteLine(); - Console.WriteLine("The row tier boxes the value the wire carried, which is not always the CLR type the column"); - Console.WriteLine("name suggests: a DateTime column arrives as uint epoch seconds. For a calendar value, read"); - Console.WriteLine("it through QueryAsync into a DateTime or DateTimeOffset property, or on the block tier"); - Console.WriteLine("match the column to IDateTimeColumn (ITimeColumn for Time) and call GetDateTimeOffset."); - } } diff --git a/examples/Tcp/Core/Tcp_002_ConnectionString.cs b/examples/Tcp/Core/Tcp_002_ConnectionString.cs index 5d93714ea..8d36e618c 100644 --- a/examples/Tcp/Core/Tcp_002_ConnectionString.cs +++ b/examples/Tcp/Core/Tcp_002_ConnectionString.cs @@ -1,169 +1,34 @@ -using ClickHouse.Driver.Compression; using ClickHouse.Driver.Tcp; namespace ClickHouse.Driver.Examples; -/// -/// Configuring the native-protocol client: what a native connection string holds, how -/// builds one, and how it becomes a -/// — the record every client is built from. -/// -/// -/// The key set is not the HTTP one. There is no Protocol key, Compression names a codec instead of -/// switching a boolean, and the pool and TLS keys have no HTTP counterpart at all. -/// -/// -/// -/// Configuration is this example's subject, so the connection strings in its output are literals. The client it -/// connects with still comes from ExampleConfig. -/// -/// +/// Builds native-protocol connection options from a connection string. public static class TcpConnectionString { public static async Task Run() { - ShowTheKeys(); - ClickHouseTcpClientOptions options = BuildOptions(); - DeriveAVariant(options); - ShowTlsKeys(); - await ConnectWithThem(options); - } - - private static void ShowTheKeys() - { - Console.WriteLine("1. A native-protocol connection string:\n"); - Console.WriteLine(" Host=localhost;Port=9000;Username=default;Password=secret;Database=default"); - Console.WriteLine(); - Console.WriteLine(" Every key is optional: Host defaults to localhost, Username to default, Database to"); - Console.WriteLine(" default, Password to empty. Port is the one to watch — the native protocol listens on"); - Console.WriteLine(" 9000, not on the HTTP interface's 8123."); - Console.WriteLine(); - Console.WriteLine(" There is no Protocol key. UseTls=true selects TLS, and an unset Port then resolves to"); - Console.WriteLine(" 9440, the secure native port, instead of 9000."); - - Console.WriteLine("\n2. The keys that are not in the HTTP set:\n"); - Console.WriteLine(" Compression=lz4|zstd|none A codec name, where the HTTP client's Compression is a"); - Console.WriteLine(" boolean. lz4 is the default, so wire blocks are"); - Console.WriteLine(" compressed in both directions unless this says none."); - Console.WriteLine(" Pool MinPoolSize, MaxPoolSize, PoolTimeout, IdleTimeout,"); - Console.WriteLine(" MaxConnectionLifetime, SweepInterval, PoolReusePolicy"); - Console.WriteLine(" TLS UseTls, TlsServerName, TlsCaCertificatePath,"); - Console.WriteLine(" TlsAllowInvalidCertificates"); - Console.WriteLine(" Deadlines DialTimeout, ReadTimeout (both in seconds)"); - Console.WriteLine(" Other QuotaKey, MaxSendBufferBytes, and set_="); - Console.WriteLine(" for a ClickHouse setting sent with every operation"); - } - - private static ClickHouseTcpClientOptions BuildOptions() - { - // Every key the builder knows has a typed property, so a name is checked at compile time rather than - // kept as an unknown key and ignored. An unreadable UseTls, TLS-authority or PoolReusePolicy value throws; - // an unreadable number falls back to its default. + // Use a builder when configuration starts as a connection string but needs code-level changes. var builder = ExampleConfig.TcpBuilder(); builder.Compression = "zstd"; builder.MaxPoolSize = 4; - builder.IdleTimeout = TimeSpan.FromSeconds(60); + builder.IdleTimeout = TimeSpan.FromMinutes(1); - // Custom settings have no typed property: any set_ key becomes a client-level ClickHouse setting. + // Prefix server settings with set_ when they should apply to every query. builder["set_max_threads"] = 2; - Console.WriteLine("\n3. ClickHouseTcpConnectionStringBuilder:\n"); - Console.WriteLine($" Host {builder.Host}"); - Console.WriteLine($" Port {builder.Port?.ToString() ?? "(unset: resolved from UseTls)"}"); - Console.WriteLine($" Username {builder.Username}"); - Console.WriteLine($" Password {(builder.Password.Length == 0 ? "(empty)" : "(set — not printed)")}"); - Console.WriteLine($" Database {builder.Database}"); - Console.WriteLine($" Compression {builder.Compression}"); - Console.WriteLine($" MaxPoolSize {builder.MaxPoolSize}"); - Console.WriteLine($" IdleTimeout {builder.IdleTimeout.TotalSeconds}s"); - Console.WriteLine($" UseTls {builder.UseTls}"); - Console.WriteLine(); - Console.WriteLine(" builder.ToString() would render all of that back as a connection string, password"); - Console.WriteLine(" included, so it is not something to log."); - - // ToOptions() materializes the keys; FromConnectionString(text) is the same thing in one call for a string - // that came from configuration. ClickHouseTcpClientOptions options = builder.ToOptions(); - ClickHouseTcpClientOptions fromText = ClickHouseTcpClientOptions.FromConnectionString(ExampleConfig.TcpConnectionString); - - Console.WriteLine("\n4. ClickHouseTcpClientOptions — what a client is really built from:\n"); - - // The record's generated ToString would print the password; this override names only the safe properties, - // so options are safe to log. The port it shows is the resolved one. - Console.WriteLine($" builder.ToOptions() {options}"); - Console.WriteLine($" Compressor {Describe(options.Compressor)}"); - Console.WriteLine($" MaxPoolSize {options.MaxPoolSize}"); - Console.WriteLine($" IdleTimeout {options.IdleTimeout}"); - Console.WriteLine($" CustomSettings {string.Join(", ", options.CustomSettings.Select(s => $"{s.Key}={s.Value}"))}"); - Console.WriteLine(); - Console.WriteLine($" FromConnectionString(ExampleConfig.TcpConnectionString) {fromText}"); - Console.WriteLine($" Compressor {Describe(fromText.Compressor)}"); - Console.WriteLine(" That string carries no Compression key, so the lz4 default stands. Only 'none'"); - Console.WriteLine(" leaves the codec null, and null means the query asks for no compression at all."); - - return options; - } - - private static string Describe(IClickHouseCompressor compressor) - => compressor?.GetType().Name ?? "(none)"; + Console.WriteLine($"Endpoint: {options.Host}:{options.Port ?? 9000}/{options.Database}"); + Console.WriteLine($"Compression: {options.Compressor?.GetType().Name ?? "none"}"); + Console.WriteLine($"Max pool size: {options.MaxPoolSize}"); - private static void DeriveAVariant(ClickHouseTcpClientOptions options) - { - // Options are an init-only record, so one instance can hold what every client shares and a 'with' - // expression derives the variant. The original is untouched. - ClickHouseTcpClientOptions wide = options with { MaxPoolSize = 32, Database = "system" }; - - Console.WriteLine("\n5. Options is a record, so 'with' derives a variant:\n"); - Console.WriteLine($" options with {{ MaxPoolSize = 32, Database = \"system\" }}"); - Console.WriteLine($" original MaxPoolSize={options.MaxPoolSize}, Database={options.Database}"); - Console.WriteLine($" variant MaxPoolSize={wide.MaxPoolSize}, Database={wide.Database}"); - } - - private static void ShowTlsKeys() - { - Console.WriteLine("\n6. The TLS keys, and how they are checked:\n"); - Console.WriteLine(" UseTls=true encrypt the transport, and dial 9440 unless Port says"); - Console.WriteLine(" otherwise. The handshake carries the password in the"); - Console.WriteLine(" clear, so this is what protects it."); - Console.WriteLine(" TlsServerName=host the name to match the certificate against, when Host"); - Console.WriteLine(" is an address or an internal alias"); - Console.WriteLine(" TlsCaCertificatePath=ca.pem validate against these authorities instead of the"); - Console.WriteLine(" host trust store"); - Console.WriteLine(" TlsAllowInvalidCertificates=true accept any certificate — development only"); - - // A TLS key with UseTls left false is refused at construction. Silently ignoring it is how a connection - // meant to be encrypted ends up in the clear. - try - { - _ = new ClickHouseTcpClient(new ClickHouseTcpClientOptions - { - Host = ExampleConfig.TcpEndpoint.Host, - TlsAllowInvalidCertificates = true, - }); - } - catch (ArgumentException ex) - { - Console.WriteLine(); - Console.WriteLine(" A TLS key set while UseTls is false is rejected, not ignored:"); - Console.WriteLine($" {ex.Message}"); - } - } + // Options are immutable records. Use with to derive a configuration variant. + ClickHouseTcpClientOptions systemOptions = options with { Database = "system" }; - private static async Task ConnectWithThem(ClickHouseTcpClientOptions options) - { - Console.WriteLine("\n7. Running with those options:\n"); - - await using var client = new ClickHouseTcpClient(options); - - var server = await client.GetServerInfoAsync(); - - // Compressor names the codec this client writes its own blocks with. It does not choose the codec - // for the blocks coming back: the query packet carries a compression flag and no codec name, so - // the server picks that one itself. - Console.WriteLine($" Connected to {server}, blocks this client writes framed with {Describe(options.Compressor)}"); - - // set_max_threads became a client-level setting, so the server sees it on every operation. + await using var client = new ClickHouseTcpClient(systemOptions); + object database = await client.ExecuteScalarAsync("SELECT currentDatabase()"); object maxThreads = await client.ExecuteScalarAsync("SELECT getSetting('max_threads')"); - Console.WriteLine($" getSetting('max_threads') = {maxThreads} — the set_max_threads key reached the server"); + + Console.WriteLine($"Connected database: {database}"); + Console.WriteLine($"max_threads: {maxThreads}"); } } diff --git a/examples/Tcp/Core/Tcp_003_DependencyInjection.cs b/examples/Tcp/Core/Tcp_003_DependencyInjection.cs index 74d16e5ef..59c415984 100644 --- a/examples/Tcp/Core/Tcp_003_DependencyInjection.cs +++ b/examples/Tcp/Core/Tcp_003_DependencyInjection.cs @@ -1,160 +1,62 @@ using ClickHouse.Driver.Tcp; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; namespace ClickHouse.Driver.Examples; -/// -/// Registering the native-protocol client in an with -/// AddClickHouseTcpDataSource: the connection-string and options overloads, injecting -/// into a consumer, the keyed overload for two clusters, and the one rule that -/// matters — the pool is a singleton, so nothing injected may dispose it. -/// +/// Registers the native client with Microsoft dependency injection. public static class TcpDependencyInjection { public static async Task Run() { - Console.WriteLine("One call registers three services, all singletons:\n"); - Console.WriteLine(" ClickHouseTcpDataSource owns the client and its connection pool; the container disposes it"); - Console.WriteLine(" IClickHouseTcpClient the client that data source owns — queries, inserts, sessions"); - Console.WriteLine(" IClickHouseTcpOperations the same object again, for code that only runs operations"); - - await FromConnectionString(); - await FromOptions(); - await TwoClusters(); - await WhoDisposesWhat(); + await RegisterOneClient(); + await RegisterMultipleClients(); } - private static async Task FromConnectionString() + private static async Task RegisterOneClient() { - Console.WriteLine("\n1. From a connection string:\n"); - var services = new ServiceCollection(); services.AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString); - - // A consumer takes the interface. Registered through a factory here only because this example's consumer - // is a private nested type; a normal AddSingleton() reaches the same client, and a keyed one - // is reached with [FromKeyedServices("key")] on the constructor parameter. - services.AddSingleton(sp => new ServerProbe(sp.GetRequiredService())); + services.AddSingleton(); await using ServiceProvider provider = services.BuildServiceProvider(); var probe = provider.GetRequiredService(); - Console.WriteLine($" ServerProbe (injected IClickHouseTcpClient): {await probe.DescribeAsync()}"); + Console.WriteLine(await probe.GetVersionAsync()); - // Every registration resolves the one client the data source owns, so there is one pool per registration - // however many consumers there are. var dataSource = provider.GetRequiredService(); var client = provider.GetRequiredService(); - var operations = provider.GetRequiredService(); - - Console.WriteLine($" IClickHouseTcpClient is dataSource.GetClient(): {ReferenceEquals(client, dataSource.GetClient())}"); - Console.WriteLine($" IClickHouseTcpOperations is the same object: {ReferenceEquals(client, operations)}"); - } - - private static async Task FromOptions() - { - Console.WriteLine("\n2. From options, with the container's logging:\n"); - - var services = new ServiceCollection(); - services.AddLogging(logging => logging.AddConsole().SetMinimumLevel(LogLevel.Warning)); - - // Options are a record, so the shape that differs from the connection string is a 'with' away. - ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions() with - { - MaxPoolSize = 4, - IdleTimeout = TimeSpan.FromSeconds(30), - }; - - services.AddClickHouseTcpDataSource(options); - - await using ServiceProvider provider = services.BuildServiceProvider(); - - var dataSource = provider.GetRequiredService(); - Console.WriteLine($" {dataSource.Options}"); - Console.WriteLine($" MaxPoolSize {dataSource.Options.MaxPoolSize}, IdleTimeout {dataSource.Options.IdleTimeout}"); - - // The registration fills in a null LoggerFactory from the container, on a copy: the options object passed - // in is left alone, so the two are no longer the same instance. - Console.WriteLine($" LoggerFactory on the options passed in: {options.LoggerFactory?.GetType().Name ?? "(null)"}"); - Console.WriteLine($" LoggerFactory the data source runs with: {dataSource.Options.LoggerFactory?.GetType().Name ?? "(null)"}"); + Console.WriteLine($"The data source owns the injected client: " + + $"{ReferenceEquals(client, dataSource.GetClient())}"); - // There is also an overload taking Func, for options that - // need something else out of the container, and one taking a Func that builds the data source itself. - object value = await provider.GetRequiredService().ExecuteScalarAsync("SELECT 'registered from options'"); - Console.WriteLine($" SELECT returned: {value}"); + // The service provider owns the data source and its pool. Do not dispose injected clients. } - private static async Task TwoClusters() + private static async Task RegisterMultipleClients() { - Console.WriteLine("\n3. Two clusters, told apart by service key:\n"); - var services = new ServiceCollection(); - - // A second unkeyed call would be a no-op: every service is added with TryAdd, so the first registration - // of a service and key wins. A key is what makes the second registration a different service. - services.AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString, serviceKey: "ingest"); + services.AddClickHouseTcpDataSource( + ExampleConfig.TcpConnectionString, + serviceKey: "ingest"); services.AddClickHouseTcpDataSource( ExampleConfig.TcpBuilder().ToOptions() with { MaxPoolSize = 2 }, serviceKey: "reporting"); - // Both point at this example's one server; in a real application they would be different endpoints. - services.AddSingleton(sp => new ServerProbe(sp.GetRequiredKeyedService("reporting"))); - await using ServiceProvider provider = services.BuildServiceProvider(); var ingest = provider.GetRequiredKeyedService("ingest"); var reporting = provider.GetRequiredKeyedService("reporting"); - Console.WriteLine($" 'ingest' MaxPoolSize {ingest.Options.MaxPoolSize}, pool of its own: {!ReferenceEquals(ingest, reporting)}"); - Console.WriteLine($" 'reporting' MaxPoolSize {reporting.Options.MaxPoolSize}"); - Console.WriteLine($" ServerProbe holding the 'reporting' client: {await provider.GetRequiredService().DescribeAsync()}"); - - // Keyed registrations are not also unkeyed, so plain injection finds nothing. Key every consumer, or - // register one of the endpoints without a key as well. - Console.WriteLine($" An unkeyed IClickHouseTcpClient is registered: {provider.GetService() is not null}"); - } - - private static async Task WhoDisposesWhat() - { - Console.WriteLine("\n4. Who disposes what:\n"); - Console.WriteLine(" The data source owns the pool and the container owns the data source, so the pool"); - Console.WriteLine(" closes once, at shutdown, when the provider is disposed. Prefer DisposeAsync where the"); - Console.WriteLine(" call site can await it, as a generic host does."); - Console.WriteLine(); - Console.WriteLine(" Never dispose an injected client. It offers DisposeAsync because a session needs one,"); - Console.WriteLine(" but disposing it closes the shared pool and every other consumer's next operation"); - Console.WriteLine(" fails. A session from OpenSessionAsync is the opposite: it is yours to dispose."); - - ServiceProvider provider = new ServiceCollection() - .AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString) - .BuildServiceProvider(); - - var client = provider.GetRequiredService(); - await client.PingAsync(); - Console.WriteLine("\n Ping before shutdown: answered"); - - await provider.DisposeAsync(); - - // What a consumer that disposed the client would leave behind for everyone else. - try - { - await client.PingAsync(); - } - catch (ObjectDisposedException) - { - Console.WriteLine(" Ping after the provider was disposed: ObjectDisposedException, as it should be"); - } + await ingest.PingAsync(); + await reporting.PingAsync(); + Console.WriteLine("Both keyed clients connected successfully."); } - // Takes IClickHouseTcpClient rather than the concrete client, so a test can substitute a double. Holds no - // disposal logic: the container owns the client's lifetime. private sealed class ServerProbe(IClickHouseTcpClient client) { - public async Task DescribeAsync() + public async Task GetVersionAsync() { - ClickHouseTcpServerInfo info = await client.GetServerInfoAsync(); - return $"{info}, protocol revision {info.ProtocolRevision}"; + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + return $"ClickHouse {server.Version}"; } } } diff --git a/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs b/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs index a86668fdd..5d119102d 100644 --- a/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs +++ b/examples/Tcp/Core/Tcp_004_MigratingFromHttp.cs @@ -2,194 +2,63 @@ namespace ClickHouse.Driver.Examples; -/// -/// Moving code from the HTTP client to the native-protocol one: the experimental opt-in a consumer has to make, -/// the same task written both ways against one server, the call-for-call mapping, and an honest list of what the -/// native client does not do. -/// -/// -/// The list matters more than the mapping. Most HTTP calls have a native counterpart, but a few capabilities have -/// none at all, and they are the ones that decide whether a migration is possible. -/// -/// +/// Shows the native-protocol equivalents of common HTTP client operations. public static class TcpMigratingFromHttp { private const string TableName = "example_tcp_migrating_from_http"; public static async Task Run() { - ShowTheOptIn(); - - // Two clients, one server: the HTTP interface on 8123 and the native protocol on 9000. using var http = ExampleConfig.CreateHttpClient(); await using var tcp = ExampleConfig.CreateTcpClient(); - await SameTaskBothWays(http, tcp); - - ShowTheMapping(); - ShowWhatIsMissing(); - } - - private static void ShowTheOptIn() - { - Console.WriteLine("1. The experimental opt-in\n"); - Console.WriteLine(" ClickHouseTcpClient, ClickHouseTcpDataSource, the three IClickHouseTcp* interfaces and"); - Console.WriteLine(" AddClickHouseTcpDataSource carry [Experimental(\"CHTCP0001\")], so naming any of them is a"); - Console.WriteLine(" compile error until you acknowledge that the surface may still change."); - Console.WriteLine(); - Console.WriteLine(" The types around them — ClickHouseTcpClientOptions, the connection-string builder, Block,"); - Console.WriteLine(" the columns, the exceptions — do not carry it, so holding one raises no diagnostic even"); - Console.WriteLine(" though it is just as experimental."); - Console.WriteLine(); - Console.WriteLine(" Per file:"); - Console.WriteLine(" #pragma warning disable CHTCP0001 // The native protocol client's API is not yet stable."); - Console.WriteLine(); - Console.WriteLine(" Or once for a project:"); - Console.WriteLine(" $(NoWarn);CHTCP0001"); - Console.WriteLine(); - Console.WriteLine(" This examples project takes the project-wide route, which is why no file under Tcp/"); - Console.WriteLine(" opens with the pragma."); - } - - private static async Task SameTaskBothWays(ClickHouseClient http, ClickHouseTcpClient tcp) - { - Console.WriteLine("\n2. The same task, both ways\n"); + await http.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {TableName}"); try { - await CompareTransports(http, tcp); - } - finally - { - await tcp.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\n Dropped '{TableName}'"); - } - } - - private static async Task CompareTransports(ClickHouseClient http, ClickHouseTcpClient tcp) - { - // DDL. HTTP: ExecuteNonQueryAsync, which returns the affected-row count ADO.NET expects. - await http.ExecuteNonQueryAsync($"DROP TABLE IF EXISTS {TableName}"); - await http.ExecuteNonQueryAsync($@" - CREATE TABLE {TableName} (id UInt64, name String, source String) - ENGINE = MergeTree() ORDER BY id"); - Step("http.ExecuteNonQueryAsync(\"CREATE TABLE ...\")", "table created"); - - // The native equivalent returns nothing: there is no row count on this path, only acknowledgement. - await tcp.ExecuteAsync($"ALTER TABLE {TableName} MODIFY COMMENT 'written over both transports'"); - Step("tcp.ExecuteAsync(\"ALTER TABLE ...\")", "comment set"); - - // Insert. HTTP names the table and the columns as arguments. - await http.InsertBinaryAsync( - TableName, - new[] { "id", "name", "source" }, - new List + await http.ExecuteNonQueryAsync($""" + CREATE TABLE {TableName} (id UInt64, name String, source String) + ENGINE = MergeTree + ORDER BY id + """); + + await http.InsertBinaryAsync( + TableName, + new[] { "id", "name", "source" }, + new[] { new object[] { 1UL, "Ada", "HTTP" } }); + + await tcp.InsertRowsAsync( + $"INSERT INTO {TableName} (id, name, source) VALUES", + new[] { new object[] { 2UL, "Grace", "TCP" } }); + + Console.WriteLine("HTTP: ExecuteReaderAsync returns a DbDataReader"); + using (var reader = await http.ExecuteReaderAsync( + $"SELECT id, name, source FROM {TableName} ORDER BY id")) { - new object[] { 1UL, "Ada", "http" }, - new object[] { 2UL, "Grace", "http" }, - }); - Step("http.InsertBinaryAsync(table, columns, rows)", "2 rows"); - - // The native client takes the statement instead, ending at VALUES, and the rows follow it as blocks. - await tcp.InsertRowsAsync( - $"INSERT INTO {TableName} (id, name, source) VALUES", - new List - { - new object[] { 3UL, "Alan", "tcp" }, - new object[] { 4UL, "Edsger", "tcp" }, - }); - Step("tcp.InsertRowsAsync(\"INSERT ... VALUES\", rows)", "2 rows"); - - Console.WriteLine(); - Console.WriteLine(" Reading the same four rows through each client:\n"); - Console.WriteLine(" ID Name Source read by"); - Console.WriteLine(" -- ------ ------ -------"); + while (reader.Read()) + { + Console.WriteLine($" {reader.GetFieldValue(0)}: " + + $"{reader.GetString(1)} ({reader.GetString(2)})"); + } + } - // HTTP reads through a DbDataReader, pulled row by row. - using (var reader = await http.ExecuteReaderAsync($"SELECT id, name, source FROM {TableName} ORDER BY id")) - { - while (reader.Read()) + Console.WriteLine("TCP: QueryAsync streams object[] rows"); + await foreach (object[] row in tcp.QueryAsync( + $"SELECT id, name, source FROM {TableName} ORDER BY id")) { - Console.WriteLine($" {reader.GetFieldValue(0),2} {reader.GetString(1),-6} {reader.GetString(2),-6} ExecuteReaderAsync"); + Console.WriteLine($" {row[0]}: {row[1]} ({row[2]})"); } - } - // The native client streams object[] rows instead. There is no DbDataReader on this transport. - await foreach (object[] row in tcp.QueryAsync($"SELECT id, name, source FROM {TableName} ORDER BY id")) + object httpCount = await http.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + object tcpCount = await tcp.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); + Console.WriteLine($"HTTP count: {httpCount}; TCP count: {tcpCount}"); + } + finally { - Console.WriteLine($" {(ulong)row[0],2} {(string)row[1],-6} {(string)row[2],-6} QueryAsync"); + await tcp.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); } - // One scalar call, spelled the same on both, and the same boxed CLR type comes back. - object httpCount = await http.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); - object tcpCount = await tcp.ExecuteScalarAsync($"SELECT count() FROM {TableName}"); - Console.WriteLine(); - Console.WriteLine($" http.ExecuteScalarAsync(\"SELECT count() ...\") = {httpCount} ({httpCount.GetType().Name})"); - Console.WriteLine($" tcp.ExecuteScalarAsync(\"SELECT count() ...\") = {tcpCount} ({tcpCount.GetType().Name})"); - Console.WriteLine($" The two transports agree: {httpCount.Equals(tcpCount)}"); + Console.WriteLine("Use the HTTP client for ADO.NET, ORMs, and non-Native formats."); + Console.WriteLine("Use the TCP client for blocks, sessions, and live progress callbacks."); } - - private static void ShowTheMapping() - { - Console.WriteLine("\n3. Call for call\n"); - - Map("ClickHouseClient", "ClickHouseTcpClient"); - Map("ClickHouseClientSettings", "ClickHouseTcpClientOptions (an init-only record)"); - Map("ClickHouseConnectionStringBuilder", "ClickHouseTcpConnectionStringBuilder"); - Map("ExecuteNonQueryAsync(sql) -> int", "ExecuteAsync(sql)"); - Map("ExecuteScalarAsync(sql)", "ExecuteScalarAsync(sql)"); - Map("ExecuteReaderAsync(sql) -> DbDataReader", "QueryAsync(sql) -> IAsyncEnumerable"); - Map(string.Empty, "QueryAsync(sql) -> IAsyncEnumerable"); - Map(string.Empty, "StreamAsync(sql) -> IAsyncEnumerable"); - Map("InsertBinaryAsync(table, columns, rows)", "InsertRowsAsync(\"INSERT INTO t (cols) VALUES\", rows)"); - Map("InsertBinaryAsync(table, rows)", "InsertRowsAsync(\"INSERT INTO t (cols) VALUES\", rows)"); - Map(string.Empty, "InsertAsync(sql, IColumn[]) — columnar, no per-row boxing"); - Map("PingAsync()", "PingAsync() — a protocol ping, not a SELECT 1"); - Map("QueryOptions", "ClickHouseTcpQueryOptions / ClickHouseTcpInsertOptions"); - Map("QueryOptions.CustomSettings (object values)", "Settings (string values)"); - Map("ClickHouseParameterCollection", "ClickHouseTcpParameterCollection"); - Map("@name, rewritten client-side", "{name:Type} only — nothing is rewritten"); - Map("UseSession / SessionId", "OpenSessionAsync() -> IClickHouseTcpSession"); - Map("AddClickHouseDataSource(...)", "AddClickHouseTcpDataSource(...)"); - Map("using (IDisposable)", "await using (IAsyncDisposable, and IDisposable)"); - Map("Port=8123, Protocol=https", "Port=9000, UseTls=true (9440 when Port is unset)"); - Map("Compression=true", "Compression=lz4|zstd|none"); - Map("ClickHouseConnection / ClickHouseCommand", "(nothing — see below)"); - } - - private static void ShowWhatIsMissing() - { - Console.WriteLine("\n4. What the native client does not do\n"); - Console.WriteLine(" A format other than Native. The protocol carries columnar blocks, so there is no CSV,"); - Console.WriteLine(" JSONEachRow or Parquet ingestion or export, and no raw stream insert."); - Console.WriteLine(); - Console.WriteLine(" ADO.NET, and so any ORM. There is no DbConnection over this transport, so Dapper, EF"); - Console.WriteLine(" Core and linq2db need the HTTP client."); - Console.WriteLine(); - Console.WriteLine(" JWT or bearer authentication. Username and password only, plus QuotaKey."); - Console.WriteLine(); - Console.WriteLine(" Custom HTTP headers, which have no equivalent on the wire."); - Console.WriteLine(); - Console.WriteLine(" A parameter type resolver, a parameter formatter, or a read value converter. There is no"); - Console.WriteLine(" hook for any of the three: a parameter's type comes from its {name:Type} placeholder or"); - Console.WriteLine(" from ClickHouseTcpParameter.ClickHouseType."); - Console.WriteLine(); - Console.WriteLine(" Per-query Roles or Database. Run SET ROLE inside a session for the first; qualify the"); - Console.WriteLine(" name, or use a client per database, for the second."); - - Console.WriteLine("\n5. What only the native client does\n"); - Console.WriteLine(" Blocks and typed columns, so a read can skip materializing rows at all, and a column"); - Console.WriteLine(" read out of one block re-inserts without being rebuilt."); - Console.WriteLine(" Sessions that are one pinned connection, so a temporary table or a SET survives."); - Console.WriteLine(" Progress, profile info and profile events while the query runs, through callbacks."); - Console.WriteLine(" Block compression on the wire, LZ4 by default."); - Console.WriteLine(); - Console.WriteLine(" Both lists in full: examples/Tcp/README.md"); - } - - private static void Step(string call, string result) - => Console.WriteLine($" {call,-46} {result}"); - - private static void Map(string http, string tcp) - => Console.WriteLine($" {http,-44} {tcp}"); } diff --git a/examples/Tcp/Observability/Tcp_001_Logging.cs b/examples/Tcp/Observability/Tcp_001_Logging.cs new file mode 100644 index 000000000..7be4d47f0 --- /dev/null +++ b/examples/Tcp/Observability/Tcp_001_Logging.cs @@ -0,0 +1,39 @@ +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.Logging; + +namespace ClickHouse.Driver.Examples; + +/// Enables diagnostic logs for client, connection, and pool activity. +public static class TcpLogging +{ + public static async Task Run() + { + using ILoggerFactory loggerFactory = LoggerFactory.Create(logging => logging + .AddFilter((category, level) => category switch + { + ClickHouseTcpDiagnostics.ClientLogCategory => level >= LogLevel.Debug, + ClickHouseTcpDiagnostics.ConnectionLogCategory => level >= LogLevel.Debug, + ClickHouseTcpDiagnostics.PoolLogCategory => level >= LogLevel.Trace, + _ => false, + }) + .AddSimpleConsole(console => console.SingleLine = true) + .SetMinimumLevel(LogLevel.Trace)); + + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions() with + { + LoggerFactory = loggerFactory, + StatementMaxLength = 80, + }; + + await using var client = new ClickHouseTcpClient(options); + await client.PingAsync(); + object value = await client.ExecuteScalarAsync("SELECT 'logged query'"); + Console.WriteLine($"Query result: {value}"); + + Console.WriteLine($"Client category: {ClickHouseTcpDiagnostics.ClientLogCategory}"); + Console.WriteLine($"Connection category: {ClickHouseTcpDiagnostics.ConnectionLogCategory}"); + Console.WriteLine($"Pool category: {ClickHouseTcpDiagnostics.PoolLogCategory}"); + + // Debug logs can contain SQL. Set StatementMaxLength to 0 to omit statement text. + } +} diff --git a/examples/Tcp/Observability/Tcp_002_OpenTelemetry.cs b/examples/Tcp/Observability/Tcp_002_OpenTelemetry.cs new file mode 100644 index 000000000..a3ca70245 --- /dev/null +++ b/examples/Tcp/Observability/Tcp_002_OpenTelemetry.cs @@ -0,0 +1,64 @@ +using System.Diagnostics; +using ClickHouse.Driver.Tcp; +using OpenTelemetry; +using OpenTelemetry.Trace; + +namespace ClickHouse.Driver.Examples; + +/// Collects OpenTelemetry spans emitted by the native client. +public static class TcpOpenTelemetry +{ + public static async Task Run() + { + var exporter = new ActivityCollector(); + + // Subscribe to the driver's ActivitySource before creating the client. + using TracerProvider provider = Sdk.CreateTracerProviderBuilder() + .AddSource(ClickHouseTcpDiagnostics.ActivitySourceName) + .AddProcessor(new SimpleActivityExportProcessor(exporter)) + .Build()!; + + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions() with + { + IncludeSqlInActivityTags = true, + StatementMaxLength = 100, + }; + + await using (var client = new ClickHouseTcpClient(options)) + { + await client.PingAsync(); + await client.ExecuteScalarAsync("SELECT count() FROM numbers(1000)"); + } + + foreach (Activity activity in exporter.Activities) + { + Console.WriteLine( + $"{activity.OperationName}: {activity.Status}, " + + $"{activity.Duration.TotalMilliseconds:0.0} ms"); + + foreach (KeyValuePair tag in activity.TagObjects) + { + Console.WriteLine($" {tag.Key} = {tag.Value}"); + } + } + + // SQL can contain sensitive data. IncludeSqlInActivityTags is disabled by default. + } + + private sealed class ActivityCollector : BaseExporter + { + private readonly List activities = []; + + public IReadOnlyList Activities => activities; + + public override ExportResult Export(in Batch batch) + { + foreach (Activity activity in batch) + { + activities.Add(activity); + } + + return ExportResult.Success; + } + } +} diff --git a/examples/Tcp/Observability/Tcp_003_MetadataBlocks.cs b/examples/Tcp/Observability/Tcp_003_MetadataBlocks.cs new file mode 100644 index 000000000..b671fb06d --- /dev/null +++ b/examples/Tcp/Observability/Tcp_003_MetadataBlocks.cs @@ -0,0 +1,79 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Receives server logs, totals, and extremes through block callbacks. +public static class TcpMetadataBlocks +{ + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + + var serverLogs = new List(); + var totals = new Dictionary(); + var extremes = new List>(); + + var options = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary + { + ["send_logs_level"] = "debug", + ["extremes"] = "1", + }, + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnLog = block => + { + IColumn source = block.Column("source"); + IColumn text = block.Column("text"); + for (int row = 0; row < block.RowCount; row++) + { + serverLogs.Add($"{source[row]}: {text[row]}"); + } + }, + OnTotals = block => + { + for (int column = 0; column < block.ColumnCount; column++) + { + totals[block.ColumnNames[column]] = block[column].GetValue(0); + } + }, + OnExtremes = block => + { + for (int row = 0; row < block.RowCount; row++) + { + var values = new Dictionary(); + for (int column = 0; column < block.ColumnCount; column++) + { + values[block.ColumnNames[column]] = block[column].GetValue(row); + } + + extremes.Add(values); + } + }, + }, + }; + + await foreach (object[] row in client.QueryAsync( + """ + SELECT number % 3 AS bucket, count() AS rows + FROM numbers(30) + GROUP BY bucket WITH TOTALS + ORDER BY bucket + """, + options)) + { + Console.WriteLine($"bucket={row[0]}, rows={row[1]}"); + } + + Console.WriteLine($"Totals: {Format(totals)}"); + Console.WriteLine($"Minimums: {Format(extremes[0])}"); + Console.WriteLine($"Maximums: {Format(extremes[1])}"); + Console.WriteLine($"Server log lines: {serverLogs.Count}"); + + // Callback blocks are borrowed. Copy values inside the callback, as above. + } + + private static string Format(IReadOnlyDictionary values) + => string.Join(", ", values.Select(item => $"{item.Key}={item.Value ?? "NULL"}")); +} diff --git a/examples/Tcp/Observability/Tcp_004_HealthChecks.cs b/examples/Tcp/Observability/Tcp_004_HealthChecks.cs new file mode 100644 index 000000000..585875494 --- /dev/null +++ b/examples/Tcp/Observability/Tcp_004_HealthChecks.cs @@ -0,0 +1,54 @@ +using ClickHouse.Driver.Tcp; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace ClickHouse.Driver.Examples; + +/// Uses PingAsync in a Microsoft.Extensions.Diagnostics health check. +public static class TcpHealthChecks +{ + public static async Task Run() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString); + services.AddHealthChecks().Add(new HealthCheckRegistration( + "clickhouse-native", + provider => new TcpPingHealthCheck( + provider.GetRequiredService()), + failureStatus: HealthStatus.Unhealthy, + tags: new[] { "clickhouse", "ready" })); + + await using ServiceProvider provider = services.BuildServiceProvider(); + HealthReport report = await provider + .GetRequiredService() + .CheckHealthAsync(); + + foreach ((string name, HealthReportEntry entry) in report.Entries) + { + Console.WriteLine($"{name}: {entry.Status} ({entry.Description})"); + } + } + + private sealed class TcpPingHealthCheck(IClickHouseTcpClient client) : IHealthCheck + { + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + try + { + // Ping verifies native connectivity, not access to an application table. + await client.PingAsync(cancellationToken); + return HealthCheckResult.Healthy("Pong"); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return new HealthCheckResult( + context.Registration.FailureStatus, + ex.Message, + ex); + } + } + } +} diff --git a/examples/Tcp/Observability/Tcp_005_Testcontainers.cs b/examples/Tcp/Observability/Tcp_005_Testcontainers.cs new file mode 100644 index 000000000..58ee1599c --- /dev/null +++ b/examples/Tcp/Observability/Tcp_005_Testcontainers.cs @@ -0,0 +1,78 @@ +using System.Diagnostics; +using System.Net.Sockets; +using ClickHouse.Driver.Tcp; +using DotNet.Testcontainers.Builders; +using Testcontainers.ClickHouse; + +namespace ClickHouse.Driver.Examples; + +/// Starts ClickHouse with Testcontainers and connects through its native port. +public static class TcpTestcontainers +{ + private const string Image = "clickhouse/clickhouse-server:25.12-alpine"; + private const ushort NativePort = 9000; + private const ushort HttpPort = 8123; + private const string User = "example"; + private const string Password = "example"; + + public static async Task Run() + { + await using ClickHouseContainer container = new ClickHouseBuilder(Image) + .WithUsername(User) + .WithPassword(Password) + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilHttpRequestIsSucceeded(request => request.ForPath("/ping").ForPort(HttpPort)) + .UntilInternalTcpPortIsAvailable(NativePort)) + .Build(); + + await container.StartAsync(); + + // GetConnectionString() targets HTTP. Build a native connection from mapped port 9000. + var builder = new ClickHouseTcpConnectionStringBuilder + { + Host = container.Hostname, + Port = container.GetMappedPublicPort(NativePort), + Username = User, + Password = Password, + Database = "default", + }; + + Console.WriteLine($"HTTP connection: {container.GetConnectionString()}"); + Console.WriteLine($"Native endpoint: {builder.Host}:{builder.Port}"); + + await using var client = new ClickHouseTcpClient(builder.ToOptions()); + ClickHouseTcpServerInfo server = await WaitForHandshake(client); + Console.WriteLine($"Connected to {server}."); + + await client.ExecuteAsync(""" + CREATE TABLE values (id UInt64, note String) + ENGINE = MergeTree + ORDER BY id + """); + await client.InsertRowsAsync( + "INSERT INTO values (id, note) VALUES", + new[] { new object[] { 1UL, "from the native client" } }); + + object value = await client.ExecuteScalarAsync("SELECT note FROM values WHERE id = 1"); + Console.WriteLine(value); + } + + private static async Task WaitForHandshake(ClickHouseTcpClient client) + { + var timeout = Stopwatch.StartNew(); + + while (true) + { + try + { + return await client.GetServerInfoAsync(); + } + catch (ClickHouseTcpTransportException ex) + when (ex.InnerException is SocketException && timeout.Elapsed < TimeSpan.FromSeconds(30)) + { + // A bound port can briefly refuse native handshakes while ClickHouse starts. + await Task.Delay(100); + } + } + } +} diff --git a/examples/Tcp/Observability/Tcp_026_Logging.cs b/examples/Tcp/Observability/Tcp_026_Logging.cs deleted file mode 100644 index b24195dd8..000000000 --- a/examples/Tcp/Observability/Tcp_026_Logging.cs +++ /dev/null @@ -1,347 +0,0 @@ -using ClickHouse.Driver.Tcp; -using Microsoft.Extensions.Logging; - -namespace ClickHouse.Driver.Examples; - -/// -/// and the three categories the native client logs under — -/// , -/// and : what each reports, at which level, and how to keep -/// one and drop the rest. -/// -/// -/// The client logs its own lifecycle. It never logs what the server reports; the server's log lines arrive -/// as a callback (Tcp_028). Nothing here reports pool state either — the pool's lines are the only window into it, -/// which is what Tcp_017 reads. -/// -/// -/// -/// The levels are the thing to plan around: the client writes almost everything at Debug or Trace, -/// and the statement text rides on a Debug line. So the two filter sets below are genuinely different -/// configurations, not one dialled up — and a stock , whose minimum level is -/// Information, shows nearly none of it. -/// -/// -public static class TcpLogging -{ - public static async Task Run() - { - await OneWorkloadThreeCategories(); - await WhichLevelsAreUsed(); - await AStockFactoryShowsAlmostNothing(); - await TwoFilterSets(); - await StatementTextRidesOnADebugLine(); - WhatIsNotHere(); - } - - private static async Task OneWorkloadThreeCategories() - { - Console.WriteLine("1. Three categories, one workload\n"); - - var recorder = new Recorder(); - using ILoggerFactory factory = LoggerFactory.Create(builder => builder - .AddProvider(recorder) - .SetMinimumLevel(LogLevel.Trace)); - - await Workload(factory); - - Console.WriteLine(" Two queries, then one that names a table that does not exist:\n"); - foreach (Line line in recorder.Lines) - { - Console.WriteLine($" {line.ShortCategory,-10} {line.Level,-11} {line.Message}"); - } - - Console.WriteLine(); - Console.WriteLine(" Client what ran, how long it took, how it ended — and the statement text"); - Console.WriteLine(" Connection the dial, the TLS negotiation, and the handshake result"); - Console.WriteLine(" Pool checkouts, retirement, exhaustion, and the background work nobody awaits"); - Console.WriteLine(); - Console.WriteLine(" Each is a full logger category, so the usual per-category configuration applies:"); - Console.WriteLine(" appsettings' Logging:LogLevel section, AddFilter, or a filter predicate as below."); - } - - private static async Task WhichLevelsAreUsed() - { - Console.WriteLine("\n2. Which levels each category actually uses\n"); - - var recorder = new Recorder(); - using ILoggerFactory factory = LoggerFactory.Create(builder => builder - .AddProvider(recorder) - .SetMinimumLevel(LogLevel.Trace)); - - // The same workload, plus the two things that log above Debug: a dial that fails, and a pool with nothing - // left to hand out. - await Workload(factory); - await FailedDial(factory); - await ExhaustedPool(factory); - - foreach (var group in recorder.Lines - .GroupBy(l => (l.ShortCategory, l.Level)) - .OrderBy(g => g.Key.ShortCategory, StringComparer.Ordinal) - .ThenByDescending(g => g.Key.Level)) - { - Console.WriteLine($" {group.Key.ShortCategory,-10} {group.Key.Level,-11} {group.Count(),2} line(s) e.g. {Trim(group.First().Message)}"); - } - - Console.WriteLine(); - Console.WriteLine(" Nothing is logged at Information or Critical. Warning is the top of the range and it is"); - Console.WriteLine(" reserved for four messages: a dial that failed, PoolTimeout, and the two background jobs"); - Console.WriteLine(" nobody awaits (a failed top-up towards MinPoolSize, a failed sweep) — which are reported"); - Console.WriteLine(" nowhere else at all. Error is a single message, an operation that threw — twice here,"); - Console.WriteLine(" once for the unknown table and once for the query that never got a connection."); - } - - private static async Task AStockFactoryShowsAlmostNothing() - { - Console.WriteLine("\n3. A factory with no minimum level set shows almost none of it\n"); - - var recorder = new Recorder(); - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddProvider(recorder)); - - await Workload(factory); - - Console.WriteLine($" LoggerFactory.Create(b => b.AddProvider(...)) with no SetMinimumLevel: {recorder.Lines.Count} line(s) kept"); - foreach (Line line in recorder.Lines) - { - Console.WriteLine($" {line.ShortCategory,-10} {line.Level,-11} {line.Message}"); - } - - Console.WriteLine(); - Console.WriteLine(" Microsoft.Extensions.Logging defaults its minimum to Information, and the client logs"); - Console.WriteLine(" nothing there, so a factory that was wired up correctly still looks broken. Set the level"); - Console.WriteLine(" for the categories you want, not globally: Trace across the whole application is a lot of"); - Console.WriteLine(" log."); - } - - private static async Task TwoFilterSets() - { - Console.WriteLine("\n4. Two filter sets: one for production, one for a connection problem\n"); - - // Production: only the lines that mean something is wrong. No statement text, because the line that - // carries it is a Debug line. - var production = new Recorder(); - using (ILoggerFactory factory = LoggerFactory.Create(builder => builder - .AddProvider(production) - .AddFilter((category, level) => category?.StartsWith("ClickHouse.Driver.Tcp.", StringComparison.Ordinal) == true && level >= LogLevel.Warning) - .SetMinimumLevel(LogLevel.Warning))) - { - await Workload(factory); - await FailedDial(factory); - await ExhaustedPool(factory); - } - - Console.WriteLine($" Production — every category, Warning and worse: {production.Lines.Count} line(s)\n"); - foreach (Line line in production.Lines) - { - Console.WriteLine($" {line.ShortCategory,-10} {line.Level,-11} {Trim(line.Message)}"); - } - - // Debugging a connection problem: the two categories that know about sockets, at Trace, and nothing else. - var connections = new Recorder(); - using (ILoggerFactory factory = LoggerFactory.Create(builder => builder - .AddProvider(connections) - .AddFilter((category, level) => category switch - { - ClickHouseTcpDiagnostics.ConnectionLogCategory => level >= LogLevel.Trace, - ClickHouseTcpDiagnostics.PoolLogCategory => level >= LogLevel.Trace, - _ => false, - }) - .SetMinimumLevel(LogLevel.Trace))) - { - await Workload(factory); - await FailedDial(factory); - await ExhaustedPool(factory); - } - - Console.WriteLine($"\n Debugging a connection problem — Connection and Pool at Trace, nothing else: {connections.Lines.Count} line(s)\n"); - foreach (Line line in connections.Lines) - { - Console.WriteLine($" {line.ShortCategory,-10} {line.Level,-11} {Trim(line.Message)}"); - } - - Console.WriteLine(); - Console.WriteLine(" The second set answers the questions the first cannot: how many connections were opened,"); - Console.WriteLine(" whether a query reused one or dialled, how old the reused one was, and whether a returned"); - Console.WriteLine(" connection went back into the pool. Note that no Client line appears in it — the query"); - Console.WriteLine(" text is deliberately out, which is what makes the set safe to turn on against a live"); - Console.WriteLine(" system."); - Console.WriteLine(); - Console.WriteLine(" Both are predicates over the category string, so they can key on"); - Console.WriteLine(" ClickHouseTcpDiagnostics.ClientLogCategory and its two siblings rather than a literal."); - } - - private static async Task StatementTextRidesOnADebugLine() - { - Console.WriteLine("\n5. The statement text, and the one line that carries it\n"); - - const string sql = "SELECT 'the whole statement, or as much of it as StatementMaxLength allows'"; - - foreach (int max in new[] { 0, 30, 200 }) - { - var recorder = new Recorder(); - using ILoggerFactory factory = LoggerFactory.Create(builder => builder - .AddProvider(recorder) - .AddFilter((category, level) => category == ClickHouseTcpDiagnostics.ClientLogCategory && level >= LogLevel.Debug) - .SetMinimumLevel(LogLevel.Debug)); - - await using (var client = new ClickHouseTcpClient(Options() with - { - LoggerFactory = factory, - StatementMaxLength = max, - })) - { - _ = await client.ExecuteScalarAsync(sql); - } - - string running = recorder.Lines.First(l => l.Message.StartsWith("Running", StringComparison.Ordinal)).Message; - Console.WriteLine($" StatementMaxLength = {max,3} {running}"); - } - - Console.WriteLine(); - Console.WriteLine($" The statement was {sql.Length} characters. Zero keeps the text out of the log line while"); - Console.WriteLine(" leaving the line itself — which is the production recipe if you want a record of what ran"); - Console.WriteLine(" and how long it took without putting query text in your logs. The same knob caps the"); - Console.WriteLine(" db.query.text span attribute (Tcp_027), and Tcp_019 covers it as a limit."); - } - - private static void WhatIsNotHere() - { - Console.WriteLine("\n6. What these categories do not carry\n"); - Console.WriteLine(" The server's own log lines. Those come from the query, not the client, and reach you"); - Console.WriteLine(" through ClickHouseTcpQueryCallbacks.OnLog with send_logs_level set — Tcp_028. Bridging"); - Console.WriteLine(" them into an ILogger is a few lines, and yours to write."); - Console.WriteLine(); - Console.WriteLine(" Pool counters. There is no open/idle/in-use to read, so the Pool category's lines are the"); - Console.WriteLine(" only window into the pool; Tcp_017 measures it that way."); - Console.WriteLine(); - Console.WriteLine(" A connection identity. The reuse line carries a use count and an age, but nothing names"); - Console.WriteLine(" the connection, so two lines about the same socket cannot be tied together."); - Console.WriteLine(); - Console.WriteLine(" In an application you would not build the factory by hand at all: register logging in the"); - Console.WriteLine(" container and AddClickHouseTcpDataSource fills LoggerFactory in from it (Tcp_003)."); - } - - private static ClickHouseTcpClientOptions Options() => ExampleConfig.TcpBuilder().ToOptions(); - - /// Two queries that succeed and one that does not, on a client of this example's own. - private static async Task Workload(ILoggerFactory factory) - { - await using var client = new ClickHouseTcpClient(Options() with - { - LoggerFactory = factory, - StatementMaxLength = 60, - }); - - _ = await client.ExecuteScalarAsync("SELECT 'the first query has to open a connection'"); - _ = await client.ExecuteScalarAsync("SELECT 'the second reuses it'"); - - // An unknown table is reported after the server accepted the query, so the connection survives it and goes - // back into the pool. The client logs one Error line and rethrows. - try - { - // Unique, so the query fails because the table does not exist rather than because it happens not - // to exist: a fixed name is one CREATE TABLE away from making this demonstration succeed. - _ = await client.ExecuteScalarAsync($"SELECT * FROM example_tcp_logging_no_such_table_{Guid.NewGuid():N}"); - } - catch (ClickHouseTcpServerException) - { - } - } - - /// A dial that fails at once, for the Connection category's one Warning. - private static async Task FailedDial(ILoggerFactory factory) - { - await using var client = new ClickHouseTcpClient(Options() with - { - LoggerFactory = factory, - Port = 1, - DialTimeout = TimeSpan.FromSeconds(2), - }); - - try - { - await client.PingAsync(); - } - catch (ClickHouseTcpTransportException) - { - } - } - - /// A pool with its only connection pinned by a session, for the Pool category's PoolTimeout Warning. - private static async Task ExhaustedPool(ILoggerFactory factory) - { - await using var client = new ClickHouseTcpClient(Options() with - { - LoggerFactory = factory, - MaxPoolSize = 1, - PoolTimeout = TimeSpan.FromMilliseconds(200), - }); - - await using IClickHouseTcpSession session = await client.OpenSessionAsync(); - - try - { - _ = await client.ExecuteScalarAsync("SELECT 1"); - } - catch (TimeoutException) - { - } - } - - private static string Trim(string message) - => message.Length <= 96 ? message : message[..96] + "..."; - - private readonly record struct Line(string Category, LogLevel Level, string Message) - { - /// The part after the last dot — Client, Connection or Pool. - public string ShortCategory => Category[(Category.LastIndexOf('.') + 1)..]; - } - - /// - /// An that keeps the lines rather than printing them, so a section can report - /// what its filter kept. Registered with AddProvider, so the builder's filters really do apply — a - /// factory that only wraps would bypass them and prove nothing. - /// - private sealed class Recorder : ILoggerProvider - { - private readonly List lines = []; - - public IReadOnlyList Lines - { - get - { - lock (lines) - { - return lines.ToArray(); - } - } - } - - public ILogger CreateLogger(string categoryName) => new Sink(categoryName, lines); - - public void Dispose() - { - } - - private sealed class Sink(string category, List lines) : ILogger - { - public bool IsEnabled(LogLevel logLevel) => true; - - public IDisposable? BeginScope(TState state) - where TState : notnull => null; - - public void Log( - LogLevel logLevel, - EventId eventId, - TState state, - Exception? exception, - Func formatter) - { - lock (lines) - { - lines.Add(new Line(category, logLevel, formatter(state, exception))); - } - } - } - } -} diff --git a/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs b/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs deleted file mode 100644 index defbec037..000000000 --- a/examples/Tcp/Observability/Tcp_027_OpenTelemetry.cs +++ /dev/null @@ -1,395 +0,0 @@ -using System.Diagnostics; -using ClickHouse.Driver.Diagnostic; -using ClickHouse.Driver.Tcp; -using ClickHouse.Driver.Utility; -using OpenTelemetry; -using OpenTelemetry.Trace; - -namespace ClickHouse.Driver.Examples; - -/// -/// Tracing the native client: , the spans it emits and -/// the attributes they carry, and . -/// -/// -/// The spans are collected here by an exporter that keeps them, and printed, because a console exporter's output -/// is too wide to read next to the code that produced it. The wiring is otherwise exactly what an application -/// does — Sdk.CreateTracerProviderBuilder().AddSource(ClickHouseTcpDiagnostics.ActivitySourceName) — so -/// swapping in AddOtlpExporter() is the only change needed to send these spans somewhere real. -/// -/// -/// -/// The attribute names are the current OpenTelemetry database conventions (db.system.name, -/// db.namespace, db.query.text, server.address), which is where this transport differs from -/// the HTTP one: it still emits the older db.system/db.statement set. The two also use different -/// source names, so either can be collected without the other. -/// -/// -public static class TcpOpenTelemetry -{ - private const string TableName = "example_tcp_open_telemetry"; - - /// Stands in for the application's own instrumentation, so the client's spans have a parent. - private static readonly ActivitySource AppSource = new("ClickHouse.Driver.Examples.Tcp027"); - - public static async Task Run() - { - Console.WriteLine($"The native client's ActivitySource: {ClickHouseTcpDiagnostics.ActivitySourceName}"); - Console.WriteLine($"The HTTP transport's, for comparison: {ClickHouseDiagnosticsOptions.ActivitySourceName}\n"); - - await using var client = ExampleConfig.CreateTcpClient(); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($"CREATE TABLE {TableName} (id UInt64, note String) ENGINE = MergeTree ORDER BY id"); - - try - { - await OneSpanPerOperation(); - await TheParentChildShape(); - await StatementTextIsOptIn(); - await TheServerJoinsTheSameTrace(client); - await TwoTransportsTwoSources(); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped {TableName}."); - } - } - - private static async Task OneSpanPerOperation() - { - Console.WriteLine("1. One span per operation, named after the statement\n"); - - var collector = new SpanCollector(); - using (TracerProvider provider = Collect(collector, ClickHouseTcpDiagnostics.ActivitySourceName)) - { - await using var client = new ClickHouseTcpClient(Options() with - { - IncludeSqlInActivityTags = true, - StatementMaxLength = 120, - }); - - await client.PingAsync(); - _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(50000)"); - await client.InsertRowsAsync( - $"INSERT INTO {TableName} (id, note) VALUES", - [[1UL, "one"], [2UL, "two"], [3UL, "three"]]); - - // An error after the server accepted the query, so the span records a failure and the connection - // stays usable. - try - { - // Unique for the same reason as in Tcp_026: the failure has to be the missing table. - _ = await client.ExecuteScalarAsync( - $"SELECT * FROM example_tcp_open_telemetry_no_such_table_{Guid.NewGuid():N}"); - } - catch (ClickHouseTcpServerException) - { - } - } - - foreach (Activity span in collector.Spans) - { - Print(span); - } - - Console.WriteLine(" The span name is the statement's leading keyword, uppercased, which keeps it low"); - Console.WriteLine(" cardinality — a generated statement that does not start with a word is named 'query'."); - Console.WriteLine(" A Ping is its own span, and so is a dial (below)."); - Console.WriteLine(); - Console.WriteLine(" db.clickhouse.read_rows and read_bytes come from the server's Progress packets, so they"); - Console.WriteLine(" describe what the query read rather than what it returned; result_rows and result_bytes"); - Console.WriteLine(" are the execution summary. An insert has neither pair: the server sends no Progress for"); - Console.WriteLine(" rows streamed to it, so db.clickhouse.written_rows is the client's own count."); - } - - private static async Task TheParentChildShape() - { - Console.WriteLine("\n2. Where the client's spans sit in a trace\n"); - - var collector = new SpanCollector(); - using (TracerProvider provider = Collect(collector, ClickHouseTcpDiagnostics.ActivitySourceName, AppSource.Name)) - { - // The application's own span. Everything the client starts while it is current becomes a descendant. - using (Activity? request = AppSource.StartActivity("handle-request")) - { - // A client of its own, so its pool is empty and the first operation has to dial. - await using var client = new ClickHouseTcpClient(Options() with - { - IncludeSqlInActivityTags = true, - StatementMaxLength = 60, - }); - - _ = await client.ExecuteScalarAsync("SELECT 'first, so this one dials'"); - _ = await client.ExecuteScalarAsync("SELECT 'second, so this one does not'"); - } - } - - PrintTree(collector.Spans); - - Console.WriteLine(); - Console.WriteLine(" 'connect' covers the socket connect, the TLS negotiation and the handshake, and it is a"); - Console.WriteLine(" child of whichever operation had to wait for the connection — so a slow first request"); - Console.WriteLine(" shows why in the trace rather than only in the total. The second statement has no such"); - Console.WriteLine(" child because it reused the pooled connection."); - Console.WriteLine(); - Console.WriteLine(" With no ambient Activity the client's spans are roots, one trace each — which is what"); - Console.WriteLine(" section 1 above produced. A parent is also what the server is told about, so the shape"); - Console.WriteLine(" above reaches further than this process: section 4."); - } - - private static async Task StatementTextIsOptIn() - { - Console.WriteLine("\n3. IncludeSqlInActivityTags, and how much text it lets through\n"); - - const string sql = "SELECT 'a statement long enough that StatementMaxLength has something to cut'"; - - (bool include, int max)[] cases = - [ - (false, 200), - (true, 40), - (true, 200), - (true, 0), - ]; - - foreach ((bool include, int max) in cases) - { - var collector = new SpanCollector(); - using (TracerProvider provider = Collect(collector, ClickHouseTcpDiagnostics.ActivitySourceName)) - { - await using var client = new ClickHouseTcpClient(Options() with - { - IncludeSqlInActivityTags = include, - StatementMaxLength = max, - }); - - _ = await client.ExecuteScalarAsync(sql); - } - - Activity span = collector.Spans.First(s => s.OperationName == "SELECT"); - object? text = span.GetTagItem("db.query.text"); - Console.WriteLine($" IncludeSqlInActivityTags = {include,-5} StatementMaxLength = {max,3} db.query.text = {(text is null ? "(not set)" : "\"" + text + "\"")}"); - } - - Console.WriteLine(); - Console.WriteLine($" The statement was {sql.Length} characters. Off is the default, because a statement can carry"); - Console.WriteLine(" data a trace is not meant to hold — a literal in a WHERE clause is often the very value"); - Console.WriteLine(" you are not allowed to export. StatementMaxLength defaults to 5, a stub rather than a"); - Console.WriteLine(" statement, so recording query text takes both settings; zero suppresses the attribute"); - Console.WriteLine(" even with the opt-in on. It caps the Debug log line by the same rule (Tcp_026)."); - } - - /// - /// The client writes the current span's W3C trace context into the Query packet, so the spans the server - /// records for the same query land under the caller's trace id. - /// - /// A client for reading the server's span log, whose own spans are not collected. - private static async Task TheServerJoinsTheSameTrace(ClickHouseTcpClient reader) - { - Console.WriteLine("\n4. The server's own spans join the same trace\n"); - - var collector = new SpanCollector(); - string traceId; - - using (TracerProvider provider = Collect(collector, ClickHouseTcpDiagnostics.ActivitySourceName, AppSource.Name)) - { - await using var client = new ClickHouseTcpClient(Options()); - - using Activity? request = AppSource.StartActivity("handle-request"); - traceId = request!.TraceId.ToHexString(); - _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(100000)"); - } - - Console.WriteLine($" Trace id on this side: {traceId}"); - Console.WriteLine($" Spans collected here: {string.Join(", ", collector.Spans.Select(s => s.OperationName))}"); - - // The server's spans are queued like any system log, so the flush and the read are retried rather than - // read once — the same shape Tcp_020 uses for system.query_log. - long serverSpans = 0; - var names = new List(); - for (int attempt = 1; attempt <= 5 && serverSpans == 0; attempt++) - { - await reader.ExecuteAsync("SYSTEM FLUSH LOGS"); - serverSpans = Convert.ToInt64(await reader.ExecuteScalarAsync( - "SELECT count() FROM system.opentelemetry_span_log WHERE lower(hex(trace_id)) = {trace:String}", - new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "trace", traceId } }, - })); - - if (serverSpans == 0) - { - await Task.Delay(50); - } - } - - await foreach (object[] row in reader.QueryAsync( - "SELECT DISTINCT operation_name FROM system.opentelemetry_span_log " + - "WHERE lower(hex(trace_id)) = {trace:String} ORDER BY operation_name LIMIT 6", - new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "trace", traceId } }, - })) - { - names.Add((string)row[0]); - } - - Console.WriteLine($" Spans the server recorded under the same trace id: {serverSpans}"); - Console.WriteLine($" {string.Join(", ", names)}"); - Console.WriteLine(); - Console.WriteLine(" The Query packet's ClientInfo carries the W3C trace context of Activity.Current when the"); - Console.WriteLine(" negotiated protocol revision is 54442 or newer, which every supported server is. So the"); - Console.WriteLine(" server's account of the query — every stage, in system.opentelemetry_span_log — is part"); - Console.WriteLine(" of the same trace as the request that issued it, with no header to set and nothing to"); - Console.WriteLine(" correlate by hand. That is the strongest reason to give the client an ambient Activity."); - Console.WriteLine(); - Console.WriteLine(" Two conditions. The current Activity's id has to be W3C, which it is unless something"); - Console.WriteLine(" set ActivityIdFormat.Hierarchical; and the server has to have its span log switched on,"); - Console.WriteLine(" which the stock configuration does. The flush above is only so this example can read the"); - Console.WriteLine(" table immediately; nothing about the propagation needs it."); - Console.WriteLine(); - Console.WriteLine(" db.clickhouse.query_id is the other join, and it needs a QueryId you chose (Tcp_020):"); - Console.WriteLine(" when you supply none, the id the server assigns never reaches the client."); - } - - private static async Task TwoTransportsTwoSources() - { - Console.WriteLine("\n5. The two transports are separate sources\n"); - - // Only the native source. The HTTP query below runs, and is not collected. - var nativeOnly = new SpanCollector(); - using (TracerProvider provider = Collect(nativeOnly, ClickHouseTcpDiagnostics.ActivitySourceName)) - { - await BothTransports(); - } - - // Both sources, same workload. - var both = new SpanCollector(); - using (TracerProvider provider = Collect( - both, - ClickHouseTcpDiagnostics.ActivitySourceName, - ClickHouseDiagnosticsOptions.ActivitySourceName)) - { - await BothTransports(); - } - - Console.WriteLine(" One native query and one HTTP query, collected twice:\n"); - Report("AddSource(native)", nativeOnly); - Report("AddSource(native, http)", both); - - Console.WriteLine(); - Console.WriteLine(" So a service that has moved its reads to the native client and left its writes on HTTP"); - Console.WriteLine(" can trace one, the other, or both, and tell them apart in the backend by source. The"); - Console.WriteLine(" attribute sets differ as well: the HTTP transport emits db.system and db.statement, this"); - Console.WriteLine(" one db.system.name and db.query.text, so a dashboard built on one does not read the"); - Console.WriteLine(" other without a rule for each. The span names differ too — the HTTP one is named after"); - Console.WriteLine(" the driver method that ran, this one after the statement's keyword."); - Console.WriteLine(); - Console.WriteLine(" Their opt-ins are separate too, and shaped differently: the HTTP transport's live on the"); - Console.WriteLine(" static ClickHouseDiagnosticsOptions, so they are process-wide, while the native client's"); - Console.WriteLine(" are per client, on the options record."); - - static void Report(string label, SpanCollector collector) - { - IEnumerable byTransport = collector.Spans - .GroupBy(s => s.Source.Name) - .OrderBy(g => g.Key, StringComparer.Ordinal) - .Select(g => $"{g.Key} -> {string.Join(", ", g.Select(s => s.OperationName))}"); - - Console.WriteLine($" {label,-24} {collector.Spans.Count} span(s): {string.Join("; ", byTransport)}"); - } - } - - /// One query over each transport, so a collector can be asked which of them it saw. - private static async Task BothTransports() - { - await using var tcp = new ClickHouseTcpClient(Options()); - _ = await tcp.ExecuteScalarAsync("SELECT 'over the native protocol'"); - - using var http = ExampleConfig.CreateHttpClient(); - _ = await http.ExecuteScalarAsync("SELECT 'over HTTP'"); - } - - private static ClickHouseTcpClientOptions Options() => ExampleConfig.TcpBuilder().ToOptions(); - - /// - /// The wiring an application writes, with an exporter that keeps the spans instead of printing them. - /// - private static TracerProvider Collect(SpanCollector collector, params string[] sources) - => Sdk.CreateTracerProviderBuilder() - .AddSource(sources) - .AddProcessor(new SimpleActivityExportProcessor(collector)) - .Build()!; - - private static void Print(Activity span) - { - Console.WriteLine($" {span.OperationName,-8} {span.Kind,-6} {span.Status,-5} {span.Duration.TotalMilliseconds,7:0.0} ms"); - foreach (KeyValuePair tag in span.TagObjects) - { - Console.WriteLine($" {tag.Key,-30} {tag.Value}"); - } - - foreach (ActivityEvent e in span.Events) - { - Console.WriteLine($" event {e.Name,-24} {e.Tags.FirstOrDefault(t => t.Key == "exception.type").Value}"); - } - - Console.WriteLine(); - } - - /// Prints the spans indented by depth, which is what a trace viewer draws. - private static void PrintTree(IReadOnlyList spans) - { - var byId = spans.ToDictionary(s => s.SpanId.ToHexString(), StringComparer.Ordinal); - - foreach (Activity span in spans.OrderBy(s => s.StartTimeUtc)) - { - int depth = 0; - for (Activity? walk = span; walk is not null && depth < 8;) - { - walk = byId.TryGetValue(walk.ParentSpanId.ToHexString(), out Activity? parent) ? parent : null; - if (walk is not null) - { - depth++; - } - } - - string? sql = span.GetTagItem("db.query.text") as string; - Console.WriteLine($" {new string(' ', depth * 3)}{span.OperationName,-8} {span.Duration.TotalMilliseconds,7:0.0##} ms{(sql is null ? string.Empty : " " + sql)}"); - } - } - - /// - /// A that keeps what it is given. Registered through - /// , so each span is handed over as it ends and nothing has to be - /// flushed before a section prints. - /// - private sealed class SpanCollector : BaseExporter - { - private readonly List spans = []; - - public IReadOnlyList Spans - { - get - { - lock (spans) - { - return spans.ToArray(); - } - } - } - - public override ExportResult Export(in Batch batch) - { - lock (spans) - { - foreach (Activity span in batch) - { - spans.Add(span); - } - } - - return ExportResult.Success; - } - } -} diff --git a/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs b/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs deleted file mode 100644 index 7b901e099..000000000 --- a/examples/Tcp/Observability/Tcp_028_MetadataBlocks.cs +++ /dev/null @@ -1,336 +0,0 @@ -using ClickHouse.Driver.Tcp; -using Microsoft.Extensions.Logging; - -namespace ClickHouse.Driver.Examples; - -/// -/// The three -shaped callbacks on : -/// with send_logs_level, -/// for WITH TOTALS, and -/// with the extremes setting. Tcp_021 covers the other -/// three, which hand over structs rather than blocks. -/// -/// -/// Every one of these blocks is borrowed. Its columns are views over pooled buffers that are released as -/// soon as the callback returns, so the rule for all three is the same: copy out what you need inside the -/// callback, and keep neither the block, its columns, nor a span over them. Every section below does the copying -/// in the callback and the printing afterwards, which is also what an application does — a callback runs -/// synchronously on the thread draining the response, so the less it does the better. -/// -/// -/// -/// The contract otherwise is the one Tcp_021 states: in packet order, on the reading thread, and never allowed to -/// throw — an exception propagates out of the operation and terminates the connection. So keep the callback to -/// copying values out, and do the parsing that can fail somewhere it is allowed to. Even a copy has to be written -/// with that in mind: a named column lookup or a span index throws if the name or the row is not there. -/// -/// -public static class TcpMetadataBlocks -{ - public static async Task Run() - { - WhatTurnsEachOneOn(); - - await using var client = ExampleConfig.CreateTcpClient(); - - await ServerLogLines(client); - await BridgingThemIntoALogger(client); - await HowMuchEachLevelSays(client); - await TheTotalsRow(client); - await TheExtremesRows(client); - await NothingFiresWhenThereIsNothingToSend(client); - } - - private static void WhatTurnsEachOneOn() - { - Console.WriteLine("Three callbacks, and what each one needs before the server sends anything:\n"); - Console.WriteLine(" OnLog Settings[\"send_logs_level\"] = \"debug\" (or trace) — the default, fatal, is silent"); - Console.WriteLine(" OnTotals WITH TOTALS in the query, right after GROUP BY"); - Console.WriteLine(" OnExtremes Settings[\"extremes\"] = \"1\""); - Console.WriteLine(); - Console.WriteLine("Setting the callback alone gets you nothing, and so does turning the feature on without the"); - Console.WriteLine("callback: the block is decoded either way, to keep the connection aligned, and then dropped."); - } - - private static async Task ServerLogLines(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. OnLog: the server's own log, for this query only\n"); - - // The copies. Everything that outlives the callback is in here, and nothing in here points into a block. - var lines = new List<(sbyte Priority, string Source, string Text, uint EventTime)>(); - int blocks = 0; - - var options = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["send_logs_level"] = "trace" }, - Callbacks = new ClickHouseTcpQueryCallbacks - { - OnLog = block => - { - blocks++; - - // Named columns, so the order on the wire does not matter. The span is borrowed; the strings - // an IColumn hands back are already copies. - ReadOnlySpan priority = block.Column("priority").Values; - ReadOnlySpan eventTime = block.Column("event_time").Values; - IColumn source = block.Column("source"); - IColumn text = block.Column("text"); - - for (int row = 0; row < block.RowCount; row++) - { - lines.Add((priority[row], source[row], text[row], eventTime[row])); - } - }, - }, - }; - - _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(200000)", options); - - Console.WriteLine($" {blocks} log blocks, {lines.Count} lines, all of them copied out before the blocks went back:\n"); - foreach ((sbyte priority, string source, string text, uint _) in lines.Take(8)) - { - Console.WriteLine($" {priority} {source,-22} {(text.Length <= 78 ? text : text[..78] + "...")}"); - } - - if (lines.Count > 8) - { - Console.WriteLine($" ... and {lines.Count - 8} more"); - } - - Console.WriteLine(); - Console.WriteLine(" The columns are event_time, event_time_microseconds, host_name, query_id, thread_id,"); - Console.WriteLine(" priority, source and text. event_time is a DateTime column, which on this tier is the"); - Console.WriteLine($" integer the wire carried — {lines[0].EventTime} whole Unix seconds, so"); - Console.WriteLine($" DateTimeOffset.FromUnixTimeSeconds gives {DateTimeOffset.FromUnixTimeSeconds(lines[0].EventTime):HH:mm:ss} UTC, with the sub-second part in the"); - Console.WriteLine(" microseconds column beside it."); - Console.WriteLine(); - Console.WriteLine(" These are the same lines the server writes to its own log, so this is how a client gets"); - Console.WriteLine(" the server's account of one query without access to the server's log file — which is what"); - Console.WriteLine(" makes it useful when a query is slow on someone else's cluster."); - } - - private static async Task BridgingThemIntoALogger(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. priority is a Poco severity, so a lower number is more severe\n"); - Console.WriteLine(" 1 fatal 2 critical 3 error 4 warning 5 notice"); - Console.WriteLine(" 6 information 7 debug 8 trace 9 test\n"); - - var seen = new SortedDictionary(); - - var options = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["send_logs_level"] = "trace" }, - Callbacks = new ClickHouseTcpQueryCallbacks - { - OnLog = block => - { - ReadOnlySpan priority = block.Column("priority").Values; - IColumn source = block.Column("source"); - - for (int row = 0; row < block.RowCount; row++) - { - seen.TryGetValue(priority[row], out (int Count, string Example) soFar); - seen[priority[row]] = (soFar.Count + 1, source[row]); - } - }, - }, - }; - - _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(200000)", options); - - Console.WriteLine(" What this query reported, and where each line would go in an ILogger:\n"); - foreach ((sbyte priority, (int count, string example)) in seen) - { - Console.WriteLine($" priority {priority} {count,2} line(s) ILogger level {ToLogLevel(priority),-11} e.g. from {example}"); - } - - Console.WriteLine(); - Console.WriteLine(" So filter with <=, and treat anything outside 1..9 as unknown rather than as severe. A"); - Console.WriteLine(" query that runs cleanly says nothing above debug, which is why raising send_logs_level to"); - Console.WriteLine(" warning is a way to be told only about the queries that had a problem."); - Console.WriteLine(); - Console.WriteLine(" Forwarding these to an ILogger is a few lines and yours to write: the client logs its own"); - Console.WriteLine(" lifecycle only (Tcp_026) and never what the server says."); - } - - private static async Task HowMuchEachLevelSays(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. What each send_logs_level costs\n"); - - foreach (string level in new[] { "none", "warning", "information", "debug", "trace" }) - { - int blocks = 0; - int rows = 0; - - _ = await client.ExecuteScalarAsync("SELECT count() FROM numbers(200000)", new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["send_logs_level"] = level }, - Callbacks = new ClickHouseTcpQueryCallbacks - { - OnLog = block => - { - blocks++; - rows += block.RowCount; - }, - }, - }); - - Console.WriteLine($" send_logs_level = {level,-12} {blocks} block(s), {rows,2} line(s)"); - } - - Console.WriteLine(); - Console.WriteLine(" The lines are packets on the same connection as the result, so they are not free: text"); - Console.WriteLine(" the server would otherwise only write to its own log crosses the wire. trace on a busy"); - Console.WriteLine(" client is a lot of it. debug on the queries you are investigating is the usable setting,"); - Console.WriteLine(" and it can be set per query rather than on the client (Tcp_020)."); - } - - private static async Task TheTotalsRow(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. OnTotals: the WITH TOTALS row, in the query's own shape\n"); - - const string sql = - "SELECT number % 3 AS bucket, count() AS rows, sum(number) AS total " + - "FROM numbers(30) GROUP BY bucket WITH TOTALS ORDER BY bucket"; - - string[] names = []; - object?[] totals = []; - int calls = 0; - - var options = new ClickHouseTcpQueryOptions - { - Callbacks = new ClickHouseTcpQueryCallbacks - { - OnTotals = block => - { - calls++; - - // ColumnNames is computed and owned, so it is safe to keep; the columns are not. One row, and - // every column here is a scalar, so GetValue boxes a copy of the value rather than a view of - // the buffer. A composite column would need materializing on purpose. - names = [.. block.ColumnNames]; - totals = [.. block.Columns.Select(column => column.GetValue(0))]; - }, - }, - }; - - Console.WriteLine($" {sql}\n"); - await foreach (object[] row in client.QueryAsync(sql, options)) - { - Console.WriteLine($" row {string.Join(" ", row.Select(v => $"{v,8}"))}"); - } - - Console.WriteLine($" names {string.Join(" ", names.Select(n => $"{n,8}"))}"); - Console.WriteLine($" totals {string.Join(" ", totals.Select(v => $"{v,8}"))}"); - Console.WriteLine(); - Console.WriteLine($" Called {calls} time, after the last row: the server sends the totals block once the result"); - Console.WriteLine(" is complete. The shape is the query's own, so the aggregate columns hold the totals over"); - Console.WriteLine(" every group, and the grouping key holds a default rather than anything meaningful."); - Console.WriteLine(); - Console.WriteLine(" It arrives on its own packet, not as an extra row, so a caller reading rows never has to"); - Console.WriteLine(" filter it out — which is the difference from reading WITH TOTALS over HTTP in a row-shaped"); - Console.WriteLine(" format."); - } - - private static async Task TheExtremesRows(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. OnExtremes: two rows, the minimum and the maximum\n"); - - const string sql = "SELECT number AS n, toString(number) AS text FROM numbers(1, 12)"; - - var rows = new List(); - string[] names = []; - - var options = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["extremes"] = "1" }, - Callbacks = new ClickHouseTcpQueryCallbacks - { - OnExtremes = block => - { - names = [.. block.ColumnNames]; - for (int row = 0; row < block.RowCount; row++) - { - rows.Add([.. block.Columns.Select(column => column.GetValue(row))]); - } - }, - }, - }; - - int count = 0; - await foreach (object[] row in client.QueryAsync(sql, options)) - { - count++; - } - - Console.WriteLine($" {sql} ({count} rows)\n"); - Console.WriteLine($" {"",-8} {string.Join(" ", names.Select(n => $"{n,6}"))}"); - Console.WriteLine($" {"minimum",-8} {string.Join(" ", rows[0].Select(v => $"{v,6}"))}"); - Console.WriteLine($" {"maximum",-8} {string.Join(" ", rows[1].Select(v => $"{v,6}"))}"); - Console.WriteLine(); - Console.WriteLine(" Row 0 is the minimum and row 1 the maximum, per column and independently, so the pair is"); - Console.WriteLine(" not two rows of the result. Each column is compared in its own type's order, which for"); - Console.WriteLine($" the String column above is lexicographic — hence \"{rows[1][1]}\" as the maximum of 1..12."); - } - - private static async Task NothingFiresWhenThereIsNothingToSend(ClickHouseTcpClient client) - { - Console.WriteLine("\n6. A callback that never fires\n"); - - int log = 0; - int totals = 0; - int extremes = 0; - - var callbacks = new ClickHouseTcpQueryCallbacks - { - OnLog = _ => log++, - OnTotals = _ => totals++, - OnExtremes = _ => extremes++, - }; - - // Nothing turned on: no send_logs_level, no WITH TOTALS, no extremes. - await foreach (object[] row in client.QueryAsync( - "SELECT number FROM numbers(5)", new ClickHouseTcpQueryOptions { Callbacks = callbacks })) - { - } - - Console.WriteLine($" A plain query with all three set: OnLog {log}, OnTotals {totals}, OnExtremes {extremes} calls."); - - // All three turned on at once, on one query. - await foreach (object[] row in client.QueryAsync( - "SELECT number % 2 AS bucket, count() AS rows FROM numbers(20) GROUP BY bucket WITH TOTALS ORDER BY bucket", - new ClickHouseTcpQueryOptions - { - Settings = new Dictionary - { - ["send_logs_level"] = "debug", - ["extremes"] = "1", - }, - Callbacks = callbacks, - })) - { - } - - Console.WriteLine($" The same callbacks on a query with all three on: OnLog {log}, OnTotals {totals}, OnExtremes {extremes} calls."); - Console.WriteLine(); - Console.WriteLine(" There is no \"none arrived\" reading to look for, because there is no block to hand over,"); - Console.WriteLine(" so a caller that needs to know whether totals came keeps its own flag or counter — the"); - Console.WriteLine(" same shape Tcp_021 uses to show OnProfileInfo is called exactly once."); - } - - /// - /// Maps a server log line's Poco severity onto an level. Unknown numbers become - /// rather than something alarming. - /// - private static LogLevel ToLogLevel(sbyte priority) => priority switch - { - 1 => LogLevel.Critical, - 2 => LogLevel.Critical, - 3 => LogLevel.Error, - 4 => LogLevel.Warning, - 5 or 6 => LogLevel.Information, - 7 => LogLevel.Debug, - 8 or 9 => LogLevel.Trace, - _ => LogLevel.Information, - }; -} diff --git a/examples/Tcp/Observability/Tcp_029_HealthChecks.cs b/examples/Tcp/Observability/Tcp_029_HealthChecks.cs deleted file mode 100644 index 20ee613c7..000000000 --- a/examples/Tcp/Observability/Tcp_029_HealthChecks.cs +++ /dev/null @@ -1,226 +0,0 @@ -using System.Diagnostics; -using ClickHouse.Driver.Tcp; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; - -namespace ClickHouse.Driver.Examples; - -/// -/// PingAsync as a health check, wired into Microsoft.Extensions.Diagnostics.HealthChecks over an -/// AddClickHouseTcpDataSource registration. -/// -/// -/// A Ping is a protocol packet pair, not a statement: the server answers Pong and nothing is parsed, planned, -/// executed or written to system.query_log. That makes it the cheapest liveness probe this transport has — -/// the numbers are below — and it is the reason a native health check does not look like the HTTP one, which has -/// to send SELECT 1. -/// -/// -/// -/// Cheap also means narrow. A Pong says the socket reaches a ClickHouse that is still talking; it does not say a -/// query will succeed. Section 4 is the list of what it does and does not prove, which matters because a probe -/// that answers the wrong question is worse than none. -/// -/// -public static class TcpHealthChecks -{ - public static async Task Run() - { - await WhatAPingCosts(); - await ThreeEndpointsOneReport(); - WhatAPongProves(); - } - - private static async Task WhatAPingCosts() - { - Console.WriteLine("1. What a Ping costs against what SELECT 1 costs\n"); - - await using var client = ExampleConfig.CreateTcpClient(); - - // Both warmed up first, so neither measurement pays for the handshake. - await client.PingAsync(); - _ = await client.ExecuteScalarAsync("SELECT 1"); - - const int rounds = 50; - - var clock = Stopwatch.StartNew(); - for (int i = 0; i < rounds; i++) - { - await client.PingAsync(); - } - - double pings = clock.Elapsed.TotalMilliseconds; - - clock.Restart(); - for (int i = 0; i < rounds; i++) - { - _ = await client.ExecuteScalarAsync("SELECT 1"); - } - - double selects = clock.Elapsed.TotalMilliseconds; - - Console.WriteLine($" {$"{rounds} × PingAsync()",-38} {pings,7:0.0} ms {pings / rounds,5:0.00} ms each"); - Console.WriteLine($" {$"{rounds} × ExecuteScalarAsync(\"SELECT 1\")",-38} {selects,7:0.0} ms {selects / rounds,5:0.00} ms each"); - Console.WriteLine(); - Console.WriteLine(" Against a loopback server the difference is almost all server-side work, because the"); - Console.WriteLine(" round trip costs the same either way: SELECT 1 is parsed, planned, executed, and recorded"); - Console.WriteLine(" in system.query_log, and a Ping is one packet answered by one packet. Across a real"); - Console.WriteLine(" network the round trip dominates both and the gap narrows — the reason to prefer the Ping"); - Console.WriteLine(" is then that a probe every few seconds from every instance leaves no trace in the query"); - Console.WriteLine(" log to read past."); - Console.WriteLine(); - Console.WriteLine(" A Ping still needs a pool connection, so the first one after a cold start pays for a dial"); - Console.WriteLine(" and a handshake like any other operation."); - } - - private static async Task ThreeEndpointsOneReport() - { - Console.WriteLine("\n2. Registered as a health check, over three endpoints\n"); - - var services = new ServiceCollection(); - services.AddLogging(logging => logging.SetMinimumLevel(LogLevel.None)); - - // The endpoint that works, registered without a key, exactly as Tcp_003 does. - services.AddClickHouseTcpDataSource(ExampleConfig.TcpConnectionString); - - // One that nothing is listening on, so its dial is refused at once. - services.AddClickHouseTcpDataSource( - ExampleConfig.TcpBuilder().ToOptions() with { Port = 1, DialTimeout = TimeSpan.FromSeconds(2) }, - serviceKey: "unreachable"); - - // One whose pool holds a single connection, which a session below will pin — a healthy server that this - // process cannot reach a connection to. - services.AddClickHouseTcpDataSource( - ExampleConfig.TcpBuilder().ToOptions() with - { - MaxPoolSize = 1, - PoolTimeout = TimeSpan.FromMilliseconds(200), - }, - serviceKey: "saturated"); - - services.AddHealthChecks() - .AddClickHouseTcpPing("clickhouse") - .AddClickHouseTcpPing("clickhouse-unreachable", serviceKey: "unreachable") - .AddClickHouseTcpPing("clickhouse-saturated", serviceKey: "saturated"); - - await using ServiceProvider provider = services.BuildServiceProvider(); - - // Hold the saturated pool's only connection for the duration of the report. - var saturated = provider.GetRequiredKeyedService("saturated"); - await using IClickHouseTcpSession pinned = await saturated.OpenSessionAsync(); - - var health = provider.GetRequiredService(); - HealthReport report = await health.CheckHealthAsync(); - - Console.WriteLine($" Overall: {report.Status}, in {report.TotalDuration.TotalMilliseconds:0} ms\n"); - foreach ((string name, HealthReportEntry entry) in report.Entries.OrderBy(e => e.Key, StringComparer.Ordinal)) - { - Console.WriteLine($" {name,-24} {entry.Status,-9} {entry.Duration.TotalMilliseconds,6:0} ms"); - Console.WriteLine($" {Trim(entry.Description)}"); - foreach (KeyValuePair item in entry.Data) - { - Console.WriteLine($" {item.Key,-10} {item.Value}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" Three states, and the middle one is the point: a timeout says nothing about the server,"); - Console.WriteLine(" so reporting it as Unhealthy would take an instance out of rotation for being busy."); - Console.WriteLine(" Degraded is the honest answer. Note what the check cannot know: TimeoutException covers"); - Console.WriteLine(" waiting for a pool slot, dialing, and reading the pong alike, and the client exposes"); - Console.WriteLine(" nothing that separates them, so treat it as 'did not finish' and no more."); - Console.WriteLine(); - Console.WriteLine(" Every registration resolves the client the data source owns, so the check runs on the"); - Console.WriteLine(" application's own pool and measures the path a request would take. That is also why the"); - Console.WriteLine(" check must never dispose it (Tcp_003): the pool is shared, and the container owns it."); - Console.WriteLine(); - Console.WriteLine(" In ASP.NET Core the rest is MapHealthChecks(\"/health\") plus, if you want the endpoints"); - Console.WriteLine(" split, a predicate over the tags each registration carries."); - } - - private static void WhatAPongProves() - { - Console.WriteLine("\n3. What a Pong does and does not prove\n"); - Console.WriteLine(" It proves:"); - Console.WriteLine(" - a connection to the endpoint exists, or could be opened inside DialTimeout;"); - Console.WriteLine(" - the process answering it speaks the native protocol;"); - Console.WriteLine(" - if the pool had to dial, that the credentials were accepted — the handshake"); - Console.WriteLine(" authenticates, so a wrong password fails there rather than at the Ping;"); - Console.WriteLine(" - the server is not so stalled that it cannot answer a packet."); - Console.WriteLine(); - Console.WriteLine(" It does not prove:"); - Console.WriteLine(" - that a query will succeed. No table is read, no permission is checked, no memory or"); - Console.WriteLine(" concurrency limit is tested, and a server refusing queries still Pongs;"); - Console.WriteLine(" - that the database the client is configured for exists;"); - Console.WriteLine(" - that replication is caught up, or that any part of a cluster beyond this one node is"); - Console.WriteLine(" reachable;"); - Console.WriteLine(" - anything at all about credentials on a warm pool, where the Ping travels over a"); - Console.WriteLine(" connection whose handshake happened minutes ago."); - Console.WriteLine(); - Console.WriteLine(" So a Ping is a liveness probe. For readiness — should this instance take traffic — pick a"); - Console.WriteLine(" statement that touches what the service actually needs (SELECT 1, or a count against one"); - Console.WriteLine(" table), accept that it costs a query, and run it far less often."); - } - - private static string Trim(string? description) - => description is null ? "(no description)" - : description.Length <= 100 ? description - : description[..100] + "..."; - - /// - /// Registers a health check that Pings whichever registration names. Kept in - /// this example rather than shipped by the driver, so that the mapping from exception to - /// stays the application's decision. - /// - /// The health check builder. - /// The name the entry appears under in the report. - /// The keyed registration to check, or null for the unkeyed one. - /// The builder, for chaining. - private static IHealthChecksBuilder AddClickHouseTcpPing( - this IHealthChecksBuilder builder, - string name, - string? serviceKey = null) - => builder.Add(new HealthCheckRegistration( - name, - provider => new TcpPingHealthCheck(serviceKey is null - ? provider.GetRequiredService() - : provider.GetRequiredKeyedService(serviceKey)), - failureStatus: HealthStatus.Unhealthy, - tags: ["clickhouse", "native"])); - - /// - /// Takes the interface rather than the concrete client, and never disposes it: the container owns the pool. - /// - private sealed class TcpPingHealthCheck(IClickHouseTcpClient client) : IHealthCheck - { - public async Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default) - { - var clock = Stopwatch.StartNew(); - - // ToString() on the options is the only public rendering that resolves the port, and it is written to - // be safe to log — it leaves the password out. - var data = new Dictionary { ["endpoint"] = client.Options.ToString() }; - - try - { - await client.PingAsync(cancellationToken); - data["pong_ms"] = Math.Round(clock.Elapsed.TotalMilliseconds, 1); - return HealthCheckResult.Healthy("Pong", data); - } - catch (TimeoutException e) - { - // One type covers three deadlines: waiting for a pool slot, dialing, and reading the pong. None - // of them is proof the server is down, and none of them can be told apart here, so the honest - // report is that the check did not finish in time. - return HealthCheckResult.Degraded("Timed out before a pong", e, data); - } - catch (Exception e) when (e is not OperationCanceledException) - { - return new HealthCheckResult(context.Registration.FailureStatus, e.Message, e, data); - } - } - } -} diff --git a/examples/Tcp/Observability/Tcp_030_Testcontainers.cs b/examples/Tcp/Observability/Tcp_030_Testcontainers.cs deleted file mode 100644 index ecc09a09c..000000000 --- a/examples/Tcp/Observability/Tcp_030_Testcontainers.cs +++ /dev/null @@ -1,144 +0,0 @@ -using System.Diagnostics; -using System.Net.Sockets; -using ClickHouse.Driver.Tcp; -using DotNet.Testcontainers.Builders; -using Testcontainers.ClickHouse; - -namespace ClickHouse.Driver.Examples; - -/// -/// Running the native client against a throwaway ClickHouse from Testcontainers. The HTTP counterpart is -/// Testing_001_Testcontainers; everything here is about the one difference that matters, which is the port. -/// -/// -/// The container publishes 8123 and 9000 on two random host ports. GetConnectionString() describes the -/// HTTP one, so a native client cannot use it: take the native port from -/// GetMappedPublicPort(9000) and build the connection string yourself. -/// -/// -/// -/// Readiness is the second difference. The ClickHouse module's own wait strategy probes the HTTP interface, and -/// WithWaitStrategy replaces it rather than adding to it — so the strategy below asks for the HTTP -/// probe back and then also waits on 9000. A port check alone is not enough: measured on this image, a wait on -/// 9000 by itself reported ready about three seconds before the server would complete a native handshake, because -/// the port is bound before it is served. -/// -/// -/// -/// Both probes together are still not proof. On a busy machine the first native handshake can be refused after -/// each one has passed, because neither tests the native protocol — one tests the HTTP listener and the other -/// tests only that the port is bound. So the client waits on a handshake instead, which is the one check that -/// succeeds exactly when the client can work. Forced with a no-condition wait strategy, that took 43 attempts -/// over 4.4 seconds. -/// -/// -/// -/// This example starts its own server, so it is one of the few that does not take its endpoint from -/// ExampleConfig. It needs a working Docker daemon and will not run on a macOS runner. -/// -/// -public static class TcpTestcontainers -{ - /// Pinned, so the example tests one known server rather than whatever latest is today. - private const string Image = "clickhouse/clickhouse-server:25.12-alpine"; - - private const ushort NativePort = 9000; - private const ushort HttpPort = 8123; - - // The module creates this user from environment variables at startup. There is no GetUsername()/GetPassword() - // on the container, so setting them here is also how the test gets to know them. - private const string User = "example"; - private const string Password = "example"; - - public static async Task Run() - { - Console.WriteLine($"Starting {Image}. First run pulls the image, which takes a while.\n"); - - await using ClickHouseContainer container = new ClickHouseBuilder(Image) - .WithUsername(User) - .WithPassword(Password) - - // Both probes: the HTTP one is what says the server is really up, and the native one is what says the - // port this client dials is bound. Testcontainers polls both; nothing here sleeps. - .WithWaitStrategy(Wait.ForUnixContainer() - .UntilHttpRequestIsSucceeded(request => request.ForPath("/ping").ForPort(HttpPort)) - .UntilInternalTcpPortIsAvailable(NativePort)) - .Build(); - - await container.StartAsync(); - - Console.WriteLine($" Container started. GetConnectionString() describes the HTTP interface only:"); - Console.WriteLine($" {container.GetConnectionString()}"); - - // The native endpoint, assembled from the mapped port. ClickHouseTcpConnectionStringBuilder rather than a - // literal, so the escaping of anything in the password is not this example's problem. - var builder = new ClickHouseTcpConnectionStringBuilder - { - Host = container.Hostname, - Port = container.GetMappedPublicPort(NativePort), - Username = User, - Password = Password, - Database = "default", - }; - - Console.WriteLine($"\n Native endpoint, from GetMappedPublicPort({NativePort}):"); - Console.WriteLine($" Host={builder.Host};Port={builder.Port};Username={builder.Username};Database={builder.Database}"); - Console.WriteLine($" HTTP is on the other mapped port, {container.GetMappedPublicPort(HttpPort)} — the two are separate listeners."); - - await using var client = new ClickHouseTcpClient(builder.ToOptions()); - - // No wait strategy can prove the native protocol is accepting. Both probes above test something adjacent: - // /ping answers on the HTTP listener, and UntilInternalTcpPortIsAvailable only says the port is bound. - // A bound port is not an accepting server, so on a loaded machine the first handshake can still be - // refused. Waiting on a handshake is the only check that tests the thing being waited for. - ClickHouseTcpServerInfo info = await HandshakeWhenReady(client); - Console.WriteLine($"\n Handshaken: {info}, protocol revision {info.ProtocolRevision}, timezone {info.Timezone}"); - Console.WriteLine($" currentUser(): {await client.ExecuteScalarAsync("SELECT currentUser()")}"); - - await client.ExecuteAsync("CREATE TABLE probe (id UInt64, note String) ENGINE = MergeTree ORDER BY id"); - await client.InsertRowsAsync("INSERT INTO probe (id, note) VALUES", [[1UL, "from the native client"]]); - Console.WriteLine($" Inserted and read back: {await client.ExecuteScalarAsync("SELECT note FROM probe WHERE id = 1")}"); - - // No DROP TABLE: the container goes with the example, and so does everything in it. That is the whole - // point of a throwaway server, and it is why a test suite built on one needs no cleanup between tests - // beyond what the container's lifetime gives it. - Console.WriteLine("\n Nothing is dropped: DisposeAsync removes the container and the table with it."); - Console.WriteLine(" In a test project the container is started once for the run (an NUnit [OneTimeSetUp],"); - Console.WriteLine(" an xUnit fixture) and shared, because a handshake is cheap and a container start is not."); - } - - /// - /// Handshakes as soon as the server will, retrying while the native listener refuses the connection. - /// This is the readiness check a test fixture wants: it succeeds exactly when the client can work. - /// - private static async Task HandshakeWhenReady(ClickHouseTcpClient client) - { - var deadline = TimeSpan.FromSeconds(30); - var started = Stopwatch.StartNew(); - - for (int attempt = 1; ; attempt++) - { - try - { - ClickHouseTcpServerInfo info = await client.GetServerInfoAsync(); - - if (attempt > 1) - { - Console.WriteLine($"\n The first handshake was refused: it took {attempt} attempts over " + - $"{started.ElapsedMilliseconds} ms before the native listener accepted one, " + - "even though both wait strategies had already passed."); - } - - return info; - } - catch (ClickHouseTcpTransportException e) - when (e.InnerException is SocketException && started.Elapsed < deadline) - { - // Only a socket failure is worth retrying: the listener is not accepting yet. The same exception - // type also carries a TLS or DNS failure, which no amount of waiting fixes, so it is the inner - // exception that decides. A server exception would mean the server answered and rejected us. - await Task.Delay(100); - } - } - } -} diff --git a/examples/Tcp/README.md b/examples/Tcp/README.md index e1455652b..4a03d940a 100644 --- a/examples/Tcp/README.md +++ b/examples/Tcp/README.md @@ -1,75 +1,44 @@ # Native protocol examples -These use `ClickHouseTcpClient`, which speaks ClickHouse's native TCP protocol, rather than the -`ClickHouseClient` / `ClickHouseConnection` pair in [../Http](../Http) that speaks HTTP. The index of -every example, and how to run one, is in [the top-level README](../README.md). +These examples use `ClickHouseTcpClient` and ClickHouse's native TCP protocol. The HTTP examples use +`ClickHouseClient` or `ClickHouseConnection` instead. See the [example index](../README.md) to choose +and run an example. -## Before you run them +## Before you start -**They need port 9000, not 8123.** The two interfaces are separate listeners, so a server reachable -over HTTP is not necessarily reachable here: +The native protocol normally listens on port 9000. Port 8123 is for HTTP. ```bash docker run -d --name clickhouse-server -p 8123:8123 -p 9000:9000 clickhouse/clickhouse-server +dotnet run -- --tcp ``` -`dotnet run -- --tcp` runs only these, and checks the endpoint before starting. - -## The API is experimental - -`ClickHouseTcpClient`, `ClickHouseTcpDataSource`, the three `IClickHouseTcp*` interfaces -(`IClickHouseTcpClient`, `IClickHouseTcpOperations`, `IClickHouseTcpSession`) and the -`AddClickHouseTcpDataSource` overloads carry `[Experimental("CHTCP0001")]`, so touching one is a -compile error until you acknowledge that the surface may change in a future release. The types around -them — the options record, the connection string builder, `Block`, the column interfaces and the -exceptions — carry nothing, so they can be named without the suppression. - -```csharp -#pragma warning disable CHTCP0001 // The native protocol client's API is not yet stable. -``` - -Per file as above, or once for a project: +The native client API is experimental. Its main client, data source, session, operations interfaces, +and dependency-injection extensions produce warning `CHTCP0001`. This project acknowledges the +warning globally. In another project, add this setting while you evaluate the API: ```xml $(NoWarn);CHTCP0001 ``` -This examples project takes the project-wide route, which is why no file here opens with the pragma. - -## What the native client does not do +## Choose the right transport -Reach for the HTTP client instead when you need: +The native client provides: -- **A format other than Native** — the protocol carries columnar blocks, so there is no CSV, JSONEachRow - or Parquet ingestion or export, and no raw stream insert. -- **ADO.NET, and so any ORM.** There is no `DbConnection` implementation over this transport, so Dapper, - EF Core and linq2db do not work with it. -- **JWT or bearer authentication.** Username and password only. -- **Custom HTTP headers**, which have no equivalent on the wire. -- **A parameter type resolver, a parameter formatter, or a read value converter.** The native client - has no hook for any of the three; a parameter's type comes from the `{name:Type}` placeholder in - the query or from `ClickHouseTcpParameter.ClickHouseType`. -- **A per-query role or database.** HTTP's `QueryOptions` carries both; `ClickHouseTcpQueryOptions` - carries only `QueryId`, `Settings`, `Parameters` and `Callbacks`. Set the database on the client, - and change roles with `SET ROLE` inside a session. +- columnar block reads through `StreamAsync`; +- pinned sessions through `OpenSessionAsync`; +- progress, profile, log, totals, and extremes callbacks; +- native block compression and `QBit` plane access; +- W3C trace context propagation to ClickHouse. -Also worth knowing before you read a timestamp: a `DateTime`, `DateTime64`, `Time` or `Time64` column -reaches the **row** tier as the integer the wire carried, not as a calendar type, because that is the -value the server sent. `QueryAsync` into a POCO converts, and on the block tier the column -pattern-matches to `IDateTimeColumn` or `ITimeColumn`, which convert and report the timezone and -scale the column type declared. +Use the HTTP client when you need: -## What only the native client does +- ADO.NET or an ORM; +- CSV, JSONEachRow, Parquet, or raw stream input and output; +- bearer authentication or custom HTTP headers; +- custom parameter type resolution, parameter formatting, or read conversion; +- a per-query database or role. -- **Blocks and columns.** `StreamAsync` yields a `Block` whose typed columns expose `ReadOnlySpan` - over the server's own layout, so a read can avoid materializing rows at all — and a column read out - of a block re-inserts without being rebuilt. -- **Real sessions.** `OpenSessionAsync` pins one connection, so a temporary table or a `SET` survives - from one operation to the next without the caveats an HTTP session carries. -- **Progress, profile info and profile events while a query runs**, through - `ClickHouseTcpQueryCallbacks`, rather than as headers after the fact. -- **Block compression** on the wire, LZ4 by default. -- **Bit-plane access to `QBit` columns**, through `IQBitColumn`. -- **W3C trace context propagation.** The client sends the current `Activity`'s trace and span ids with - each query, so the spans the server records in `system.opentelemetry_span_log` join the same trace as - the caller's. The HTTP transport sends no `traceparent`. +One type detail is easy to miss: row reads return `DateTime`, `DateTime64`, `Time`, and `Time64` as +their wire integer values. `QueryAsync` converts mapped POCO properties, while block reads expose +`IDateTimeColumn` and `ITimeColumn` for typed conversion. diff --git a/examples/Tcp/Read/Tcp_001_ReadTiers.cs b/examples/Tcp/Read/Tcp_001_ReadTiers.cs new file mode 100644 index 000000000..95484c28f --- /dev/null +++ b/examples/Tcp/Read/Tcp_001_ReadTiers.cs @@ -0,0 +1,80 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Reads the same result as rows, objects, and columnar blocks. +public static class TcpReadTiers +{ + private const string TableName = "example_tcp_read_tiers"; + private static string Sql => $"SELECT id, city, temperature FROM {TableName} ORDER BY id"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + id UInt64, + city String, + temperature Float64 + ) + ENGINE = MergeTree + ORDER BY id + """); + + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, city, temperature) VALUES", + new[] + { + new object[] { 1UL, "Amsterdam", 17.5 }, + new object[] { 2UL, "Reykjavik", 9.5 }, + new object[] { 3UL, "Singapore", 28.0 }, + }); + + // Row reads need no model and expose the values in their wire representation. + Console.WriteLine("QueryAsync: flexible object[] rows"); + await foreach (object[] row in client.QueryAsync(Sql)) + { + Console.WriteLine($" {row[0]}: {row[1]}, {row[2]} °C"); + } + + // POCO reads map column names to properties and convert compatible values. + Console.WriteLine("QueryAsync: strongly typed objects"); + await foreach (Reading row in client.QueryAsync(Sql)) + { + Console.WriteLine($" {row.Id}: {row.City}, {row.Temperature} °C"); + } + + // Block reads are best for column-oriented work and avoid materializing each row. + Console.WriteLine("StreamAsync: columnar blocks"); + await foreach (Block block in client.StreamAsync(Sql)) + { + ReadOnlySpan temperatures = block.Column("temperature").Values; + double total = 0; + foreach (double temperature in temperatures) + { + total += temperature; + } + + Console.WriteLine($" {block.RowCount} rows; average {total / block.RowCount:0.0} °C"); + } + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } + + private sealed class Reading + { + public ulong Id { get; set; } + + public string City { get; set; } = string.Empty; + + public double Temperature { get; set; } + } +} diff --git a/examples/Tcp/Read/Tcp_002_BlocksAndColumns.cs b/examples/Tcp/Read/Tcp_002_BlocksAndColumns.cs new file mode 100644 index 000000000..fdf47e77e --- /dev/null +++ b/examples/Tcp/Read/Tcp_002_BlocksAndColumns.cs @@ -0,0 +1,76 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Reads typed columns and composite values from borrowed result blocks. +public static class TcpBlocksAndColumns +{ + private const string TableName = "example_tcp_blocks_and_columns"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + id UInt64, + sensor String, + readings Array(Float64), + captured_at DateTime64(3, 'UTC') + ) + ENGINE = MergeTree + ORDER BY id + """); + + var capturedAt = new DateTime(2026, 6, 1, 10, 0, 0, DateTimeKind.Utc); + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, sensor, readings, captured_at) VALUES", + new[] + { + new object[] { 1UL, "north", new[] { 0.5, 0.75 }, capturedAt }, + new object[] { 2UL, "south", new[] { 1.0, 1.25, 1.5 }, capturedAt.AddMinutes(1) }, + }); + + double[]? readingsToKeep = null; + + await foreach (Block block in client.StreamAsync( + $"SELECT id, sensor, readings, captured_at FROM {TableName} ORDER BY id")) + { + Console.WriteLine($"Block: {block.RowCount} rows, columns [{string.Join(", ", block.ColumnNames)}]"); + + IColumn ids = block.Column("id"); + IColumn sensors = block.Column("sensor"); + Console.WriteLine($"First row: {ids[0]}, {sensors[0]}"); + + if (block["readings"] is IArrayColumn arrays) + { + ReadOnlySpan values = arrays.InnerValues; + ReadOnlySpan offsets = arrays.Offsets; + Console.WriteLine($"Array storage: {values.Length} values, offsets " + + $"[{string.Join(", ", offsets.ToArray())}]"); + + // Row i occupies values[offsets[i]..offsets[i + 1]]. + // Copy data that must remain valid after this block is released. + readingsToKeep ??= values[offsets[0]..offsets[1]].ToArray(); + } + + if (block["captured_at"] is IDateTimeColumn timestamps) + { + DateTimeOffset value = timestamps.GetDateTimeOffset(0); + Console.WriteLine(value.ToString("O", CultureInfo.InvariantCulture)); + } + } + + Console.WriteLine($"Copied readings: [{string.Join(", ", readingsToKeep!)}]"); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } +} diff --git a/examples/Tcp/Read/Tcp_003_Parameters.cs b/examples/Tcp/Read/Tcp_003_Parameters.cs new file mode 100644 index 000000000..8297afb08 --- /dev/null +++ b/examples/Tcp/Read/Tcp_003_Parameters.cs @@ -0,0 +1,92 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Binds typed values and identifiers to native-protocol queries. +public static class TcpParameters +{ + private const string TableName = "example_tcp_parameters"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + id UInt64, + city String, + temperature Float64, + recorded_at DateTime('UTC') + ) + ENGINE = MergeTree + ORDER BY id + """); + + var recordedAt = new DateTime(2026, 6, 1, 12, 0, 0, DateTimeKind.Utc); + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, city, temperature, recorded_at) VALUES", + new[] + { + new object[] { 1UL, "Amsterdam", 21.0, recordedAt }, + new object[] { 2UL, "Reykjavik", 11.25, recordedAt }, + new object[] { 3UL, "Singapore", 31.75, recordedAt }, + }); + + var parameters = new ClickHouseTcpParameterCollection + { + { "city", "Amsterdam" }, + { "minimum", 18.0 }, + { "ids", new[] { 1UL, 2UL } }, + }; + + var options = new ClickHouseTcpQueryOptions { Parameters = parameters }; + + // Native queries use {name:Type}; @name placeholders are not rewritten. + string sql = $$""" + SELECT id, city, temperature + FROM {{TableName}} + WHERE city = {city:String} + OR (temperature >= {minimum:Float64} AND id IN {ids:Array(UInt64)}) + ORDER BY id + """; + + await foreach (object[] row in client.QueryAsync(sql, options)) + { + Console.WriteLine($"{row[0]}: {row[1]}, {row[2]} °C"); + } + + var identifierOptions = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection + { + { "table", TableName }, + { "column", "temperature" }, + }, + }; + + object maximum = await client.ExecuteScalarAsync( + "SELECT max({column:Identifier}) FROM {table:Identifier}", + identifierOptions); + Console.WriteLine($"Maximum temperature: {maximum}"); + + var timeOptions = new ClickHouseTcpQueryOptions + { + Parameters = new ClickHouseTcpParameterCollection { { "start", recordedAt } }, + }; + + // Declare a timezone when a DateTime or DateTimeOffset represents an instant. + object count = await client.ExecuteScalarAsync( + $"SELECT count() FROM {TableName} WHERE recorded_at >= {{start:DateTime('UTC')}}", + timeOptions); + Console.WriteLine($"Rows at or after {recordedAt:O}: {count}"); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } +} diff --git a/examples/Tcp/Read/Tcp_004_Poco.cs b/examples/Tcp/Read/Tcp_004_Poco.cs new file mode 100644 index 000000000..9a2eb4146 --- /dev/null +++ b/examples/Tcp/Read/Tcp_004_Poco.cs @@ -0,0 +1,81 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Inserts and reads strongly typed objects with custom column mappings. +public static class TcpPoco +{ + private const string TableName = "example_tcp_poco"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + id UInt64, + full_name String, + signal_count UInt32, + recorded_at DateTime('UTC'), + internal_notes String + ) + ENGINE = MergeTree + ORDER BY id + """); + + var rows = new[] + { + new Observation + { + Id = 1, + DisplayName = "Ada Lovelace", + SignalCount = 12, + RecordedAt = new DateTime(2026, 6, 1, 6, 0, 0, DateTimeKind.Utc), + }, + new Observation + { + Id = 2, + DisplayName = "Grace Hopper", + SignalCount = 7, + RecordedAt = new DateTime(2026, 6, 1, 9, 0, 0, DateTimeKind.Utc), + }, + }; + + await client.InsertRowsAsync( + $"INSERT INTO {TableName} (id, full_name, signal_count, recorded_at) VALUES", + rows); + + await foreach (Observation row in client.QueryAsync( + $"SELECT id, full_name, signal_count, recorded_at, internal_notes " + + $"FROM {TableName} ORDER BY id")) + { + Console.WriteLine( + $"{row.Id}: {row.DisplayName}, {row.SignalCount} signals at {row.RecordedAt:O}"); + } + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } + + private sealed class Observation + { + public ulong Id { get; set; } + + [ClickHouseTcpColumn(Name = "full_name")] + public string DisplayName { get; set; } = string.Empty; + + // signal_count matches SignalCount by ignoring case and underscores. + public uint SignalCount { get; set; } + + public DateTime RecordedAt { get; set; } + + [ClickHouseTcpNotMapped] + public string? InternalNotes { get; set; } + } +} diff --git a/examples/Tcp/Read/Tcp_005_ReadTiers.cs b/examples/Tcp/Read/Tcp_005_ReadTiers.cs deleted file mode 100644 index d97cae29d..000000000 --- a/examples/Tcp/Read/Tcp_005_ReadTiers.cs +++ /dev/null @@ -1,303 +0,0 @@ -using System.Globalization; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// The three ways the native client reads a result, run one after another over the same query: QueryAsync -/// (one object[] per row, value-type columns boxed), QueryAsync<T> (one POCO per row, values -/// converted) and StreamAsync (whole s, typed columns, no per-row work at all). -/// -/// -/// The tiers are not three spellings of one thing. They differ in what they allocate, and they differ in what a -/// timestamp looks like when it arrives — the row tier hands back the integer the wire carried, while the other -/// two convert it. Section 4 measures the first difference and section 1 shows the second. -/// -/// -public static class TcpReadTiers -{ - private const string TableName = "example_tcp_read_tiers"; - - // Rows for the allocation measurement in section 4. Large enough that the tiers separate, small enough that - // the whole example stays under a second. - private const int MeasuredRows = 200_000; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await Seed(client); - await RowTier(client); - await PocoTier(client); - await BlockTier(client); - await WhatEachCosts(client); - ShowTheChoice(); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped '{TableName}'"); - } - } - - private static async Task Seed(ClickHouseTcpClient client) - { - // recorded_at declares its timezone. A bare DateTime would take the server's, which is what the block - // tier reports as the column's TimeZone; naming UTC makes this example's output the same everywhere. - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - id UInt64, - city String, - temperature Float64, - recorded_at DateTime('UTC') - ) - ENGINE = MergeTree() - ORDER BY id"); - - var midnight = new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc); - - await client.InsertRowsAsync( - $"INSERT INTO {TableName} (id, city, temperature, recorded_at) VALUES", - new List - { - new object[] { 1UL, "Amsterdam", 17.5, midnight.AddHours(6) }, - new object[] { 2UL, "Amsterdam", 21.0, midnight.AddHours(14) }, - new object[] { 3UL, "Reykjavik", 9.5, midnight.AddHours(6) }, - new object[] { 4UL, "Reykjavik", 11.25, midnight.AddHours(14) }, - new object[] { 5UL, "Singapore", 28.0, midnight.AddHours(6) }, - new object[] { 6UL, "Singapore", 31.75, midnight.AddHours(14) }, - }); - - Console.WriteLine($"Seeded '{TableName}' with 6 rows (id UInt64, city String, temperature Float64, recorded_at DateTime('UTC'))"); - } - - private static string Sql => $"SELECT id, city, temperature, recorded_at FROM {TableName} ORDER BY id"; - - private static async Task RowTier(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. QueryAsync — one object[] per row, value-type columns boxed\n"); - Console.WriteLine(" ID City Temp recorded_at CLR types"); - Console.WriteLine(" -- --------- ----- ----------- ---------"); - - object[]? first = null; - - // Values arrive in the order the SELECT names them; there are no names on this tier. Each array is yours - // to keep, so collecting rows into a list is safe. - await foreach (object[] row in client.QueryAsync(Sql)) - { - first ??= row; - Console.WriteLine( - $" {(ulong)row[0],2} {(string)row[1],-9} {(double)row[2],5} {row[3],11} " + - string.Join(", ", row.Select(v => v.GetType().Name))); - } - - Console.WriteLine(); - Console.WriteLine(" The last column is the trap. recorded_at is a DateTime('UTC'), but a DateTime column"); - Console.WriteLine(" is stored as a count of epoch seconds and that count is what the box holds:"); - - // Reading a calendar value off this tier means converting the count by hand, which needs the timezone the - // column declared — and nothing on this tier reports it. The other two tiers do the conversion for you. - uint seconds = (uint)first![3]; - Console.WriteLine($" row[3] is {first[3].GetType().Name} = {seconds}"); - - try - { - _ = (DateTime)first[3]; - } - catch (InvalidCastException ex) - { - Console.WriteLine($" (DateTime)row[3] throws: {ex.Message}"); - } - - Console.WriteLine($" converted by hand: {DateTimeOffset.FromUnixTimeSeconds(seconds).UtcDateTime:yyyy-MM-dd HH:mm:ss} UTC"); - Console.WriteLine(" Date, DateTime64, Time and Time64 behave the same way. Read them through one of the"); - Console.WriteLine(" next two tiers."); - } - - private static async Task PocoTier(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. QueryAsync — one POCO per row, values converted\n"); - Console.WriteLine(" Each column fills the property of the same name (case- and underscore-insensitively,"); - Console.WriteLine(" so recorded_at reaches RecordedAt), converting to the property's type on the way:\n"); - Console.WriteLine(" ID City Temp RecordedAt (DateTime) Kind"); - Console.WriteLine(" -- --------- ----- --------------------- ----"); - - await foreach (Reading reading in client.QueryAsync(Sql)) - { - Console.WriteLine( - $" {reading.Id,2} {reading.City,-9} {reading.Temperature,5} " + - $"{reading.RecordedAt.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture),-21} {reading.RecordedAt.Kind}"); - } - - Console.WriteLine(); - Console.WriteLine(" Kind is Utc because the column declares UTC. A column in a zone with an offset yields"); - Console.WriteLine(" Kind=Unspecified — the wall-clock reading in that zone — so declare the property as a"); - Console.WriteLine(" DateTimeOffset when the offset matters."); - Console.WriteLine(); - Console.WriteLine(" A column no property maps to is skipped, and a property no column maps to keeps its"); - Console.WriteLine(" default. Tcp_008_Poco covers the mapping rules and the insert direction."); - } - - private static async Task BlockTier(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. StreamAsync — whole blocks, typed columns, nothing boxed\n"); - - await foreach (Block block in client.StreamAsync(Sql)) - { - Console.WriteLine($" Block of {block.RowCount} rows x {block.ColumnCount} columns: {string.Join(", ", block.ColumnNames)}"); - - // Bound once, outside any row loop: a name lookup is a scan of the block's columns. - IColumn temperature = block.Column("temperature"); - IColumn ids = block.Column("id"); - - // A span over the block's own buffer. Read into a local and iterate that; the property recomputes the - // span on every access, and it cannot be cached in a field because it is a ref struct. - ReadOnlySpan values = temperature.Values; - double total = 0; - foreach (double value in values) - { - total += value; - } - - Console.WriteLine($" temperature is {temperature.TypeName} -> ReadOnlySpan<{temperature.ElementType.Name}>, mean {total / values.Length:0.###}"); - Console.WriteLine($" id is {ids.TypeName} -> ReadOnlySpan<{ids.ElementType.Name}>, {ids.RowCount} values, first {ids[0]}"); - - // The typed view of a DateTime column is IColumn — the same count the row tier boxed. The - // calendar reading lives on IDateTimeColumn, which the column also implements. - IColumn recordedAt = block["recorded_at"]; - Console.WriteLine($" recorded_at is {recordedAt.TypeName} -> ReadOnlySpan<{recordedAt.ElementType.Name}>, the raw epoch seconds"); - - if (recordedAt is IDateTimeColumn instants) - { - Console.WriteLine($" ... and it pattern-matches to IDateTimeColumn: TimeZone {instants.TimeZone.Id}, Scale {instants.Scale}"); - Console.WriteLine($" GetDateTimeOffset(0) = {instants.GetDateTimeOffset(0).ToString("yyyy-MM-dd HH:mm:ss zzz", CultureInfo.InvariantCulture)}"); - - // ToDateTimeOffsets allocates, and the array it returns is the caller's: unlike Values it stays - // valid after the block is released. - DateTimeOffset[] all = instants.ToDateTimeOffsets(); - Console.WriteLine($" ToDateTimeOffsets() = {all.Length} instants, last {all[^1].ToString("HH:mm:ss zzz", CultureInfo.InvariantCulture)}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" A yielded block is borrowed: it is released when the loop advances. Copy out what has"); - Console.WriteLine(" to outlive the iteration. Tcp_006_BlocksAndColumns is the whole contract."); - } - - private static async Task WhatEachCosts(ClickHouseTcpClient client) - { - Console.WriteLine($"\n4. What each costs, summing one Float64 column over {MeasuredRows:N0} rows\n"); - - // Two numeric columns and no strings, so the measurement is the tier's own overhead rather than the cost - // of materializing values every tier has to materialize anyway. - string sql = $"SELECT number AS id, number * 0.5 AS temperature FROM system.numbers LIMIT {MeasuredRows}"; - - // Warm up: the first read of a result compiles the POCO plan and grows the pooled buffers, and charging - // that to whichever tier ran first would be the whole difference at this size. - await SumWithRows(client, sql); - await SumWithPoco(client, sql); - await SumWithBlocks(client, sql); - - await Measure("QueryAsync object[] per row, 2 boxes per row", () => SumWithRows(client, sql)); - await Measure("QueryAsync one POCO per row, no boxing", () => SumWithPoco(client, sql)); - await Measure("StreamAsync spans over the block's buffers", () => SumWithBlocks(client, sql)); - - Console.WriteLine(); - Console.WriteLine(" Allocation is measured process-wide (GC.GetTotalAllocatedBytes), so it includes the"); - Console.WriteLine(" client's own read buffers — which is why the block tier is not zero rather than why it"); - Console.WriteLine(" is small. Absolute numbers move with the machine; the ratio is the point."); - Console.WriteLine(); - Console.WriteLine(" A String column narrows the gap, because every tier materializes one string per value."); - } - - private static async Task Measure(string label, Func> read) - { - GC.Collect(); - GC.WaitForPendingFinalizers(); - - long before = GC.GetTotalAllocatedBytes(precise: true); - var started = System.Diagnostics.Stopwatch.StartNew(); - double sum = await read(); - started.Stop(); - long allocated = GC.GetTotalAllocatedBytes(precise: true) - before; - - Console.WriteLine($" {label,-52} {allocated / 1024.0 / 1024.0,7:0.00} MB {started.ElapsedMilliseconds,4} ms (sum {sum:0})"); - } - - private static async Task SumWithRows(ClickHouseTcpClient client, string sql) - { - double sum = 0; - await foreach (object[] row in client.QueryAsync(sql)) - { - sum += (double)row[1]; - } - - return sum; - } - - private static async Task SumWithPoco(ClickHouseTcpClient client, string sql) - { - double sum = 0; - await foreach (Reading reading in client.QueryAsync(sql)) - { - sum += reading.Temperature; - } - - return sum; - } - - private static async Task SumWithBlocks(ClickHouseTcpClient client, string sql) - { - double sum = 0; - await foreach (Block block in client.StreamAsync(sql)) - { - ReadOnlySpan values = block.Column("temperature").Values; - for (int i = 0; i < values.Length; i++) - { - sum += values[i]; - } - } - - return sum; - } - - private static void ShowTheChoice() - { - Console.WriteLine("\n5. Which to pick\n"); - Console.WriteLine(" QueryAsync A result whose shape you do not know at compile time, or a few rows"); - Console.WriteLine(" where the boxing does not matter. No names — pair it with"); - Console.WriteLine(" Block.ColumnNames if you need them. Date and time columns arrive raw."); - Console.WriteLine(); - Console.WriteLine(" QueryAsync The default for application code. One object per row instead of an"); - Console.WriteLine(" array plus a box per value-type column, values converted to the"); - Console.WriteLine(" property's type, and each row owns its values, so a row can be kept."); - Console.WriteLine(); - Console.WriteLine(" StreamAsync Aggregating, scanning, or handing a column to something that wants a"); - Console.WriteLine(" span. Nothing is materialized per row, and a column read out of one"); - Console.WriteLine(" block re-inserts without being rebuilt. The cost is the borrowing"); - Console.WriteLine(" contract: nothing may outlive the iteration unless you copy it."); - Console.WriteLine(); - Console.WriteLine(" All three hold a connection until the enumeration ends, so read to the end (or stop"); - Console.WriteLine(" with a break, which tells the server the result is abandoned and drops the connection"); - Console.WriteLine(" rather than returning it to the pool)."); - } - - // One property per column. String is initialized because the project enables nullable reference types and - // the materializer assigns every mapped property anyway. - private sealed class Reading - { - public ulong Id { get; set; } - - public string City { get; set; } = string.Empty; - - public double Temperature { get; set; } - - // The DateTime('UTC') column's epoch-second count, converted with the column's timezone. Declare this as - // a DateTimeOffset instead to keep the offset. - public DateTime RecordedAt { get; set; } - } -} diff --git a/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs b/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs deleted file mode 100644 index e4ea319a3..000000000 --- a/examples/Tcp/Read/Tcp_006_BlocksAndColumns.cs +++ /dev/null @@ -1,425 +0,0 @@ -using System.Globalization; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// The block tier in depth: what StreamAsync yields, how a is addressed -/// (ColumnNames, the two indexers, TryGetColumn, the typed Column<T>), what an -/// reports about itself, and how its values are read as a . -/// -/// -/// Section 7 is the part to read twice. A yielded block is borrowed: its storage is returned to a pool -/// when the iteration moves on, so a column, a span, or the block itself is invalid the moment the loop -/// advances. Everything else here is convenience; this one is correctness. -/// -/// -public static class TcpBlocksAndColumns -{ - private const string TableName = "example_tcp_blocks_and_columns"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await Seed(client); - await OneResultManyBlocks(client); - await AddressingAColumn(client); - await WhatAColumnReports(client); - await ValuesAsSpans(client); - await DateAndTimeColumns(client); - await ArrayColumns(client); - await TheBorrowingContract(client); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped '{TableName}'"); - } - } - - private static async Task Seed(ClickHouseTcpClient client) - { - // captured_at names a zone with an offset and a daylight-saving rule, so the block tier has something to - // report; uptime is a Time, which is a count from midnight and has no zone at all. - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - id UInt64, - sensor String, - voltage Float64, - readings Array(Float64), - captured_at DateTime64(3, 'Europe/Amsterdam'), - uptime Time - ) - ENGINE = MergeTree() - ORDER BY id"); - - var baseline = new DateTime(2026, 6, 1, 10, 0, 0, DateTimeKind.Utc); - - await client.InsertRowsAsync( - $"INSERT INTO {TableName} (id, sensor, voltage, readings, captured_at, uptime) VALUES", - new List - { - new object[] { 1UL, "north", 3.31, new[] { 0.5, 0.75, 1.0 }, baseline.AddMilliseconds(125), TimeSpan.FromMinutes(90) }, - new object[] { 2UL, "north", 3.28, new[] { 1.25, 1.5 }, baseline.AddMilliseconds(250), TimeSpan.FromMinutes(150) }, - new object[] { 3UL, "south", 3.35, Array.Empty(), baseline.AddMilliseconds(375), TimeSpan.FromMinutes(210) }, - new object[] { 4UL, "south", 3.30, new[] { 2.0 }, baseline.AddMilliseconds(500), TimeSpan.FromMinutes(270) }, - new object[] { 5UL, "west", 3.22, new[] { 2.25, 2.5, 2.75, 3.0 }, baseline.AddMilliseconds(625), TimeSpan.FromMinutes(330) }, - new object[] { 6UL, "west", 3.40, new[] { 3.25 }, baseline.AddMilliseconds(750), TimeSpan.FromMinutes(390) }, - }); - - Console.WriteLine($"Seeded '{TableName}' with 6 rows:"); - Console.WriteLine(" id UInt64, sensor String, voltage Float64, readings Array(Float64),"); - Console.WriteLine(" captured_at DateTime64(3, 'Europe/Amsterdam'), uptime Time"); - } - - private static async Task OneResultManyBlocks(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. One result is a sequence of blocks\n"); - Console.WriteLine(" How many, and how tall, is the server's decision. This table's six rows fit in one"); - Console.WriteLine(" granule, so they arrive together:\n"); - Console.WriteLine(" Block Rows Columns Name"); - Console.WriteLine(" ----- ---- ------- ----"); - await ShowShapes(client, $"SELECT id, sensor FROM {TableName}", null); - - // A generator does honour max_block_size row for row, which a six-row MergeTree read does not: the part is - // read whole and the setting only caps it. - var capped = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["max_block_size"] = "3" }, - }; - - Console.WriteLine("\n The same shape of query over a generator, with max_block_size = 3, splits:\n"); - Console.WriteLine(" Block Rows Columns Name"); - Console.WriteLine(" ----- ---- ------- ----"); - await ShowShapes(client, "SELECT number, toString(number) AS text FROM system.numbers LIMIT 8", capped); - - Console.WriteLine(); - Console.WriteLine(" A result block carries no name. A named block is how the server labels the extras a"); - Console.WriteLine(" query can produce — WITH TOTALS, extremes — which reach a caller through"); - Console.WriteLine(" ClickHouseTcpQueryOptions.Callbacks rather than through this stream."); - Console.WriteLine(); - Console.WriteLine(" So write the loop for any number of blocks of any height, and never for one."); - } - - private static async Task ShowShapes(ClickHouseTcpClient client, string sql, ClickHouseTcpQueryOptions? options) - { - int index = 0; - await foreach (Block block in client.StreamAsync(sql, options)) - { - Console.WriteLine($" {++index,5} {block.RowCount,4} {block.ColumnCount,7} {(block.Name.Length == 0 ? "(empty)" : block.Name)}"); - } - } - - private static async Task AddressingAColumn(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. Addressing a column\n"); - - await foreach (Block block in client.StreamAsync(Sql)) - { - // Owned, unlike the columns: computed once, cached, and safe to keep after the block is released. - Console.WriteLine($" block.ColumnNames = [{string.Join(", ", block.ColumnNames)}]"); - Console.WriteLine($" block.ColumnCount = {block.ColumnCount}, block.RowCount = {block.RowCount}"); - Console.WriteLine($" block[0] = '{block[0].Name}' by position"); - Console.WriteLine($" block[\"sensor\"] = '{block["sensor"].Name}' by name — ordinal and case-sensitive, like ClickHouse itself"); - - // The name lookup is a scan of the block's columns, so bind a column once and then loop over rows, - // never the other way round. - Console.WriteLine($" TryGetColumn(\"sensor\", out _) = {block.TryGetColumn("sensor", out _)}"); - Console.WriteLine($" TryGetColumn(\"Sensor\", out _) = {block.TryGetColumn("Sensor", out _)} (capital S is a different name)"); - - try - { - _ = block["missing"]; - } - catch (ArgumentException ex) - { - Console.WriteLine($" block[\"missing\"] throws: {ex.Message.Split(" (Parameter")[0]}"); - } - - // The typed overload is the same lookup plus a cast to IColumn, which is where the values live. - IColumn voltage = block.Column("voltage"); - Console.WriteLine($" block.Column(\"voltage\") = IColumn over '{voltage.TypeName}'"); - - try - { - _ = block.Column("captured_at"); - } - catch (InvalidCastException ex) - { - Console.WriteLine($" block.Column(\"captured_at\") throws: {ex.Message}"); - Console.WriteLine(" T must be the type the column's values are stored as, not the type you want them in."); - } - - // The first block is enough for the sections that follow. Breaking out is allowed: the client tells the - // server the result is abandoned, and drops that connection instead of returning it to the pool. - } - } - - private static async Task WhatAColumnReports(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. What a column reports about itself\n"); - Console.WriteLine(" Name TypeName ElementType Rows Extra interface"); - Console.WriteLine(" ----------- ----------------------------------- ----------- ---- ---------------"); - - await foreach (Block block in client.StreamAsync(Sql)) - { - foreach (IColumn column in block.Columns) - { - Console.WriteLine( - $" {column.Name,-11} {column.TypeName,-35} {Describe(column.ElementType),-11} {column.RowCount,4} {ExtraInterface(column)}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" TypeName is the header text the server sent, so it is the type as ClickHouse spells it."); - Console.WriteLine(" ElementType is the T of the column's IColumn — what Values hands back. Where those"); - Console.WriteLine(" two disagree in kind, a second interface bridges them (sections 5 and 6)."); - } - - private static async Task ValuesAsSpans(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. Values, as a span over the server's own layout\n"); - - await foreach (Block block in client.StreamAsync(Sql)) - { - IColumn ids = block.Column("id"); - IColumn voltage = block.Column("voltage"); - IColumn sensor = block.Column("sensor"); - - // Read the span into a local: the property recomputes it on every access, and being a ref struct it - // cannot be stored in a field. Do not let it escape this iteration. - ReadOnlySpan volts = voltage.Values; - - double min = double.MaxValue; - double max = double.MinValue; - for (int i = 0; i < volts.Length; i++) - { - min = Math.Min(min, volts[i]); - max = Math.Max(max, volts[i]); - } - - Console.WriteLine($" voltage.Values is ReadOnlySpan of {volts.Length}: min {min}, max {max} — no allocation, no boxing"); - Console.WriteLine($" id.Values is ReadOnlySpan of {ids.Values.Length}: {string.Join(", ", ids.Values.ToArray())}"); - Console.WriteLine($" voltage[2] = {voltage[2]} (the indexer, for one value)"); - Console.WriteLine($" block[\"voltage\"].GetValue(2) = {block["voltage"].GetValue(2)} boxed — the untyped escape hatch"); - Console.WriteLine(); - Console.WriteLine($" A String column is a span too, of references: sensor.Values = [{string.Join(", ", sensor.Values.ToArray())}]"); - Console.WriteLine(" Reading it decodes one string per value, so the block tier saves less on String than"); - Console.WriteLine(" on a fixed-width type. It still saves the object[] and the boxes."); - } - } - - private static async Task DateAndTimeColumns(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. Date and time columns: a count, plus an interface that reads it\n"); - Console.WriteLine(" These types are stored as a plain integer, so IColumn hands back that integer — the"); - Console.WriteLine(" layout the wire carried, at no conversion cost. Turning it into a calendar value needs"); - Console.WriteLine(" the column's timezone and scale, which only these two interfaces report.\n"); - - await foreach (Block block in client.StreamAsync(Sql)) - { - IColumn capturedAt = block["captured_at"]; - Console.WriteLine($" captured_at {capturedAt.TypeName}"); - Console.WriteLine($" as IColumn: {string.Join(", ", block.Column("captured_at").Values[..3].ToArray())}, ... (milliseconds since the epoch)"); - - if (capturedAt is IDateTimeColumn instants) - { - Console.WriteLine($" as IDateTimeColumn: TimeZone {instants.TimeZone.Id}, Scale {instants.Scale}"); - Console.WriteLine($" GetDateTimeOffset(0) = {Format(instants.GetDateTimeOffset(0))}"); - Console.WriteLine($" GetDateTimeOffset(5) = {Format(instants.GetDateTimeOffset(5))}"); - - // Allocates one array, and that array is the caller's: it stays valid after the block is gone. - DateTimeOffset[] all = instants.ToDateTimeOffsets(); - Console.WriteLine($" ToDateTimeOffsets() = {all.Length} instants, and the array outlives the block"); - Console.WriteLine(" The +02:00 offset is the zone's, in June. The same column read in January"); - Console.WriteLine(" would report +01:00, which is why the timezone and not a fixed offset is what"); - Console.WriteLine(" the interface exposes."); - } - - IColumn uptime = block["uptime"]; - Console.WriteLine($" uptime {uptime.TypeName}"); - Console.WriteLine($" as IColumn: {string.Join(", ", block.Column("uptime").Values[..3].ToArray())}, ... (seconds from midnight)"); - - if (uptime is ITimeColumn times) - { - Console.WriteLine($" as ITimeColumn: Scale {times.Scale}, no timezone — a Time is a time of day, not an instant"); - Console.WriteLine($" GetTimeSpan(0) = {times.GetTimeSpan(0)}"); - Console.WriteLine($" ToTimeSpans() = {string.Join(", ", times.ToTimeSpans().Take(3))}, ... (also caller-owned)"); - Console.WriteLine(" The count is signed and is not clamped to one day, so a TimeSpan here can be"); - Console.WriteLine(" negative or longer than 24 hours."); - } - } - - Console.WriteLine(); - Console.WriteLine(" Pattern-match rather than test the type name: DateTime and DateTime64 both give an"); - Console.WriteLine(" IDateTimeColumn, Time and Time64 both give an ITimeColumn, and neither interface is"); - Console.WriteLine(" generic, so one branch handles both widths."); - Console.WriteLine(); - Console.WriteLine(" A Nullable(DateTime) does not match, though — the wrapper is a column in its own right"); - Console.WriteLine(" and it is the wrapped column that reads the calendar. Go through INullableColumn:\n"); - - // The inner column holds one entry per row, with a placeholder where the row is null, so the null map and - // the inner column are indexed by the same row number. - await foreach (Block block in client.StreamAsync( - "SELECT if(number = 1, NULL, toDateTime(1780308000 + number, 'UTC')) AS maybe_at FROM system.numbers LIMIT 3")) - { - IColumn maybeAt = block["maybe_at"]; - Console.WriteLine($" {maybeAt.TypeName}, ElementType {Describe(maybeAt.ElementType)}"); - Console.WriteLine($" is IDateTimeColumn: {maybeAt is IDateTimeColumn,-5} (the Nullable wrapper itself)"); - - // INullableColumn because a DateTime is stored as uint: the wrapper's T is the inner storage - // type, so reaching Inner means knowing that type. - if (maybeAt is INullableColumn nullable && nullable.Inner is IDateTimeColumn inner) - { - Console.WriteLine($" is IDateTimeColumn: {true,-5} (INullableColumn.Inner)"); - ReadOnlySpan nulls = nullable.NullMap; - for (int row = 0; row < maybeAt.RowCount; row++) - { - string reading = nulls[row] != 0 ? "NULL" : Format(inner.GetDateTimeOffset(row)); - Console.WriteLine($" row {row}: NullMap {nulls[row]} -> {reading}"); - } - } - } - } - - private static async Task ArrayColumns(ClickHouseTcpClient client) - { - Console.WriteLine("\n6. An Array(T) column has two views, and they cost different things\n"); - - await foreach (Block block in client.StreamAsync(Sql)) - { - IColumn readings = block["readings"]; - Console.WriteLine($" readings is {readings.TypeName}, ElementType {Describe(readings.ElementType)}"); - - if (readings is IArrayColumn arrays) - { - // The wire layout: every row's elements end to end, plus one offset per row boundary. Both spans - // are borrowed, and this is the view that costs nothing to produce. - ReadOnlySpan flat = arrays.InnerValues; - ReadOnlySpan offsets = arrays.Offsets; - - Console.WriteLine(); - Console.WriteLine($" Borrowed view — InnerValues + Offsets, no allocation at all:"); - Console.WriteLine($" InnerValues ({flat.Length} elements) = {string.Join(", ", flat.ToArray())}"); - Console.WriteLine($" Offsets ({offsets.Length} entries, one more than the rows) = {string.Join(", ", offsets.ToArray())}"); - Console.WriteLine(" Row i is InnerValues.Slice(Offsets[i], Offsets[i + 1] - Offsets[i]):"); - - for (int row = 0; row < readings.RowCount; row++) - { - ReadOnlySpan slice = flat.Slice(offsets[row], offsets[row + 1] - offsets[row]); - double sum = 0; - foreach (double value in slice) - { - sum += value; - } - - Console.WriteLine($" row {row}: {slice.Length} element(s), sum {sum}"); - } - - Console.WriteLine(); - Console.WriteLine($" Inner is that same flat run as a column rather than a span — IColumn<{Describe(arrays.Inner.ElementType)}> here."); - Console.WriteLine(" Use it for an Array(Tuple(...)) or an Array(Array(T)), where the inner column"); - Console.WriteLine(" pattern-matches to ITupleColumn or IArrayColumn in turn, so a nested composite"); - Console.WriteLine(" can be walked all the way down without materializing a level."); - } - - // The other view. Each row is copied into a fresh double[], so these arrays are the caller's and stay - // valid after the block is released — at one allocation per row. - Console.WriteLine(); - Console.WriteLine(" Allocating view — Values and the indexer materialize one double[] per row:"); - IColumn rows = block.Column("readings"); - Console.WriteLine($" rows[4] = [{string.Join(", ", rows[4])}] (the indexer: one double[], allocated here)"); - Console.WriteLine($" Values[0] = [{string.Join(", ", rows.Values[0])}] (Values: every row's array, built at once)"); - Console.WriteLine(" Those arrays outlive the block. The span holding them does not, being a span."); - Console.WriteLine(" So prefer the indexer when only a few rows out of a tall block are wanted."); - } - - Console.WriteLine(); - Console.WriteLine(" The same split runs through the other composites: Map, Tuple, Nested, Nullable and"); - Console.WriteLine(" LowCardinality each expose a borrowed columnar view plus a materializing one."); - } - - private static async Task TheBorrowingContract(ClickHouseTcpClient client) - { - Console.WriteLine("\n7. The borrowing contract\n"); - Console.WriteLine(" Valid only for the current iteration — released when the loop advances, you stop"); - Console.WriteLine(" enumerating, or the enumerator is disposed:"); - Console.WriteLine(" the Block, every IColumn on it, IColumn.Values,"); - Console.WriteLine(" IArrayColumn.InnerValues / Offsets / Inner, INullableColumn.NullMap / Inner,"); - Console.WriteLine(" and the other composites' views."); - Console.WriteLine(); - Console.WriteLine(" Yours to keep:"); - Console.WriteLine(" Block.ColumnNames, a string or a struct value you read out,"); - Console.WriteLine(" the per-row arrays from an Array(T) column's Values or indexer,"); - Console.WriteLine(" IDateTimeColumn.ToDateTimeOffsets() and ITimeColumn.ToTimeSpans(),"); - Console.WriteLine(" and anything you copy: Values.ToArray(), a slice's ToArray()."); - Console.WriteLine(); - Console.WriteLine(" Do not dispose a yielded block. Block is IDisposable because the reader that produced"); - Console.WriteLine(" it disposes it; doing so yourself returns pooled storage the reader still manages."); - Console.WriteLine(); - Console.WriteLine(" So the shape of a correct loop is: read, aggregate, or copy — inside the body.\n"); - - // The aggregate is a value type, and the copies are arrays of our own, so both are safe to use after the - // enumeration has finished and every block has been released. - long rowsSeen = 0; - double voltageTotal = 0; - var strongestSensor = string.Empty; - double strongest = double.MinValue; - double[]? firstRowReadings = null; - - await foreach (Block block in client.StreamAsync(Sql)) - { - IColumn sensors = block.Column("sensor"); - ReadOnlySpan volts = block.Column("voltage").Values; - - for (int row = 0; row < block.RowCount; row++) - { - rowsSeen++; - voltageTotal += volts[row]; - if (volts[row] > strongest) - { - strongest = volts[row]; - - // A string read out of the block is a reference to an object the block does not own, so - // holding it is fine. A span is not. - strongestSensor = sensors[row]; - } - } - - // ToArray inside the loop is the copy. Taking the span out of the loop instead would be reading - // storage the next iteration has already handed back to the pool. - firstRowReadings ??= block.Column("readings")[0].ToArray(); - } - - Console.WriteLine($" After the loop, from copies only: {rowsSeen} rows, mean voltage {voltageTotal / rowsSeen:0.####},"); - Console.WriteLine($" highest on sensor '{strongestSensor}' at {strongest}, first row's readings [{string.Join(", ", firstRowReadings!)}]"); - } - - private static string Sql - => $"SELECT id, sensor, voltage, readings, captured_at, uptime FROM {TableName} ORDER BY id"; - - private static string Format(DateTimeOffset value) - => value.ToString("yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture); - - private static string Describe(Type type) => type switch - { - _ when type == typeof(double[]) => "double[]", - _ when type == typeof(uint?) => "uint?", - _ => type.Name, - }; - - // Which of the block tier's extra read surfaces a column offers, found by pattern-matching rather than by - // reading TypeName. - private static string ExtraInterface(IColumn column) => column switch - { - IDateTimeColumn => "IDateTimeColumn", - ITimeColumn => "ITimeColumn", - IArrayColumn => "IArrayColumn", - _ => "-", - }; -} diff --git a/examples/Tcp/Read/Tcp_007_Parameters.cs b/examples/Tcp/Read/Tcp_007_Parameters.cs deleted file mode 100644 index f5afefcb4..000000000 --- a/examples/Tcp/Read/Tcp_007_Parameters.cs +++ /dev/null @@ -1,409 +0,0 @@ -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// Binding values into a query with and -/// ClickHouseTcpQueryOptions.Parameters — and the three ways it goes wrong, which are worth more of your -/// attention than the happy path. -/// -/// -/// The query text must carry each parameter's type ({id:Int32}): there is no @name rewriting on this -/// transport. A value that names an instant is refused unless the placeholder declares a timezone. And a -/// parameter named after a server setting is applied as that setting rather than bound, which fails with an error -/// that names neither. -/// -/// -public static class TcpParameters -{ - private const string TableName = "example_tcp_parameters"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await Seed(client); - await BindingValues(client); - await TheCollection(client); - await NoAtNameRewriting(client); - await Identifiers(client); - await InstantsNeedATimezone(client); - await NamesThatCollideWithSettings(client); - ShowWhatIsAbsent(); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped '{TableName}'"); - } - } - - private static async Task Seed(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - id UInt64, - city String, - temperature Float64, - recorded_at DateTime('UTC') - ) - ENGINE = MergeTree() - ORDER BY id"); - - var midnight = new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc); - - await client.InsertRowsAsync( - $"INSERT INTO {TableName} (id, city, temperature, recorded_at) VALUES", - new List - { - new object[] { 1UL, "Amsterdam", 17.5, midnight.AddHours(6) }, - new object[] { 2UL, "Amsterdam", 21.0, midnight.AddHours(14) }, - new object[] { 3UL, "Reykjavik", 9.5, midnight.AddHours(6) }, - new object[] { 4UL, "Reykjavik", 11.25, midnight.AddHours(14) }, - new object[] { 5UL, "Singapore", 28.0, midnight.AddHours(6) }, - new object[] { 6UL, "Singapore", 31.75, midnight.AddHours(14) }, - }); - - Console.WriteLine($"Seeded '{TableName}' with 6 rows (id, city, temperature, recorded_at DateTime('UTC'))"); - - // Parameters travel in the Query packet's settings list, which is why they need a protocol revision that - // knows about them. An older server rejects the query rather than run it unparameterized. - ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); - Console.WriteLine($"Server protocol revision {server.ProtocolRevision}; parameters need 54459 or above"); - } - - private static async Task BindingValues(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. Binding values\n"); - - // Names, not positions. The collection keeps insertion order, but the query refers to each by name. - var parameters = new ClickHouseTcpParameterCollection(); - parameters.Add("city", "Amsterdam"); - parameters.Add("floor", 18.0); - parameters.Add("wanted", new[] { 1UL, 2UL, 5UL }); - - // Every placeholder states its type. That is what the server parses the value as, and what the client - // formats it as, so the two cannot disagree. - string sql = $@" - SELECT id, city, temperature - FROM {TableName} - WHERE (city = {{city:String}} OR temperature >= {{floor:Float64}}) - AND id IN {{wanted:Array(UInt64)}} - ORDER BY id"; - - Console.WriteLine(" SELECT ... WHERE (city = {city:String} OR temperature >= {floor:Float64})"); - Console.WriteLine(" AND id IN {wanted:Array(UInt64)}"); - Console.WriteLine(" city='Amsterdam', floor=18.0, wanted=[1, 2, 5]\n"); - - await foreach (object[] row in client.QueryAsync(sql, new ClickHouseTcpQueryOptions { Parameters = parameters })) - { - Console.WriteLine($" id {row[0],2} {(string)row[1],-9} {row[2]}"); - } - - Console.WriteLine(); - Console.WriteLine(" A collection works on any operation that takes ClickHouseTcpQueryOptions, so the same"); - Console.WriteLine(" parameters bind on ExecuteAsync, ExecuteScalarAsync, QueryAsync, StreamAsync and"); - Console.WriteLine(" InsertAsync — an INSERT ... SELECT can be parameterized too."); - - object count = await client.ExecuteScalarAsync( - $"SELECT count() FROM {TableName} WHERE city = {{city:String}}", - new ClickHouseTcpQueryOptions { Parameters = new ClickHouseTcpParameterCollection { { "city", "Reykjavik" } } }); - Console.WriteLine($" ExecuteScalarAsync(count() WHERE city = {{city:String}}) with city='Reykjavik' = {count}"); - } - - private static async Task TheCollection(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. The collection itself\n"); - - var parameters = new ClickHouseTcpParameterCollection - { - { "city", "Singapore" }, - { "floor", 30.0 }, - }; - - Console.WriteLine($" Count {parameters.Count}"); - Console.WriteLine($" Contains(\"city\") {parameters.Contains("city")}"); - Console.WriteLine($" Contains(\"City\") {parameters.Contains("City")} (names are ordinal, like the server's)"); - Console.WriteLine($" this[\"floor\"].Value {parameters["floor"].Value}"); - Console.WriteLine($" TryGetValue(\"nope\", out _) {parameters.TryGetValue("nope", out _)}"); - Console.WriteLine($" enumerates in order {string.Join(", ", parameters.Select(p => p.Name))}"); - - // The wire format is a name/value list, so a repeated name has no meaning and is refused rather than - // silently taking one of the two values. - try - { - parameters.Add("city", "Reykjavik"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" Adding 'city' twice throws: {ex.Message.Split(" (Parameter")[0]}"); - } - - Console.WriteLine(); - Console.WriteLine(" The collection is mutable and not thread-safe, and a client is meant to be shared, so"); - Console.WriteLine(" build one per operation and then leave it alone."); - - // The options record makes that cheap: keep the shared settings in one instance and derive the variant. - var shared = new ClickHouseTcpQueryOptions { Settings = new Dictionary { ["max_threads"] = "2" } }; - object hottest = await client.ExecuteScalarAsync( - $"SELECT max(temperature) FROM {TableName} WHERE city = {{city:String}}", - shared with { Parameters = parameters }); - - Console.WriteLine($" shared with {{ Parameters = parameters }} -> max(temperature) in Singapore = {hottest}"); - } - - private static async Task NoAtNameRewriting(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. Trap one: the query text carries the type, and @name is not rewritten\n"); - Console.WriteLine(" The HTTP client rewrites @city into {city:String} before sending. Nothing rewrites"); - Console.WriteLine(" anything here — the text goes to the server as you wrote it:\n"); - - var parameters = new ClickHouseTcpParameterCollection { { "city", "Amsterdam" } }; - var options = new ClickHouseTcpQueryOptions { Parameters = parameters }; - - try - { - await client.ExecuteScalarAsync($"SELECT count() FROM {TableName} WHERE city = @city", options); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" WHERE city = @city -> {Describe(ex)}"); - } - - // Without the type the server cannot parse the placeholder either. - try - { - await client.ExecuteScalarAsync($"SELECT count() FROM {TableName} WHERE city = {{city}}", options); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" WHERE city = {{city}} -> {Describe(ex)}"); - } - - object ok = await client.ExecuteScalarAsync($"SELECT count() FROM {TableName} WHERE city = {{city:String}}", options); - Console.WriteLine($" WHERE city = {{city:String}} -> {ok}"); - Console.WriteLine(); - Console.WriteLine(" So a query written for Dapper does not port over unchanged, and neither does one that"); - Console.WriteLine(" relied on the HTTP client inferring a type from the .NET value."); - } - - private static async Task Identifiers(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. Where the type comes from, and the Identifier placeholder\n"); - Console.WriteLine(" Three places, first match wins:"); - Console.WriteLine(" 1. ClickHouseTcpParameter.ClickHouseType, set on the parameter"); - Console.WriteLine(" 2. the query's {name:Type} placeholder"); - Console.WriteLine(" 3. the value's CLR type — which only ever applies to a parameter the query does"); - Console.WriteLine(" not name, because a query that does name it must state the type for the server"); - Console.WriteLine(); - Console.WriteLine(" So rung 1 exists for the case where the placeholder is not the format the value should"); - Console.WriteLine(" be written in. The server still reads the type from the query text, so an override that"); - Console.WriteLine(" disagrees with the placeholder makes the server parse text it did not expect: most"); - Console.WriteLine(" queries want rung 2 and nothing else."); - Console.WriteLine(); - - // Identifier is not a data type: the server splices the value in as a name rather than as a literal, so a - // table or column can be bound instead of concatenated into the query text. - var parameters = new ClickHouseTcpParameterCollection { { "tbl", TableName }, { "col", "temperature" } }; - object rows = await client.ExecuteScalarAsync( - "SELECT count({col:Identifier}) FROM {tbl:Identifier}", - new ClickHouseTcpQueryOptions { Parameters = parameters }); - - Console.WriteLine($" Identifier binds a name rather than a value — the one placeholder that is not a type:"); - Console.WriteLine($" SELECT count({{col:Identifier}}) FROM {{tbl:Identifier}} with tbl='{TableName}', col='temperature' = {rows}"); - } - - private static async Task InstantsNeedATimezone(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. Trap two: a value that names an instant needs a timezone in the placeholder\n"); - Console.WriteLine(" The wire carries a wall-clock time and no timezone, so the server reads the value in"); - Console.WriteLine(" its session timezone. For a value that names a point in time that silently moves the"); - Console.WriteLine(" instant, so the client refuses to send it rather than let it move:\n"); - - var noon = new DateTime(2026, 6, 1, 12, 0, 0, DateTimeKind.Utc); - - // Kind=Utc names an instant, and DateTime with no timezone argument declares none. - await Refused("DateTime Kind=Utc into {t:DateTime}", client, $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime}}", noon); - - // A DateTimeOffset always names an instant, whatever its offset is. - await Refused( - "DateTimeOffset into {t:DateTime}", - client, - $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime}}", - new DateTimeOffset(noon)); - - Console.WriteLine(); - Console.WriteLine(" Two fixes. Declare the timezone in the placeholder, which is what you want whenever"); - Console.WriteLine(" the value really is an instant:"); - - object declared = await Count(client, $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime('UTC')}}", noon); - Console.WriteLine($" {{t:DateTime('UTC')}} with Kind=Utc -> {declared} rows"); - - object offsetDeclared = await Count(client, $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime('UTC')}}", new DateTimeOffset(noon).ToOffset(TimeSpan.FromHours(5))); - Console.WriteLine($" {{t:DateTime('UTC')}} with a +05:00 offset -> {offsetDeclared} rows (the same instant, moved into UTC)"); - - Console.WriteLine(); - Console.WriteLine(" Or pass Kind=Unspecified, which says \"this wall-clock time, in whatever timezone the"); - Console.WriteLine(" server reads it in\" — no instant is claimed, so nothing can be lost:"); - - var wallClock = new DateTime(2026, 6, 1, 12, 0, 0, DateTimeKind.Unspecified); - object unspecified = await Count(client, $"SELECT count() FROM {TableName} WHERE recorded_at >= {{t:DateTime}}", wallClock); - Console.WriteLine($" {{t:DateTime}} with Kind=Unspecified -> {unspecified} rows"); - Console.WriteLine(); - Console.WriteLine(" DateTime64 is the same rule: {t:DateTime64(3, 'UTC')} declares one, {t:DateTime64(3)}"); - Console.WriteLine(" does not."); - } - - private static async Task NamesThatCollideWithSettings(ClickHouseTcpClient client) - { - Console.WriteLine("\n6. Trap three: a parameter named after a server setting\n"); - Console.WriteLine(" Parameters ride in the Query packet's settings list. A server that reads the name as a"); - Console.WriteLine(" setting applies it as that setting instead of binding it, and the query then fails while"); - Console.WriteLine(" the server is reading the setting's value. The names to avoid are the ordinary setting"); - Console.WriteLine(" names: limit and offset above all, and max_threads, readonly and log_comment too.\n"); - - string sql = $"SELECT id FROM {TableName} ORDER BY id LIMIT {{limit:UInt64}}"; - var collided = new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "limit", 2UL } }, - }; - - Console.WriteLine($" Server {(await client.GetServerInfoAsync()).Version}, parameter named 'limit':"); - - // A client of its own for the query that is meant to fail. The server rejects this one while it is still - // reading the settings list, and then closes the socket, so the connection it was on is dead even though - // the client saw an ordinary server error. The pool checks a connection for a closed socket both on - // return and on checkout, so it usually discards this one; a close notice that arrives after both checks - // can still be handed out. Disposing a throwaway client keeps that race out of the shared pool. - await using (ClickHouseTcpClient throwaway = ExampleConfig.CreateTcpClient()) - { - try - { - var ids = new List(); - await foreach (object[] row in throwaway.QueryAsync(sql, collided)) - { - ids.Add(row[0]); - } - - Console.WriteLine($" bound correctly — LIMIT {{limit:UInt64}} returned {ids.Count} row(s): {string.Join(", ", ids)}"); - Console.WriteLine(" This server is new enough to tell a parameter from a setting."); - } - catch (ClickHouseTcpException ex) - { - Console.WriteLine($" {Describe(ex)}"); - Console.WriteLine(" The error names neither the parameter nor the setting, so nothing in it points at"); - Console.WriteLine(" the name as the cause. (Code prints as Unknown when ClickHouseErrorCode has no"); - Console.WriteLine(" name for the raw number; the raw number is always there.)"); - Console.WriteLine(" The server also closes the connection after this one, so the next operation on"); - Console.WriteLine(" that connection can fail with a transport error instead — which is why this"); - Console.WriteLine(" example runs the failing query on a client of its own."); - } - } - - Console.WriteLine(); - Console.WriteLine(" The fix is a rename, and it always works:"); - - var renamed = new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "row_limit", 2UL } }, - }; - - var kept = new List(); - await foreach (object[] row in client.QueryAsync($"SELECT id FROM {TableName} ORDER BY id LIMIT {{row_limit:UInt64}}", renamed)) - { - kept.Add(row[0]); - } - - Console.WriteLine($" LIMIT {{row_limit:UInt64}} returned {kept.Count} row(s): {string.Join(", ", kept)}"); - Console.WriteLine(); - Console.WriteLine(" This is the server's behaviour and it is version-dependent — 25.8 through 26.6 apply"); - Console.WriteLine(" the name as a setting, newer servers bind it. clickhouse-client --param_limit= fails"); - Console.WriteLine(" the same way, and the driver's HTTP transport is unaffected because it carries the"); - Console.WriteLine(" name separately. So avoid a setting name for a parameter if you support any server in"); - Console.WriteLine(" that range, whatever the server in front of you does today."); - } - - private static void ShowWhatIsAbsent() - { - Console.WriteLine("\n7. What the HTTP client has here and this one does not\n"); - Console.WriteLine(" @name placeholders, rewritten client-side. Write {name:Type}."); - Console.WriteLine(" IParameterTypeResolver. The type comes from the placeholder, or from"); - Console.WriteLine(" ClickHouseTcpParameter.ClickHouseType, or — only for a parameter the query does not"); - Console.WriteLine(" name — from the value's CLR type."); - Console.WriteLine(" IParameterFormatter. There is no hook for how a value is written."); - Console.WriteLine(" DbParameter and DbType. This client is not an ADO.NET provider."); - Console.WriteLine(); - Console.WriteLine(" Null and DBNull both send the null marker, so a Nullable placeholder is the way to"); - Console.WriteLine(" bind an absent value: {city:Nullable(String)}."); - } - - private static async Task Refused(string label, ClickHouseTcpClient client, string sql, object value) - { - try - { - await Count(client, sql, value); - Console.WriteLine($" {label,-38} -> accepted (unexpected)"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" {label}:"); - Console.WriteLine($" {Wrap(ex.Message.Split(" (Parameter")[0])}"); - } - } - - private static ValueTask Count(ClickHouseTcpClient client, string sql, object value) - => client.ExecuteScalarAsync( - sql, - new ClickHouseTcpQueryOptions { Parameters = new ClickHouseTcpParameterCollection { { "t", value } } }); - - // The mapped error code, the number the server actually sent, and the first line of the message. Code is - // Unknown for a code the enum does not name, which is why RawCode is worth printing next to it. - private static string Describe(ClickHouseTcpException exception) - { - string message = exception.Message; - int newline = message.IndexOf('\n'); - if (newline >= 0) - { - message = message[..newline]; - } - - const string prefix = "DB::Exception: "; - if (message.StartsWith(prefix, StringComparison.Ordinal)) - { - message = message[prefix.Length..]; - } - - if (message.Length > 120) - { - message = message[..120] + " ..."; - } - - return exception is ClickHouseTcpServerException server - ? $"{server.Code} (code {server.RawCode}): {message}" - : $"{exception.GetType().Name}: {message}"; - } - - // Reflows a long driver message so the console output stays readable. - private static string Wrap(string message) - { - var lines = new List(); - var line = new System.Text.StringBuilder(); - foreach (string word in message.Split(' ')) - { - if (line.Length + word.Length + 1 > 92) - { - lines.Add(line.ToString()); - line.Clear(); - } - - line.Append(line.Length == 0 ? word : " " + word); - } - - lines.Add(line.ToString()); - return string.Join("\n ", lines); - } -} diff --git a/examples/Tcp/Read/Tcp_008_Poco.cs b/examples/Tcp/Read/Tcp_008_Poco.cs deleted file mode 100644 index dd8aa1304..000000000 --- a/examples/Tcp/Read/Tcp_008_Poco.cs +++ /dev/null @@ -1,282 +0,0 @@ -using System.Globalization; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// Reading a result into a class with QueryAsync<T> and writing one back with -/// InsertRowsAsync<T> — one type, both directions, and the two attributes that adjust the mapping: -/// to rename a property and -/// to take one out of the mapping entirely. -/// -/// -/// This is the tier most application code should use. It converts values to the property's type, so a -/// DateTime column reaches a DateTime property, and each row owns its values, so a row can be -/// returned from the method that read it. -/// -/// -public static class TcpPoco -{ - private const string TableName = "example_tcp_poco"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await CreateTable(client); - await WriteFromPocos(client); - await ReadIntoPocos(client); - await ShowTheMapping(client); - await ShowNotMappedOnInsert(client); - await ShowWhatDoesNotMap(client); - ShowTheRules(); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped '{TableName}'"); - } - } - - private static async Task CreateTable(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - id UInt64, - full_name String, - signal_count UInt32, - recorded_at DateTime('UTC'), - internal_notes String - ) - ENGINE = MergeTree() - ORDER BY id"); - - Console.WriteLine($"Created '{TableName}':"); - Console.WriteLine(" id UInt64, full_name String, signal_count UInt32, recorded_at DateTime('UTC'), internal_notes String"); - } - - private static async Task WriteFromPocos(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. InsertRowsAsync — the columns the INSERT names are read off each object\n"); - - var midnight = new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc); - - var rows = new List - { - new() { Id = 1, DisplayName = "Ada Lovelace", SignalCount = 12, RecordedAt = midnight.AddHours(6), Notes = "not written" }, - new() { Id = 2, DisplayName = "Grace Hopper", SignalCount = 7, RecordedAt = midnight.AddHours(9), Notes = "not written" }, - new() { Id = 3, DisplayName = "Alan Turing", SignalCount = 21, RecordedAt = midnight.AddHours(14), Notes = "not written" }, - }; - - // The statement ends at VALUES and names the columns to fill. Each is matched to a property; a property no - // named column matches is simply not read, which is how Notes stays out of this insert. - await client.InsertRowsAsync( - $"INSERT INTO {TableName} (id, full_name, signal_count, recorded_at) VALUES", - rows); - - Console.WriteLine($" Inserted {rows.Count} Observation objects into (id, full_name, signal_count, recorded_at)"); - Console.WriteLine(" id <- Id matched on the name"); - Console.WriteLine(" full_name <- DisplayName matched by [ClickHouseTcpColumn(Name = \"full_name\")]"); - Console.WriteLine(" signal_count <- SignalCount matched by ignoring case and underscores"); - Console.WriteLine(" recorded_at <- RecordedAt a DateTime property written as epoch seconds"); - Console.WriteLine(" internal_notes is not in the statement, so nothing was read for it and it took the"); - Console.WriteLine(" column's default. Notes carries [ClickHouseTcpNotMapped] and could not fill it anyway."); - } - - private static async Task ReadIntoPocos(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. QueryAsync — every result column fills the property it maps to\n"); - Console.WriteLine(" ID DisplayName Signals RecordedAt (Kind) Notes"); - Console.WriteLine(" -- -------------- ------- ------------------------- -----"); - - // SELECT * brings internal_notes back too, and it maps to nothing: Notes is [ClickHouseTcpNotMapped], so the - // column is skipped rather than assigned. - await foreach (Observation row in client.QueryAsync($"SELECT * FROM {TableName} ORDER BY id")) - { - Console.WriteLine( - $" {row.Id,2} {row.DisplayName,-14} {row.SignalCount,7} " + - $"{row.RecordedAt.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture)} ({row.RecordedAt.Kind}) " + - $"{(row.Notes is null ? "(null)" : row.Notes)}"); - } - - Console.WriteLine(); - Console.WriteLine(" RecordedAt is a real DateTime, converted with the timezone the column declares — the"); - Console.WriteLine(" conversion the object[] tier does not do. Kind is Utc here because the column says UTC."); - Console.WriteLine(" Notes is null: internal_notes was in the result and was skipped."); - } - - private static async Task ShowTheMapping(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. How a column finds its property\n"); - Console.WriteLine(" In order, first match wins:"); - Console.WriteLine(" the exact property name, then case-insensitively, then ignoring underscores."); - Console.WriteLine(" So signal_count reaches SignalCount with no attribute at all. Reach for"); - Console.WriteLine(" [ClickHouseTcpColumn(Name = ...)] only when the names differ by more than that —"); - Console.WriteLine(" full_name and DisplayName here."); - Console.WriteLine(); - Console.WriteLine(" The names matched are the result's, not the table's, so a SELECT alias lines a query up"); - Console.WriteLine(" with a type just as well as an attribute does — and it is the only way to name a"); - Console.WriteLine(" computed column:\n"); - - // The names in the result are the aliases the query chose, not the table's, so an alias is the other way to - // line a result up with a type. - await foreach (Summary row in client.QueryAsync( - $"SELECT count() AS rows, sum(signal_count) AS total_signals, max(recorded_at) AS latest FROM {TableName}")) - { - Console.WriteLine($" SELECT count() AS rows, sum(signal_count) AS total_signals, max(recorded_at) AS latest"); - Console.WriteLine($" Rows {row.Rows}, TotalSignals {row.TotalSignals}, Latest {row.Latest.ToString("u", CultureInfo.InvariantCulture)}"); - Console.WriteLine(" Latest is a DateTimeOffset property, so the offset the column's timezone gives is kept."); - } - } - - private static async Task ShowNotMappedOnInsert(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. [ClickHouseTcpNotMapped] excludes a property in both directions\n"); - Console.WriteLine(" Section 2 showed the read half: internal_notes was skipped. On an insert the exclusion"); - Console.WriteLine(" means the property cannot fill a column, so naming that column is an error rather than"); - Console.WriteLine(" a silent default:\n"); - - try - { - await client.InsertRowsAsync( - $"INSERT INTO {TableName} (id, full_name, internal_notes) VALUES", - new List { new() { Id = 9, DisplayName = "nobody", Notes = "would have gone here" } }); - } - catch (InvalidOperationException ex) - { - Console.WriteLine($" INSERT INTO ... (id, full_name, internal_notes) throws:"); - Console.WriteLine(Wrap(ex.Message, " ")); - } - - Console.WriteLine(); - Console.WriteLine(" Without the attribute, Notes would match internal_notes by ignoring the underscore, so"); - Console.WriteLine(" the attribute is what makes a property the driver never touches — a cache key, a"); - Console.WriteLine(" computed column, something loaded from elsewhere."); - } - - private static async Task ShowWhatDoesNotMap(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. What a mismatch does\n"); - - // Mapping is resolved against the first block of the result, so a type nothing maps to fails on the first - // row rather than yielding wrong values. - try - { - await foreach (Unrelated _ in client.QueryAsync($"SELECT id, full_name FROM {TableName}")) - { - break; - } - } - catch (InvalidOperationException ex) - { - Console.WriteLine(" A type no result column maps to:"); - Console.WriteLine(Wrap(ex.Message, " ")); - } - - // A property that some column does map to, but whose type the column cannot be read as. - try - { - await foreach (WrongType _ in client.QueryAsync($"SELECT id, full_name FROM {TableName}")) - { - break; - } - } - catch (InvalidOperationException ex) - { - Console.WriteLine("\n A property whose type the column cannot be read as:"); - Console.WriteLine(Wrap(ex.Message, " ")); - } - - Console.WriteLine(); - Console.WriteLine(" Both are checked against the first block, so an empty result yields nothing and"); - Console.WriteLine(" validates nothing. A property that no column reaches is not an error: it keeps its"); - Console.WriteLine(" default, which is what lets one type serve several queries."); - } - - private static void ShowTheRules() - { - Console.WriteLine("\n6. What T has to be\n"); - Console.WriteLine(" A concrete class with a public parameterless constructor."); - Console.WriteLine(" Every property a result column reaches needs a public setter — an init-only or"); - Console.WriteLine(" get-only property cannot be filled, so a record with positional parameters does not"); - Console.WriteLine(" work for reading."); - Console.WriteLine(" Every column an INSERT names needs a public getter of a type that column can be"); - Console.WriteLine(" written from."); - Console.WriteLine(" Rows own their values and stay valid after the enumeration advances. LowCardinality"); - Console.WriteLine(" elements can be shared within a block, so do not mutate an array-valued property"); - Console.WriteLine(" in place."); - Console.WriteLine(" The read and write plans are compiled once per type per client, so a client meant to"); - Console.WriteLine(" be a singleton pays the reflection once."); - } - - // Reflows a long driver message so the console output stays readable. - private static string Wrap(string message, string indent) - { - var lines = new List(); - var line = new System.Text.StringBuilder(); - foreach (string word in message.Split(' ')) - { - if (line.Length + word.Length + 1 > 95) - { - lines.Add(line.ToString()); - line.Clear(); - } - - line.Append(line.Length == 0 ? word : " " + word); - } - - lines.Add(line.ToString()); - return indent + string.Join("\n" + indent, lines); - } - - /// - /// One row of the example's table, used for both the insert and the read. The attributes are the only two the - /// native client has. - /// - private sealed class Observation - { - public ulong Id { get; set; } - - // The column is full_name, which no name-matching rule reaches from DisplayName. - [ClickHouseTcpColumn(Name = "full_name")] - public string DisplayName { get; set; } = string.Empty; - - // signal_count matches this by ignoring case and underscores, so no attribute is needed. - public uint SignalCount { get; set; } - - // A DateTime('UTC') column's epoch-second count, converted on the way in and out. - public DateTime RecordedAt { get; set; } - - // Excluded in both directions. Without this it would match internal_notes. - [ClickHouseTcpNotMapped] - public string? Notes { get; set; } - } - - // A second shape over the same table: the mapping is per query, so one table can feed several types. - private sealed class Summary - { - public ulong Rows { get; set; } - - public ulong TotalSignals { get; set; } - - // DateTimeOffset keeps the offset the column's timezone gives; DateTime would flatten it. - public DateTimeOffset Latest { get; set; } - } - - private sealed class Unrelated - { - public string Something { get; set; } = string.Empty; - } - - private sealed class WrongType - { - public ulong Id { get; set; } - - // full_name is a String column, and a String cannot be read as a Guid. - public Guid FullName { get; set; } - } -} diff --git a/examples/Tcp/Types/Tcp_001_ScalarTypes.cs b/examples/Tcp/Types/Tcp_001_ScalarTypes.cs new file mode 100644 index 000000000..491986335 --- /dev/null +++ b/examples/Tcp/Types/Tcp_001_ScalarTypes.cs @@ -0,0 +1,131 @@ +using System.Globalization; +using System.Net; +using System.Numerics; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Shows the CLR values used for representative ClickHouse scalar types. +public static class TcpScalarTypes +{ + private const string TableName = "example_tcp_scalar_types"; + private const string Columns = + "u8, i64, u128, i128, u256, i256, f32, f64, bf16, d64, d128, " + + "flag, text, fixed5, id, ip4, ip6, colour"; + + public static async Task Run() + { + var builder = ExampleConfig.TcpBuilder(); + + // BFloat16 is setting-gated on older supported ClickHouse versions. + builder["set_allow_experimental_bfloat16_type"] = 1; + + await using var client = new ClickHouseTcpClient(builder.ToOptions()); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + u8 UInt8, + i64 Int64, + u128 UInt128, + i128 Int128, + u256 UInt256, + i256 Int256, + f32 Float32, + f64 Float64, + bf16 BFloat16, + d64 Decimal64(4), + d128 Decimal128(20), + flag Bool, + text String, + fixed5 FixedString(5), + id UUID, + ip4 IPv4, + ip6 IPv6, + colour Enum8('red' = 1, 'green' = 2) + ) + ENGINE = MergeTree + ORDER BY id + """); + + await client.InsertAsync( + $"INSERT INTO {TableName} ({Columns}) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("u8", new byte[] { 200 }), + ClickHouseTcpColumn.Create("i64", new[] { -42L }), + ClickHouseTcpColumn.Create("u128", new[] { UInt128.MaxValue }), + ClickHouseTcpColumn.Create("i128", new[] { Int128.MinValue }), + ClickHouseTcpColumn.Create( + "u256", + new[] { UInt256.FromBigInteger(BigInteger.Pow(2, 255)) }), + ClickHouseTcpColumn.Create( + "i256", + new[] { Int256.FromBigInteger(-BigInteger.Pow(2, 255)) }), + ClickHouseTcpColumn.Create("f32", new[] { 1.5f }), + ClickHouseTcpColumn.Create("f64", new[] { -2.25 }), + ClickHouseTcpColumn.Create("bf16", new[] { 0.1f }), + ClickHouseTcpColumn.Create("d64", new[] { 1.2345m }), + + // Decimal128 has 38-digit precision, so it uses ClickHouseTcpDecimal. + ClickHouseTcpColumn.Create( + "d128", + new[] + { + new ClickHouseTcpDecimal( + BigInteger.Parse( + "123456789012345678901234567890", + CultureInfo.InvariantCulture), + 20), + }), + ClickHouseTcpColumn.Create("flag", new[] { true }), + ClickHouseTcpColumn.Create("text", new[] { "hello" }), + + // FixedString is binary data; byte[] preserves zeros and non-UTF-8 bytes. + ClickHouseTcpColumn.Create( + "fixed5", + new[] { new byte[] { 0x61, 0x00, 0x62, 0xFF, 0x10 } }), + ClickHouseTcpColumn.Create( + "id", + new[] { Guid.Parse("61f0c404-5cb3-11e7-907b-a6006ad3dba0") }), + ClickHouseTcpColumn.Create("ip4", new[] { IPAddress.Parse("192.168.0.1") }), + ClickHouseTcpColumn.Create("ip6", new[] { IPAddress.Parse("2001:db8::1") }), + + // Enum columns use their signed integer storage type on the block tier. + ClickHouseTcpColumn.Create("colour", new sbyte[] { 2 }), + }); + + await foreach (Block block in client.StreamAsync($"SELECT {Columns} FROM {TableName}")) + { + foreach (IColumn column in block.Columns) + { + Console.WriteLine( + $"{column.Name,-7} {column.TypeName,-28} " + + $"-> {FriendlyName(column.ElementType),-20} {Render(column.GetValue(0))}"); + } + } + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } + + private static string FriendlyName(Type type) => type switch + { + _ when type == typeof(byte[]) => "byte[]", + _ => type.Name, + }; + + private static string Render(object? value) => value switch + { + byte[] bytes => "0x" + Convert.ToHexString(bytes), + string text => $"\"{text}\"", + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + null => "NULL", + _ => value.ToString() ?? "NULL", + }; +} diff --git a/examples/Tcp/Types/Tcp_002_DateTimeAndTimezones.cs b/examples/Tcp/Types/Tcp_002_DateTimeAndTimezones.cs new file mode 100644 index 000000000..9c2f0b685 --- /dev/null +++ b/examples/Tcp/Types/Tcp_002_DateTimeAndTimezones.cs @@ -0,0 +1,112 @@ +using System.Globalization; +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Reads and writes ClickHouse date, timestamp, and time values. +public static class TcpDateTimeAndTimezones +{ + private const string TableName = "example_tcp_datetime_timezones"; + + public static async Task Run() + { + var builder = ExampleConfig.TcpBuilder(); + + // ClickHouse 25.8 requires both settings for Time and Time64. + builder["set_enable_time_time64_type"] = 1; + builder["set_allow_experimental_time_time64_type"] = 1; + + await using var client = new ClickHouseTcpClient(builder.ToOptions()); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + day Date, + old_day Date32, + captured_at DateTime('Europe/Amsterdam'), + precise_at DateTime64(3, 'UTC'), + elapsed Time, + precise_elapsed Time64(3) + ) + ENGINE = MergeTree + ORDER BY day + """); + + // UTC and Local DateTime values name an instant. Unspecified values use the column timezone. + var instant = new DateTime(2026, 6, 1, 12, 0, 0, DateTimeKind.Utc); + await client.InsertAsync( + $"INSERT INTO {TableName} " + + "(day, old_day, captured_at, precise_at, elapsed, precise_elapsed) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("day", new[] { new DateOnly(2026, 6, 1) }), + ClickHouseTcpColumn.Create("old_day", new[] { new DateOnly(1920, 3, 4) }), + ClickHouseTcpColumn.Create("captured_at", new[] { instant }), + ClickHouseTcpColumn.Create( + "precise_at", + new[] { instant.AddMilliseconds(123) }), + + // Time and Time64 surface as TimeSpan, including values longer than one day. + ClickHouseTcpColumn.Create("elapsed", new[] { TimeSpan.FromHours(27) }), + ClickHouseTcpColumn.Create( + "precise_elapsed", + new[] { TimeSpan.FromMilliseconds(1234) }), + }); + + await foreach (Block block in client.StreamAsync($"SELECT * FROM {TableName}")) + { + Console.WriteLine($"day: {block.Column("day")[0]:yyyy-MM-dd}"); + Console.WriteLine($"old_day: {block.Column("old_day")[0]:yyyy-MM-dd}"); + + // IDateTimeColumn converts raw epoch counts with the column's timezone and scale. + PrintTimestamp((IDateTimeColumn)block["captured_at"]); + PrintTimestamp((IDateTimeColumn)block["precise_at"]); + PrintTime((ITimeColumn)block["elapsed"]); + PrintTime((ITimeColumn)block["precise_elapsed"]); + } + + var utc = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["session_timezone"] = "UTC" }, + }; + var tokyo = new ClickHouseTcpQueryOptions + { + Settings = new Dictionary { ["session_timezone"] = "Asia/Tokyo" }, + }; + + // A DateTime type with no zone uses session_timezone, then the server's default timezone. + Console.WriteLine("A DateTime without a declared zone uses session_timezone:"); + await PrintBareTimestamp(client, utc); + await PrintBareTimestamp(client, tokyo); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } + + private static void PrintTimestamp(IDateTimeColumn column) + { + DateTimeOffset value = column.GetDateTimeOffset(0); + Console.WriteLine( + $"{column.Name}: {value.ToString("O", CultureInfo.InvariantCulture)} " + + $"(zone {column.TimeZone.Id}, scale {column.Scale})"); + } + + private static void PrintTime(ITimeColumn column) + => Console.WriteLine($"{column.Name}: {column.GetTimeSpan(0)} (scale {column.Scale})"); + + private static async Task PrintBareTimestamp( + ClickHouseTcpClient client, + ClickHouseTcpQueryOptions options) + { + await foreach (Block block in client.StreamAsync("SELECT toDateTime(0) AS value", options)) + { + var column = (IDateTimeColumn)block["value"]; + Console.WriteLine($" {column.TimeZone.Id}: {column.GetDateTimeOffset(0):O}"); + } + } +} diff --git a/examples/Tcp/Types/Tcp_003_CompositeRead.cs b/examples/Tcp/Types/Tcp_003_CompositeRead.cs new file mode 100644 index 000000000..47e93126d --- /dev/null +++ b/examples/Tcp/Types/Tcp_003_CompositeRead.cs @@ -0,0 +1,104 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Reads composite values through their typed columnar views. +public static class TcpCompositeRead +{ + private const string TableName = "example_tcp_composite_read"; + private const string Columns = "id, readings, attributes, point, score, city"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + id UInt64, + readings Array(Float64), + attributes Map(String, Int64), + point Tuple(x Int32, y String), + score Nullable(Float64), + city LowCardinality(String) + ) + ENGINE = MergeTree + ORDER BY id + """); + + await client.InsertAsync( + $"INSERT INTO {TableName} ({Columns}) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2 }), + ClickHouseTcpColumn.Create( + "readings", + new[] { new[] { 0.5, 0.75 }, Array.Empty() }), + ClickHouseTcpColumn.Create( + "attributes", + new[] + { + new[] { new KeyValuePair("floor", 3) }, + Array.Empty>(), + }), + ClickHouseTcpColumn.Create("point", new[] { (1, "one"), (2, "two") }), + ClickHouseTcpColumn.Create("score", new double?[] { 1.25, null }), + ClickHouseTcpColumn.Create("city", new[] { "Amsterdam", "Amsterdam" }), + }); + + // Composite interfaces expose flattened native storage without allocating one object per row. + await foreach (Block block in client.StreamAsync( + $"SELECT {Columns} FROM {TableName} ORDER BY id")) + { + if (block["readings"] is IArrayColumn readings) + { + Console.WriteLine( + $"Array values [{string.Join(", ", readings.InnerValues.ToArray())}], " + + $"offsets [{string.Join(", ", readings.Offsets.ToArray())}]"); + } + + if (block["attributes"] is IMapColumn attributes) + { + Console.WriteLine( + $"Map keys [{string.Join(", ", attributes.KeyColumn.Values.ToArray())}], " + + $"values [{string.Join(", ", attributes.ValueColumn.Values.ToArray())}]"); + } + + if (block["point"] is ITupleColumn point) + { + Console.WriteLine( + $"Tuple fields [{string.Join(", ", point.FieldNames ?? Array.Empty())}]"); + } + + if (block["score"] is INullableColumn score) + { + Console.WriteLine( + $"Nullable null map [{string.Join(", ", score.NullMap.ToArray())}]"); + } + + if (block["city"] is ILowCardinalityColumn city) + { + // LowCardinality stores values once and addresses them with integer keys. + Console.WriteLine( + $"LowCardinality dictionary [{string.Join(", ", city.Dictionary.Values.ToArray())}], " + + $"keys [{string.Join(", ", city.Keys.ToArray())}]"); + } + } + + // Geo aliases use the same column shape as their underlying tuple types. + await foreach (Block block in client.StreamAsync( + "SELECT CAST((1.0, 2.0), 'Point') AS point")) + { + Console.WriteLine($"Point uses {block["point"].GetType().Name} and materializes as " + + $"{block["point"].GetValue(0)}"); + } + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } +} diff --git a/examples/Tcp/Types/Tcp_004_VariantDynamicJson.cs b/examples/Tcp/Types/Tcp_004_VariantDynamicJson.cs new file mode 100644 index 000000000..9c3e2c540 --- /dev/null +++ b/examples/Tcp/Types/Tcp_004_VariantDynamicJson.cs @@ -0,0 +1,137 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Reads and writes Variant, Dynamic, and JSON columns. +public static class TcpVariantDynamicJson +{ + private const string VariantTable = "example_tcp_variant"; + private const string DynamicTable = "example_tcp_dynamic"; + private const string JsonTable = "example_tcp_json"; + + public static async Task Run() + { + var builder = ExampleConfig.TcpBuilder(); + + // These types are setting-gated on older supported ClickHouse versions. + builder["set_allow_experimental_variant_type"] = 1; + builder["set_allow_experimental_dynamic_type"] = 1; + builder["set_allow_experimental_json_type"] = 1; + + await using var client = new ClickHouseTcpClient(builder.ToOptions()); + string[] tables = { VariantTable, DynamicTable, JsonTable }; + + foreach (string table in tables) + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {VariantTable} + (id UInt64, value Variant(String, UInt64, Array(Int32))) + ENGINE = MergeTree + ORDER BY id + """); + await client.ExecuteAsync($""" + CREATE TABLE {DynamicTable} + (id UInt64, value Dynamic) + ENGINE = MergeTree + ORDER BY id + """); + await client.ExecuteAsync($""" + CREATE TABLE {JsonTable} + (id UInt64, document JSON) + ENGINE = MergeTree + ORDER BY id + """); + + // Variant alternatives are fixed by its declaration. + await client.InsertAsync( + $"INSERT INTO {VariantTable} (id, value) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3, 4 }), + ClickHouseTcpColumn.Create( + "value", + new object?[] { 42UL, "hello", new[] { 1, 2 }, null }), + }); + + // Dynamic records the concrete types that occur in the inserted values. + await client.InsertAsync( + $"INSERT INTO {DynamicTable} (id, value) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3, 4 }), + ClickHouseTcpColumn.Create( + "value", + new object?[] { 42UL, "hello", 1.5, null }), + }); + + const string json = "{ \"b\": 1, \"a\": 2 }"; + await client.InsertAsync( + $"INSERT INTO {JsonTable} (id, document) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1 }), + ClickHouseTcpColumn.Create("document", new[] { json }), + }); + + await PrintVariant(client); + await PrintDynamic(client); + + // ClickHouse parses and normalizes JSON, so the returned text may differ from the input. + object normalized = await client.ExecuteScalarAsync( + $"SELECT document FROM {JsonTable} WHERE id = 1"); + Console.WriteLine($"JSON written: {json}"); + Console.WriteLine($"JSON read: {normalized}"); + } + finally + { + foreach (string table in tables) + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + } + } + + private static async Task PrintVariant(ClickHouseTcpClient client) + { + await foreach (Block block in client.StreamAsync( + $"SELECT value FROM {VariantTable} ORDER BY id")) + { + var column = (IVariantColumn)block["value"]; + Console.WriteLine( + $"Variant: {column.TypeCount} alternatives, discriminators " + + $"[{string.Join(", ", column.Discriminators.ToArray())}]"); + + for (int row = 0; row < column.RowCount; row++) + { + Console.WriteLine($" {FormatValue(block["value"].GetValue(row))}"); + } + } + } + + private static async Task PrintDynamic(ClickHouseTcpClient client) + { + await foreach (Block block in client.StreamAsync( + $"SELECT value FROM {DynamicTable} ORDER BY id")) + { + var column = (IDynamicColumn)block["value"]; + Console.WriteLine($"Dynamic types: [{string.Join(", ", column.TypeNames)}]"); + + for (int row = 0; row < column.RowCount; row++) + { + Console.WriteLine($" {FormatValue(block["value"].GetValue(row))}"); + } + } + } + + private static string FormatValue(object? value) => value switch + { + Array items => $"[{string.Join(", ", items.Cast())}]", + null => "NULL", + _ => value.ToString() ?? "NULL", + }; +} diff --git a/examples/Tcp/Types/Tcp_005_QBitVectorSearch.cs b/examples/Tcp/Types/Tcp_005_QBitVectorSearch.cs new file mode 100644 index 000000000..de17dd03b --- /dev/null +++ b/examples/Tcp/Types/Tcp_005_QBitVectorSearch.cs @@ -0,0 +1,91 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Stores vectors in QBit columns and reads their transposed bit planes. +public static class TcpQBitVectorSearch +{ + private const string TableName = "example_tcp_qbit"; + private static readonly Version QBitFrom = new(25, 11); + + private static readonly (string Word, float[] Vector)[] Corpus = + { + ("apple", new[] { 0.9f, 0.1f, 0.8f, 0.2f, 0.7f }), + ("banana", new[] { 0.85f, 0.15f, 0.75f, 0.25f, 0.65f }), + ("dog", new[] { 0.1f, 0.9f, 0.2f, 0.8f, 0.3f }), + }; + + public static async Task Run() + { + var builder = ExampleConfig.TcpBuilder(); + + // QBit is setting-gated on the earliest server versions that provide it. + builder["set_allow_experimental_qbit_type"] = 1; + + await using var client = new ClickHouseTcpClient(builder.ToOptions()); + ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); + + if (server.Version < QBitFrom) + { + Console.WriteLine($"QBit requires ClickHouse {QBitFrom} or newer; found {server.Version}."); + return; + } + + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + (word String, vector QBit(Float32, 5)) + ENGINE = MergeTree + ORDER BY word + """); + + await client.InsertAsync( + $"INSERT INTO {TableName} (word, vector) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create( + "word", + Corpus.Select(item => item.Word).ToArray()), + ClickHouseTcpColumn.Create( + "vector", + Corpus.Select(item => item.Vector).ToArray()), + }); + + Console.WriteLine("Nearest vectors:"); + + // The last argument keeps 16 high-order bit planes for the approximate distance. + await foreach (object[] row in client.QueryAsync($""" + SELECT word, + L2DistanceTransposed(vector, [0.9, 0.1, 0.8, 0.2, 0.7], 16) AS distance + FROM {TableName} + ORDER BY distance + """)) + { + Console.WriteLine($" {row[0]}: {row[1]}"); + } + + await foreach (Block block in client.StreamAsync( + $"SELECT vector FROM {TableName} ORDER BY word")) + { + var qbit = (IQBitColumn)block["vector"]; + Console.WriteLine( + $"QBit layout: dimension={qbit.Dimension}, bit width={qbit.BitWidth}, " + + $"bytes per row={qbit.BytesPerRow}"); + + // Each plane contains one bit from every vector element, packed for every row. + ReadOnlySpan signPlane = qbit.GetPlane(qbit.BitWidth - 1); + Console.WriteLine($"Sign plane: {Convert.ToHexString(signPlane)}"); + + float[] firstVector = block.Column("vector")[0]; + Console.WriteLine($"Materialized vector: [{string.Join(", ", firstVector)}]"); + } + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } +} diff --git a/examples/Tcp/Types/Tcp_011_ScalarTypes.cs b/examples/Tcp/Types/Tcp_011_ScalarTypes.cs deleted file mode 100644 index 28b0812a7..000000000 --- a/examples/Tcp/Types/Tcp_011_ScalarTypes.cs +++ /dev/null @@ -1,472 +0,0 @@ -using System.Globalization; -using System.Net; -using System.Numerics; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// What CLR type each ClickHouse scalar becomes, and the four families where the answer is not the obvious one: -/// the 256-bit integers, the decimals, BFloat16, and the enums. -/// -/// -/// One rule underlies most of it: the client hands back the value the wire carried, in the narrowest CLR -/// type that holds it. So UInt8 is a and not an , -/// FixedString(N) is a [] and not a , and an Enum8 is its -/// ordinal and not its label. The same type is what an insert column must hold, in both directions. -/// -/// -/// -/// Three types are a chosen CLR surface rather than the wire bytes: BFloat16 widens to a -/// , IPv4 and IPv6 become an IPAddress, and String is decoded as -/// UTF-8, which a ClickHouse String is not required to be. Sections 4 and 5 show what that costs. -/// -/// -/// -/// Tcp_012 covers the date and time family, Tcp_013 the composites. This one assumes the block tier -/// from Tcp_006. -/// -/// -public static class TcpScalarTypes -{ - private const string TableName = "example_tcp_scalar_types"; - - // Every column of the table above, in one list, so the DDL, the insert and the read agree. - private const string Columns = - "u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, u256, i256, " + - "f32, f64, bf16, d32, d64, d128, d256, flag, text, fixed5, id, ip4, ip6, e8, e16"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await Seed(client); - await TheWholeMap(client); - await WideIntegers(client); - await Decimals(client); - await Floats(client); - await StringsAndBytes(client); - await Enums(client); - await Nothing(client); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - Console.WriteLine($"\nDropped '{TableName}'"); - } - } - - private static async Task Seed(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - u8 UInt8, i8 Int8, u16 UInt16, i16 Int16, u32 UInt32, i32 Int32, u64 UInt64, i64 Int64, - u128 UInt128, i128 Int128, u256 UInt256, i256 Int256, - f32 Float32, f64 Float64, bf16 BFloat16, - d32 Decimal32(2), d64 Decimal64(4), d128 Decimal128(20), d256 Decimal256(40), - flag Bool, text String, fixed5 FixedString(5), - id UUID, ip4 IPv4, ip6 IPv6, - e8 Enum8('red' = 1, 'green' = 2), e16 Enum16('small' = 100, 'big' = 3000) - ) - ENGINE = MergeTree() - ORDER BY u64"); - - // One row, written column by column. Each array's element type is the CLR type that column accepts, so - // this list is the write-side answer to the same question the read side answers below. - await client.InsertAsync( - $"INSERT INTO {TableName} ({Columns}) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("u8", new byte[] { 200 }), - ClickHouseTcpColumn.Create("i8", new sbyte[] { -100 }), - ClickHouseTcpColumn.Create("u16", new ushort[] { 60000 }), - ClickHouseTcpColumn.Create("i16", new short[] { -30000 }), - ClickHouseTcpColumn.Create("u32", new uint[] { 4000000000 }), - ClickHouseTcpColumn.Create("i32", new[] { -2000000000 }), - ClickHouseTcpColumn.Create("u64", new ulong[] { ulong.MaxValue }), - ClickHouseTcpColumn.Create("i64", new[] { long.MinValue }), - ClickHouseTcpColumn.Create("u128", new[] { UInt128.MaxValue }), - ClickHouseTcpColumn.Create("i128", new[] { Int128.MinValue }), - - // The only two numeric types the driver defines itself: .NET stops at 128 bits. - ClickHouseTcpColumn.Create("u256", new[] { UInt256.FromBigInteger(BigInteger.Pow(2, 255)) }), - ClickHouseTcpColumn.Create("i256", new[] { Int256.FromBigInteger(-BigInteger.Pow(2, 255)) }), - - ClickHouseTcpColumn.Create("f32", new[] { 1.5f }), - ClickHouseTcpColumn.Create("f64", new[] { -2.25 }), - - // BFloat16 has no CLR type of its own, so it is written from and read as a float. - ClickHouseTcpColumn.Create("bf16", new[] { 0.1f }), - - // Precision decides the CLR type: 2 and 4 digits fit a decimal, 20 and 40 do not. - ClickHouseTcpColumn.Create("d32", new[] { 1.25m }), - ClickHouseTcpColumn.Create("d64", new[] { 1.2345m }), - ClickHouseTcpColumn.Create("d128", new[] { new ClickHouseTcpDecimal(BigInteger.Parse("123456789012345678901234567890", CultureInfo.InvariantCulture), 20) }), - ClickHouseTcpColumn.Create("d256", new[] { new ClickHouseTcpDecimal(BigInteger.Pow(10, 45) + 7, 40) }), - - ClickHouseTcpColumn.Create("flag", new[] { true }), - ClickHouseTcpColumn.Create("text", new[] { "hello" }), - - // FixedString is bytes, and exactly N of them. - ClickHouseTcpColumn.Create("fixed5", new[] { new byte[] { 0x61, 0x00, 0x62, 0xFF, 0x10 } }), - - ClickHouseTcpColumn.Create("id", new[] { Guid.Parse("61f0c404-5cb3-11e7-907b-a6006ad3dba0") }), - ClickHouseTcpColumn.Create("ip4", new[] { IPAddress.Parse("192.168.0.1") }), - ClickHouseTcpColumn.Create("ip6", new[] { IPAddress.Parse("2001:db8::1") }), - - // An enum is written as its ordinal, never as its label. - ClickHouseTcpColumn.Create("e8", new sbyte[] { 2 }), - ClickHouseTcpColumn.Create("e16", new short[] { 3000 }), - }); - - Console.WriteLine($"Seeded '{TableName}' with one row of every scalar type, written column by column."); - Console.WriteLine("Each Create above states the CLR type that column accepts; the table below is the"); - Console.WriteLine("same answer read back."); - } - - private static async Task TheWholeMap(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. The whole scalar map\n"); - Console.WriteLine(" ClickHouse type IColumn is The value read back"); - Console.WriteLine(" ------------------------------------ -------------------- -------------------"); - - await foreach (Block block in client.StreamAsync($"SELECT {Columns} FROM {TableName}")) - { - foreach (IColumn column in block.Columns) - { - object value = column.GetValue(0); - Console.WriteLine($" {column.TypeName,-36} {Describe(column.ElementType),-20} {Render(value)}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" ElementType is what an insert column must hold as well as what a read gives back. For most"); - Console.WriteLine(" of that table it is the wire's own type, so a read costs a copy at most. BFloat16, IPv4,"); - Console.WriteLine(" IPv6 and String are the exceptions: each is a CLR surface built from the wire bytes."); - } - - private static async Task WideIntegers(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. The wide integers: two from .NET, two from this driver\n"); - Console.WriteLine(" Int128 and UInt128 are BCL types, so UInt128.MaxValue and Int128.MinValue work as you"); - Console.WriteLine(" expect. Int256 and UInt256 have no BCL counterpart, so the driver defines them:\n"); - - await foreach (Block block in client.StreamAsync($"SELECT u128, i128, u256, i256 FROM {TableName}")) - { - Console.WriteLine($" UInt128 = {block.Column("u128")[0]}"); - Console.WriteLine($" Int128 = {block.Column("i128")[0]}"); - - UInt256 wide = block.Column("u256")[0]; - Int256 signed = block.Column("i256")[0]; - Console.WriteLine($" UInt256 = {wide}"); - Console.WriteLine($" Int256 = {signed} IsNegative {signed.IsNegative}"); - - Console.WriteLine(); - Console.WriteLine(" They are 32-byte value types, four ulong limbs least significant first, and they"); - Console.WriteLine(" carry exactly what a wire value needs — no arithmetic:"); - Console.WriteLine($" Int256.Size {Int256.Size} bytes"); - Console.WriteLine($" Int256.Zero {Int256.Zero}"); - Console.WriteLine($" new Int256(0, 1, 0, 0) {new Int256(0, 1, 0, 0)} (limb 1 is 2^64)"); - Console.WriteLine($" signed.ToBigInteger() == -2^255 {signed.ToBigInteger() == -BigInteger.Pow(2, 255)}"); - - // Round-tripping through BigInteger is how arithmetic is done: the struct has comparison operators - // but no +, -, * or /. - Int256 doubled = Int256.FromBigInteger(Int256.FromBigInteger(21).ToBigInteger() * 2); - Console.WriteLine($" 21 * 2 via BigInteger {doubled} (there is no Int256 operator *)"); - - Span raw = stackalloc byte[Int256.Size]; - signed.WriteLittleEndian(raw); - Console.WriteLine($" WriteLittleEndian {Convert.ToHexString(raw)}"); - Console.WriteLine($" ReadLittleEndian round trip {Int256.ReadLittleEndian(raw) == signed}"); - } - } - - private static async Task Decimals(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. Decimals: the declared precision decides the CLR type\n"); - Console.WriteLine(" A Decimal(P, S) is a signed integer mantissa of a width P chooses, and the value is"); - Console.WriteLine(" mantissa / 10^S. P up to 18 fits a System.Decimal; wider does not, so it surfaces as"); - Console.WriteLine(" ClickHouseTcpDecimal. That is decided by P alone, never by the value:\n"); - - await foreach (Block block in client.StreamAsync( - @"SELECT CAST('1.25', 'Decimal(18, 2)') AS at_18, CAST('1.25', 'Decimal(19, 2)') AS at_19")) - { - foreach (IColumn column in block.Columns) - { - Console.WriteLine($" {column.TypeName,-16} -> {Describe(column.ElementType),-22} value {column.GetValue(0)}"); - } - - Console.WriteLine(" Both hold 1.25. Only the declared precision differs."); - } - - Console.WriteLine(); - Console.WriteLine(" ClickHouseTcpDecimal is the mantissa and the scale, unchanged from the wire:\n"); - - await foreach (Block block in client.StreamAsync($"SELECT d128, d256 FROM {TableName}")) - { - foreach (IColumn column in block.Columns) - { - var value = (ClickHouseTcpDecimal)column.GetValue(0); - bool narrows = value.TryToDecimal(out decimal narrowed); - Console.WriteLine($" {column.Name} {column.TypeName}"); - Console.WriteLine($" Mantissa {value.Mantissa}"); - Console.WriteLine($" Scale {value.Scale}, Sign {value.Sign}, ToString() {value}"); - Console.WriteLine($" TryToDecimal {narrows}{(narrows ? $" -> {narrowed}" : " (out of a System.Decimal's range)")}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" A System.Decimal holds a 96-bit mantissa and a scale of 0 to 28, so TryToDecimal fails"); - Console.WriteLine(" on either count — and ToDecimal() throws where TryToDecimal returns false:"); - - await foreach (Block block in client.StreamAsync( - @"SELECT CAST('1.25', 'Decimal(20, 2)') AS fits, - CAST('1234567890123456789012345678901.5', 'Decimal(38, 1)') AS mantissa_too_wide, - CAST('1.2345678901234567890123456789012345', 'Decimal(38, 34)') AS scale_too_deep")) - { - foreach (IColumn column in block.Columns) - { - var value = (ClickHouseTcpDecimal)column.GetValue(0); - Console.WriteLine($" {column.Name,-18} {column.TypeName,-16} TryToDecimal {value.TryToDecimal(out _),-5} {value}"); - } - } - - // Two values of different scale can be the same number, and comparison says so. - var oneDotZero = new ClickHouseTcpDecimal((Int128)10, 1); - var oneDotZeroZero = new ClickHouseTcpDecimal((Int128)100, 2); - Console.WriteLine(); - Console.WriteLine(" Equality and ordering compare the value, not the representation:"); - Console.WriteLine($" ClickHouseTcpDecimal(10, 1) == ClickHouseTcpDecimal(100, 2) {oneDotZero == oneDotZeroZero} ('{oneDotZero}' and '{oneDotZeroZero}')"); - Console.WriteLine($" FromDecimal(1.2500m) keeps the trailing zeros: '{ClickHouseTcpDecimal.FromDecimal(1.2500m)}', Scale {ClickHouseTcpDecimal.FromDecimal(1.2500m).Scale}"); - Console.WriteLine(); - Console.WriteLine(" ToString() is always the invariant fixed-point rendering with exactly Scale digits."); - Console.WriteLine(" The type implements IFormattable, but the format and the provider are ignored:"); - Console.WriteLine($" ToString(\"F3\", InvariantCulture) = '{oneDotZero.ToString("F3", CultureInfo.InvariantCulture)}' (not 1.000)"); - } - - private static async Task Floats(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. Floats, and BFloat16's missing mantissa\n"); - Console.WriteLine(" Float32 is a float and Float64 a double. BFloat16 is a float too — it is a float32 with"); - Console.WriteLine(" the low 16 mantissa bits cut off, so widening it is exact and there is nothing narrower"); - Console.WriteLine(" to hand back. What you lose is precision, on the way in:\n"); - - await foreach (Block block in client.StreamAsync($"SELECT f32, f64, bf16 FROM {TableName}")) - { - Console.WriteLine($" Float32 wrote 1.5f read {block.Column("f32")[0]}"); - Console.WriteLine($" Float64 wrote -2.25 read {block.Column("f64")[0]}"); - Console.WriteLine($" BFloat16 wrote 0.1f read {block.Column("bf16")[0]:R}"); - Console.WriteLine(" 7 stored mantissa bits, so 0.1 is not representable and the nearest value comes back."); - } - } - - private static async Task StringsAndBytes(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. String is text, FixedString(N) is bytes\n"); - - await foreach (Block block in client.StreamAsync($"SELECT text, fixed5, id, ip4, ip6 FROM {TableName}")) - { - byte[] fixedBytes = block.Column("fixed5")[0]; - Console.WriteLine($" String -> string \"{block.Column("text")[0]}\""); - Console.WriteLine($" FixedString(5) -> byte[] {Convert.ToHexString(fixedBytes)} ({fixedBytes.Length} bytes)"); - Console.WriteLine(" No decoding and no trimming: an embedded 0x00 and a byte that is not valid UTF-8"); - Console.WriteLine(" both survive, which a string could not carry."); - Console.WriteLine($" UUID -> Guid {block.Column("id")[0]}"); - Console.WriteLine($" IPv4 -> IPAddress {block.Column("ip4")[0]}"); - Console.WriteLine($" IPv6 -> IPAddress {block.Column("ip6")[0]}"); - Console.WriteLine(" One CLR type for both, told apart by AddressFamily."); - } - - await foreach (Block block in client.StreamAsync( - "SELECT toIPv4('10.0.0.1') AS four, toIPv6('10.0.0.1') AS six")) - { - Console.WriteLine(); - Console.WriteLine(" The same address in each column, and the family is what differs:"); - foreach (IColumn column in block.Columns) - { - var address = (IPAddress)column.GetValue(0)!; - Console.WriteLine($" {column.TypeName,-5} {address,-18} AddressFamily {address.AddressFamily}"); - } - - Console.WriteLine(" An IPv4 address in an IPv6 column is the mapped form, ::ffff:a.b.c.d."); - } - - Console.WriteLine(); - Console.WriteLine(" A String is decoded as UTF-8, so a String column carrying arbitrary bytes is lossy:"); - - await foreach (Block block in client.StreamAsync( - "SELECT CAST(unhex('C3A9') AS String) AS valid, CAST(unhex('FFFE') AS String) AS invalid")) - { - foreach (IColumn column in block.Columns) - { - string text = (string)column.GetValue(0); - Console.WriteLine($" {column.Name,-8} = \"{text}\" -> re-encoded {Convert.ToHexString(System.Text.Encoding.UTF8.GetBytes(text))}"); - } - - Console.WriteLine(" 0xFFFE came back as two replacement characters. Use FixedString(N) for bytes."); - } - - Console.WriteLine(); - Console.WriteLine(" An insert supplies exactly N bytes. Shorter is refused rather than padded:"); - - try - { - await client.InsertAsync( - $"INSERT INTO {TableName} (u64, fixed5) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("u64", new ulong[] { 2 }), - ClickHouseTcpColumn.Create("fixed5", new[] { new byte[] { 0x61, 0x62 } }), - }); - } - catch (ArgumentException ex) - { - Console.WriteLine($" {Wrap(ex.Message.Split(" (Parameter")[0])}"); - } - } - - private static async Task Enums(ClickHouseTcpClient client) - { - Console.WriteLine("\n6. An enum is its ordinal; the labels live in the type string\n"); - - await foreach (Block block in client.StreamAsync($"SELECT e8, e16 FROM {TableName}")) - { - foreach (IColumn column in block.Columns) - { - Console.WriteLine($" {column.Name,-4} {column.TypeName}"); - Console.WriteLine($" reads as {Describe(column.ElementType)} = {column.GetValue(0)}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" Enum8 is an Int8 ordinal and Enum16 an Int16 one, and no tier maps either back to its"); - Console.WriteLine(" label. The definition is in IColumn.TypeName, so the map is recoverable — but you parse"); - Console.WriteLine(" it, or you ask the server:"); - - object label = await client.ExecuteScalarAsync($"SELECT toString(e8) FROM {TableName} LIMIT 1"); - Console.WriteLine($" SELECT toString(e8) -> '{label}' (the server's own reverse lookup)"); - - Console.WriteLine(); - Console.WriteLine(" The POCO tier refuses a string property over an enum column rather than guessing:"); - - try - { - await foreach (EnumRow _ in client.QueryAsync($"SELECT e8 AS Colour FROM {TableName}")) - { - break; - } - } - catch (InvalidOperationException ex) - { - Console.WriteLine($" {Wrap(ex.Message)}"); - } - - Console.WriteLine(); - Console.WriteLine(" And an insert takes the ordinal, so map your own enum to its numeric value:"); - Console.WriteLine(" ClickHouseTcpColumn.Create(\"e8\", new sbyte[] { (sbyte)Colour.Green })"); - } - - private static async Task Nothing(ClickHouseTcpClient client) - { - Console.WriteLine("\n7. Nothing: the type of a value that has no type\n"); - Console.WriteLine(" The server gives an untyped NULL and an untyped empty array the Nothing type. It cannot"); - Console.WriteLine(" be a column of a table, so you only ever meet it in an expression's result:\n"); - - await foreach (Block block in client.StreamAsync("SELECT NULL AS nothing_at_all, [] AS empty_array")) - { - foreach (IColumn column in block.Columns) - { - Console.WriteLine($" {column.Name,-14} {column.TypeName,-18} reads as {Describe(column.ElementType),-10} value {Render(column.GetValue(0))}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" object is the element type because there is no value to have a type. The server refuses"); - Console.WriteLine(" to store one at all:"); - - try - { - await client.ExecuteAsync($"CREATE TABLE {TableName}_nothing (c Nothing) ENGINE = MergeTree ORDER BY tuple()"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}_nothing"); - Console.WriteLine(" accepted, which this example did not expect"); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" {FirstLine(ex.Message)}"); - } - - Console.WriteLine(); - Console.WriteLine(" So a query whose column may be Nothing wants a CAST: SELECT CAST(NULL, 'Nullable(Int32)')."); - } - - // A POCO whose property type is deliberately wrong for the column, to show what the mapping reports. - private sealed class EnumRow - { - public string Colour { get; set; } = string.Empty; - } - - // The C# spelling of a CLR type, which is how a reader will write it. - private static string Describe(Type type) => type switch - { - _ when type == typeof(byte) => "byte", - _ when type == typeof(sbyte) => "sbyte", - _ when type == typeof(ushort) => "ushort", - _ when type == typeof(short) => "short", - _ when type == typeof(uint) => "uint", - _ when type == typeof(int) => "int", - _ when type == typeof(ulong) => "ulong", - _ when type == typeof(long) => "long", - _ when type == typeof(float) => "float", - _ when type == typeof(double) => "double", - _ when type == typeof(decimal) => "decimal", - _ when type == typeof(bool) => "bool", - _ when type == typeof(string) => "string", - _ when type == typeof(byte[]) => "byte[]", - _ when type == typeof(object) => "object", - _ when type == typeof(object[]) => "object[]", - _ => type.Name, - }; - - private static string Render(object? value) => value switch - { - null => "NULL", - byte[] bytes => "0x" + Convert.ToHexString(bytes), - string text => $"\"{text}\"", - bool flag => flag ? "true" : "false", - object[] { Length: 0 } => "[]", - float single => single.ToString("R", CultureInfo.InvariantCulture), - IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), - _ => value.ToString() ?? "NULL", - }; - - private static string FirstLine(string message) - { - int newline = message.IndexOf('\n'); - string line = newline < 0 ? message : message[..newline]; - return line.StartsWith("DB::Exception: ", StringComparison.Ordinal) ? line["DB::Exception: ".Length..] : line; - } - - // Reflows a long driver message so the console output stays readable. - private static string Wrap(string message) - { - var lines = new List(); - var line = new System.Text.StringBuilder(); - foreach (string word in message.Split(' ')) - { - if (line.Length + word.Length + 1 > 88) - { - lines.Add(line.ToString()); - line.Clear(); - } - - line.Append(line.Length == 0 ? word : " " + word); - } - - lines.Add(line.ToString()); - return string.Join("\n ", lines); - } -} diff --git a/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs b/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs deleted file mode 100644 index ee46bc49d..000000000 --- a/examples/Tcp/Types/Tcp_012_DateTimeAndTimezones.cs +++ /dev/null @@ -1,462 +0,0 @@ -using System.Globalization; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// The six date and time types — Date, Date32, DateTime, DateTime64(scale), -/// Time, Time64(scale) — and the timezone model that decides what a value means. -/// -/// -/// Three facts carry the whole subject: -/// -/// -/// -/// A DateTime or DateTime64 stores an instant: a count of seconds (or of -/// 10^-scale seconds) since the Unix epoch, in UTC. A timezone in the type string changes no stored -/// byte. It decides only how that count is presented, and how a wall-clock value is turned into it. -/// -/// -/// When the type string names no timezone, the presentation timezone comes from the session_timezone -/// query setting, falling back to the server's own timezone. Section 3 measures it. -/// -/// -/// A DateTime whose Kind is Utc or Local, and any DateTimeOffset, names an -/// instant. On an insert that is lossless, because the target column's timezone is known. As a query -/// parameter it is refused, because a parameter travels as text with no timezone attached. Section 6. -/// -/// -/// -/// -/// Tcp_006 covers the block-tier mechanics of IDateTimeColumn and ITimeColumn; -/// Tcp_007 demonstrates the parameter refusal. This example is about what the values mean. -/// -/// -public static class TcpDateTimeAndTimezones -{ - private const string TableName = "example_tcp_datetime_timezones"; - private const string KindTable = "example_tcp_datetime_kinds"; - - // 2026-06-01 12:00:00 UTC. Europe/Amsterdam is +02:00 that day, Asia/Tokyo +09:00. - private const long NoonUtcSeconds = 1780315200; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await Seed(client); - await SixTypes(client); - await WhatTheWireCarries(client); - await WhereThePresentationTimezoneComesFrom(client); - await Scale(client); - await KindOnTheWritePath(client); - await KindOnTheParameterPath(client); - await TimeIsNotATimeOfDay(client); - WhatToRemember(); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {KindTable}"); - Console.WriteLine("\nDropped every table this example created."); - } - } - - private static async Task Seed(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - d Date, - d32 Date32, - dt DateTime, - dt_tz DateTime('Europe/Amsterdam'), - dt64 DateTime64(3), - dt64_tz DateTime64(9, 'Asia/Tokyo'), - t Time, - t64 Time64(3) - ) - ENGINE = MergeTree() - ORDER BY d"); - - await client.InsertAsync( - $"INSERT INTO {TableName} (d, d32, dt, dt_tz, dt64, dt64_tz, t, t64) VALUES", - new IColumn[] - { - // A Date is a day number, so it takes a DateOnly and nothing else — not a DateTime. - ClickHouseTcpColumn.Create("d", new[] { new DateOnly(2026, 6, 1) }), - ClickHouseTcpColumn.Create("d32", new[] { new DateOnly(1920, 3, 4) }), - - // Kind=Utc names the instant, which every one of these four columns then stores exactly. - ClickHouseTcpColumn.Create("dt", new[] { DateTime.UnixEpoch.AddSeconds(NoonUtcSeconds) }), - ClickHouseTcpColumn.Create("dt_tz", new[] { DateTime.UnixEpoch.AddSeconds(NoonUtcSeconds) }), - ClickHouseTcpColumn.Create("dt64", new[] { DateTime.UnixEpoch.AddSeconds(NoonUtcSeconds).AddMilliseconds(123) }), - - // A DateTime cannot hold nanoseconds, so the raw count goes in directly. Every one of these - // columns also accepts the integer the wire carries. - ClickHouseTcpColumn.Create("dt64_tz", new[] { (NoonUtcSeconds * 1_000_000_000L) + 123456789L }), - - // A Time is a count from midnight, so it takes a TimeSpan, which can also be negative or - // longer than a day. A TimeOnly cannot express either and is not accepted. - ClickHouseTcpColumn.Create("t", new[] { new TimeSpan(12, 34, 56) }), - ClickHouseTcpColumn.Create("t64", new[] { new TimeSpan(0, 12, 34, 56, 789) }), - }); - - ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); - Console.WriteLine($"Server {server.Version}, handshake timezone '{server.Timezone}'."); - Console.WriteLine($"Seeded '{TableName}' with one row of each of the six types."); - } - - private static async Task SixTypes(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. The six types, what they store, and what reads them\n"); - Console.WriteLine(" ClickHouse type IColumn The raw count Extra interface"); - Console.WriteLine(" ---------------------------- ---------- ------------------- ---------------"); - - await foreach (Block block in client.StreamAsync($"SELECT * FROM {TableName}")) - { - foreach (IColumn column in block.Columns) - { - Console.WriteLine( - $" {column.TypeName,-28} {Describe(column.ElementType),-10} {Raw(column.GetValue(0)),-19} {Extra(column)}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" Date and Date32 are the two that already read as a calendar type: a day number needs no"); - Console.WriteLine(" timezone and no scale, so DateOnly loses nothing and there is no interface to add. The"); - Console.WriteLine(" other four read as the integer the wire carried, and the interface converts it."); - } - - private static async Task WhatTheWireCarries(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. What each count counts\n"); - - await foreach (Block block in client.StreamAsync($"SELECT * FROM {TableName}")) - { - var dt = (IDateTimeColumn)block["dt"]; - var dt64 = (IDateTimeColumn)block["dt64_tz"]; - var t = (ITimeColumn)block["t"]; - - Console.WriteLine($" Date {Raw(block.Column("d")[0]),-19} days since 1970-01-01, unsigned 16-bit"); - Console.WriteLine($" Date32 {Raw(block.Column("d32")[0]),-19} days since 1970-01-01, signed 32-bit, so it reaches before the epoch"); - Console.WriteLine($" DateTime {block.Column("dt")[0],-19} seconds since the epoch, UTC"); - Console.WriteLine($" DateTime64(9) {block.Column("dt64_tz")[0],-19} nanoseconds since the epoch, UTC"); - Console.WriteLine($" Time {block.Column("t")[0],-19} seconds from midnight, signed"); - Console.WriteLine($" Time64(3) {block.Column("t64")[0],-19} milliseconds from midnight, signed"); - - Console.WriteLine(); - Console.WriteLine(" For the two DateTime families that count is a UTC instant. The timezone the column"); - Console.WriteLine(" declares changes no stored byte, only the reading:"); - Console.WriteLine($" dt {block["dt"].TypeName,-30} count {block.Column("dt")[0]} -> {Format(dt.GetDateTimeOffset(0))}"); - Console.WriteLine($" dt_tz {block["dt_tz"].TypeName,-30} count {block.Column("dt_tz")[0]} -> {Format(((IDateTimeColumn)block["dt_tz"]).GetDateTimeOffset(0))}"); - Console.WriteLine(" Same count, different offset. One instant, two presentations."); - - Console.WriteLine(); - Console.WriteLine(" A Time carries no timezone at all, because it is not an instant:"); - Console.WriteLine($" t {block["t"].TypeName,-30} Scale {t.Scale}, GetTimeSpan(0) {t.GetTimeSpan(0)}"); - Console.WriteLine($" dt64_tz {block["dt64_tz"].TypeName,-30} Scale {dt64.Scale}, TimeZone {dt64.TimeZone.Id}"); - } - } - - private static async Task WhereThePresentationTimezoneComesFrom(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. Where the presentation timezone comes from\n"); - Console.WriteLine(" Measured, not assumed. The same query runs once per session_timezone over one fixed"); - Console.WriteLine(" instant, with a bare DateTime and a DateTime('Europe/Amsterdam') side by side:\n"); - Console.WriteLine(" session_timezone DateTime count bare presented as declared presented as"); - Console.WriteLine(" -------------------- -------------- ------------------------------ ------------------------------"); - - string sql = $@"SELECT toDateTime({NoonUtcSeconds}) AS bare, - toDateTime({NoonUtcSeconds}, 'Europe/Amsterdam') AS declared"; - - foreach (string zone in new[] { string.Empty, "UTC", "Europe/Amsterdam", "Asia/Tokyo", "America/Los_Angeles" }) - { - ClickHouseTcpQueryOptions? options = zone.Length == 0 - ? null - : new ClickHouseTcpQueryOptions { Settings = new Dictionary { ["session_timezone"] = zone } }; - - await foreach (Block block in client.StreamAsync(sql, options)) - { - var bare = (IDateTimeColumn)block["bare"]; - var declared = (IDateTimeColumn)block["declared"]; - Console.WriteLine( - $" {(zone.Length == 0 ? "(not set)" : zone),-20} {block.Column("bare")[0],-14} {Format(bare.GetDateTimeOffset(0)),-30} {Format(declared.GetDateTimeOffset(0))}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" What that shows:"); - Console.WriteLine(" The stored count never moves. It is the same instant in every row."); - Console.WriteLine(" A bare DateTime is presented in the session timezone, and IDateTimeColumn.TimeZone"); - Console.WriteLine(" reports that zone."); - Console.WriteLine(" A DateTime('Europe/Amsterdam') ignores the setting entirely. The type string wins."); - Console.WriteLine(" With the setting unset, the presentation zone is the server's own — the one the"); - Console.WriteLine(" handshake reported, printed at the top of this example."); - Console.WriteLine(); - Console.WriteLine(" So the presentation timezone is: the type string's, or else session_timezone, or else the"); - Console.WriteLine(" server's. The server also sends a TimezoneUpdate packet on the wire; it is not what the"); - Console.WriteLine(" client resolves a bare column against."); - Console.WriteLine(); - Console.WriteLine(" The practical consequence: declare the timezone on any column you care about. A bare"); - Console.WriteLine(" DateTime read by two callers with different session settings gives two different"); - Console.WriteLine(" DateTimeOffsets — correctly, since they are the same instant, but a DateTime with"); - Console.WriteLine(" Kind=Unspecified taken from one of them is not comparable with the other's."); - } - - private static async Task Scale(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. Scale: DateTime64(0..9), and where .NET stops\n"); - Console.WriteLine(" The scale is how many decimal digits of a second the count carries. A .NET tick is"); - Console.WriteLine(" 100 ns, which is scale 7, so scales 8 and 9 hold digits DateTimeOffset cannot:\n"); - Console.WriteLine(" Type Raw count GetDateTimeOffset(0)"); - Console.WriteLine(" -------------------- --------------------- ------------------------------"); - - await foreach (Block block in client.StreamAsync( - @"SELECT toDateTime64('2026-06-01 12:00:00.123456789', 0, 'UTC') AS s0, - toDateTime64('2026-06-01 12:00:00.123456789', 3, 'UTC') AS s3, - toDateTime64('2026-06-01 12:00:00.123456789', 6, 'UTC') AS s6, - toDateTime64('2026-06-01 12:00:00.123456789', 7, 'UTC') AS s7, - toDateTime64('2026-06-01 12:00:00.123456789', 9, 'UTC') AS s9")) - { - foreach (IColumn column in block.Columns) - { - var instants = (IDateTimeColumn)column; - Console.WriteLine($" {column.TypeName,-20} {column.GetValue(0),-21} {instants.GetDateTimeOffset(0):yyyy-MM-dd HH:mm:ss.fffffff}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" Scale 9's last two digits (89) are gone from the DateTimeOffset and still present in the"); - Console.WriteLine(" count. Read IColumn.Values when you need them; the truncation is toward zero."); - Console.WriteLine(); - Console.WriteLine(" Time64 has the same scale range and the same limit: ITimeColumn.GetTimeSpan truncates to"); - Console.WriteLine(" ticks, and IColumn keeps the count."); - } - - private static async Task KindOnTheWritePath(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. DateTime.Kind on the way in\n"); - Console.WriteLine(" A .NET DateTime is a number plus a Kind, and the Kind is what says whether the number is"); - Console.WriteLine(" an instant or a wall-clock reading. An insert honours it, because the target column's"); - Console.WriteLine(" timezone comes from the schema the server sent, so the conversion is never a guess.\n"); - Console.WriteLine($" The host's local zone is {TimeZoneInfo.Local.Id}. Same 12:00 in each case:\n"); - - var noon = new DateTime(2026, 6, 1, 12, 0, 0); - var values = new (string What, object Value)[] - { - ("Kind=Utc", DateTime.SpecifyKind(noon, DateTimeKind.Utc)), - ("Kind=Unspecified", DateTime.SpecifyKind(noon, DateTimeKind.Unspecified)), - ("Kind=Local", DateTime.SpecifyKind(noon, DateTimeKind.Local)), - ("DateTimeOffset +05:00", new DateTimeOffset(2026, 6, 1, 12, 0, 0, TimeSpan.FromHours(5))), - }; - - foreach (string columnType in new[] { "DateTime('UTC')", "DateTime('Europe/Amsterdam')" }) - { - Console.WriteLine($" Target column {columnType}:"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {KindTable}"); - await client.ExecuteAsync($"CREATE TABLE {KindTable} (t {columnType}) ENGINE = MergeTree ORDER BY tuple()"); - - foreach ((string what, object value) in values) - { - IColumn column = value is DateTimeOffset offset - ? ClickHouseTcpColumn.Create("t", new[] { offset }) - : ClickHouseTcpColumn.Create("t", new[] { (DateTime)value }); - - await client.InsertAsync($"INSERT INTO {KindTable} (t) VALUES", new[] { column }); - - await foreach (Block block in client.StreamAsync($"SELECT t FROM {KindTable}")) - { - var stored = (IDateTimeColumn)block["t"]; - Console.WriteLine($" {what,-22} -> count {block.Column("t")[0]}, presented {Format(stored.GetDateTimeOffset(0))}"); - } - - await client.ExecuteAsync($"TRUNCATE TABLE {KindTable}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" Reading those two blocks together:"); - Console.WriteLine(" Kind=Utc and DateTimeOffset name an instant, so the count is the same whichever column"); - Console.WriteLine(" they go into. Lossless."); - Console.WriteLine(" Kind=Unspecified is a wall clock, read in the COLUMN's timezone — the count differs"); - Console.WriteLine(" between the two targets by the offset. Lossless, and it means what you want when the"); - Console.WriteLine(" value came from a source that had no timezone."); - Console.WriteLine(" Kind=Local is a wall clock read in the HOST's timezone, not the column's. Correct, and"); - Console.WriteLine(" it makes the stored value depend on where your process runs. Prefer Utc or"); - Console.WriteLine(" Unspecified in anything that is deployed more than once."); - Console.WriteLine(); - Console.WriteLine(" A read produces Kind=Utc for a zero-offset column and Kind=Unspecified otherwise, so"); - Console.WriteLine(" an insert of a read value is lossless only if the two columns share a timezone. Take"); - Console.WriteLine(" DateTimeOffset from IDateTimeColumn.GetDateTimeOffset instead, which is unambiguous."); - } - - private static async Task KindOnTheParameterPath(ClickHouseTcpClient client) - { - Console.WriteLine("\n6. The same value as a query parameter: an instant needs a declared timezone\n"); - Console.WriteLine(" A parameter does not travel as a count. It travels as text in the Query packet's"); - Console.WriteLine(" settings list, and the text carries no timezone, so the server reads it in whatever"); - Console.WriteLine(" session_timezone is in force — which section 3 just showed is not something the client"); - Console.WriteLine(" controls. An instant would therefore move silently, so it is refused:\n"); - - var noonUtc = DateTime.SpecifyKind(new DateTime(2026, 6, 1, 12, 0, 0), DateTimeKind.Utc); - - try - { - await client.ExecuteScalarAsync( - "SELECT {t:DateTime}", - new ClickHouseTcpQueryOptions { Parameters = new ClickHouseTcpParameterCollection { { "t", noonUtc } } }); - Console.WriteLine(" accepted, which this example did not expect"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" {{t:DateTime}} with Kind=Utc:"); - Console.WriteLine($" {Wrap(ex.Message.Split(" (Parameter")[0])}"); - } - - Console.WriteLine(); - Console.WriteLine(" Declaring the timezone in the placeholder makes it lossless, because the client can"); - Console.WriteLine(" then move the instant into that zone before writing the text:"); - - foreach ((string placeholder, object value, string note) in new (string, object, string)[] - { - ("{t:DateTime('UTC')}", noonUtc, "Kind=Utc, declared UTC"), - ("{t:DateTime('Asia/Tokyo')}", noonUtc, "Kind=Utc, declared Tokyo — same instant, +09:00 wall clock"), - ("{t:DateTime('UTC')}", new DateTimeOffset(2026, 6, 1, 17, 0, 0, TimeSpan.FromHours(5)), "DateTimeOffset +05:00 — the same instant again"), - ("{t:DateTime}", DateTime.SpecifyKind(new DateTime(2026, 6, 1, 12, 0, 0), DateTimeKind.Unspecified), "Kind=Unspecified — a wall clock, so no timezone is needed"), - }) - { - // session_timezone is pinned, because the last case below is read in it and a server left on its - // own default would make this comparison say something different on every machine. - object? epoch = await client.ExecuteScalarAsync( - $"SELECT toUnixTimestamp(toDateTime({placeholder}, 'UTC'))", - new ClickHouseTcpQueryOptions - { - Parameters = new ClickHouseTcpParameterCollection { { "t", value } }, - Settings = new Dictionary { ["session_timezone"] = "UTC" }, - }); - Console.WriteLine($" {placeholder,-27} -> {epoch} {note}"); - } - - Console.WriteLine(); - Console.WriteLine(" The last row is the one to notice: with Kind=Unspecified the count is whatever the"); - Console.WriteLine(" session timezone makes of 12:00, so it agrees with the others only because these queries"); - Console.WriteLine(" set session_timezone=UTC. Without that it follows the server, and the same value means a"); - Console.WriteLine(" different instant on a differently configured one. That is the ambiguity the refusal"); - Console.WriteLine(" above protects an instant from."); - Console.WriteLine(); - Console.WriteLine(" Same rule for DateTime64: {t:DateTime64(3, 'UTC')} declares one, {t:DateTime64(3)} does"); - Console.WriteLine(" not. Date, Date32, Time and Time64 have no timezone to declare, so none of this applies"); - Console.WriteLine(" to them."); - } - - private static async Task TimeIsNotATimeOfDay(ClickHouseTcpClient client) - { - Console.WriteLine("\n7. Time is a duration from midnight, not a time of day\n"); - Console.WriteLine(" The count is signed and is not reduced modulo a day, so a Time holds values no clock"); - Console.WriteLine(" face has. That is why the CLR type is TimeSpan and not TimeOnly:\n"); - Console.WriteLine(" Literal Raw count GetTimeSpan(0)"); - Console.WriteLine(" ---------------- ---------- --------------"); - - await foreach (Block block in client.StreamAsync( - @"SELECT CAST('12:34:56', 'Time') AS ordinary, - CAST('-01:30:00', 'Time') AS negative, - CAST('999:00:00', 'Time') AS past_a_day, - CAST('12:34:56.789', 'Time64(3)') AS with_millis")) - { - foreach (IColumn column in block.Columns) - { - var times = (ITimeColumn)column; - Console.WriteLine($" {column.Name,-16} {column.GetValue(0),-10} {times.GetTimeSpan(0)}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" A TimeOnly is refused on an insert for the same reason — it can express neither:"); - - await client.ExecuteAsync($"DROP TABLE IF EXISTS {KindTable}"); - await client.ExecuteAsync($"CREATE TABLE {KindTable} (t Time) ENGINE = MergeTree ORDER BY tuple()"); - - try - { - await client.InsertAsync( - $"INSERT INTO {KindTable} (t) VALUES", - new[] { ClickHouseTcpColumn.Create("t", new[] { new TimeOnly(12, 34, 56) }) }); - Console.WriteLine(" accepted, which this example did not expect"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" {Wrap(ex.Message.Split(" (Parameter")[0])}"); - } - - Console.WriteLine(); - Console.WriteLine(" Pass a TimeSpan, or the raw count as an int (Time) or a long (Time64)."); - } - - private static void WhatToRemember() - { - Console.WriteLine("\n8. What to remember\n"); - Console.WriteLine(" Declare the timezone on a DateTime or DateTime64 column you care about. Without one the"); - Console.WriteLine(" reading depends on session_timezone, which is set per query and not by you."); - Console.WriteLine(" Read instants through IDateTimeColumn.GetDateTimeOffset, not through a DateTime. An"); - Console.WriteLine(" offset is never ambiguous; a DateTime's Kind is Unspecified for any non-UTC column."); - Console.WriteLine(" Write Kind=Utc or a DateTimeOffset for an instant, Kind=Unspecified for a wall clock."); - Console.WriteLine(" Avoid Kind=Local unless the host's zone really is part of the value's meaning."); - Console.WriteLine(" A parameter that names an instant needs {t:DateTime('Zone')}. An insert does not, and"); - Console.WriteLine(" that difference is not a bug: only one of the two carries the column's timezone."); - Console.WriteLine(" Date and Date32 are DateOnly, Time and Time64 are TimeSpan, and none of the four has a"); - Console.WriteLine(" timezone at all."); - Console.WriteLine(" Keep the raw count when the scale is 8 or 9, or when the precision matters more than the"); - Console.WriteLine(" calendar type: it is what the wire carried and it truncates nothing."); - } - - private static string Describe(Type type) => type switch - { - _ when type == typeof(uint) => "uint", - _ when type == typeof(int) => "int", - _ when type == typeof(long) => "long", - _ => type.Name, - }; - - private static string Extra(IColumn column) => column switch - { - IDateTimeColumn instants => $"IDateTimeColumn (TimeZone {instants.TimeZone.Id}, Scale {instants.Scale})", - ITimeColumn times => $"ITimeColumn (Scale {times.Scale}, no timezone)", - _ => "- (already a calendar type)", - }; - - private static string Format(DateTimeOffset value) - => value.ToString("yyyy-MM-dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture); - - // The boxed wire value, rendered culture-invariantly so the output does not depend on the host. - private static string Raw(object? value) => value switch - { - DateOnly day => day.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), - IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), - null => "NULL", - _ => value.ToString() ?? "NULL", - }; - - // Reflows a long driver message so the console output stays readable. - private static string Wrap(string message) - { - var lines = new List(); - var line = new System.Text.StringBuilder(); - foreach (string word in message.Split(' ')) - { - if (line.Length + word.Length + 1 > 90) - { - lines.Add(line.ToString()); - line.Clear(); - } - - line.Append(line.Length == 0 ? word : " " + word); - } - - lines.Add(line.ToString()); - return string.Join("\n ", lines); - } -} diff --git a/examples/Tcp/Types/Tcp_013_CompositeRead.cs b/examples/Tcp/Types/Tcp_013_CompositeRead.cs deleted file mode 100644 index 4794debfe..000000000 --- a/examples/Tcp/Types/Tcp_013_CompositeRead.cs +++ /dev/null @@ -1,567 +0,0 @@ -using System.Globalization; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// Reading the composite types through their typed views: , -/// , , , -/// and — how they nest, and what the geo -/// aliases resolve to. -/// -/// -/// Two things decide how you write the pattern match. First, the view's type argument is the wire's -/// element type, not the row's: a Nullable(Int32) reads as int? but its view is -/// INullableColumn<int>, and a LowCardinality(Nullable(String)) is -/// ILowCardinalityColumn<string>. Second, a composite's child is a column in its own right, so -/// reaching into a nested composite is another pattern match rather than an index into a materialized value. -/// -/// -/// -/// Tcp_006 covers the block tier itself and IArrayColumn in particular; Tcp_010 covers -/// writing these shapes. This example is about the types. -/// -/// -public static class TcpCompositeRead -{ - private const string TableName = "example_tcp_composite_read"; - private const string NestedTable = "example_tcp_composite_read_nested"; - - private const string Columns = - "id, readings, attrs, point, named_point, score, city, nick, matrix, tagged, buckets"; - - // Geometry, the Variant over the six geo aliases, is newer than the rest of this example. - private static readonly Version GeometryFrom = new(25, 11); - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); - - try - { - await Seed(client); - await WhichViewEachCompositeOffers(client); - await MapsAndArrays(client); - await Tuples(client); - await Nulls(client); - await LowCardinalities(client); - await Nesting(client); - await NestedColumns(client); - await GeoAliases(client); - await Geometry(client, server); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {NestedTable}"); - Console.WriteLine("\nDropped every table this example created."); - } - } - - private static async Task Seed(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - id UInt64, - readings Array(Float64), - attrs Map(String, Int64), - point Tuple(Float64, Float64), - named_point Tuple(x Int32, y String), - score Nullable(Float64), - city LowCardinality(String), - nick LowCardinality(Nullable(String)), - matrix Array(Array(Int32)), - tagged Array(Tuple(Int32, String)), - buckets Map(String, Array(Int32)) - ) - ENGINE = MergeTree() - ORDER BY id"); - - await client.InsertAsync( - $"INSERT INTO {TableName} ({Columns}) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3 }), - ClickHouseTcpColumn.Create("readings", new[] { new[] { 0.5, 0.75 }, Array.Empty(), new[] { 1.5 } }), - ClickHouseTcpColumn.Create("attrs", new[] - { - new[] { new KeyValuePair("floor", 3), new KeyValuePair("room", 12) }, - Array.Empty>(), - - // The wire carries keys in order, duplicates and all, which is why a row is a pair array - // rather than a Dictionary. - new[] { new KeyValuePair("floor", 1), new KeyValuePair("floor", 2) }, - }), - ClickHouseTcpColumn.Create("point", new[] { (1.0, 2.0), (3.0, 4.0), (5.0, 6.0) }), - ClickHouseTcpColumn.Create("named_point", new[] { (10, "ten"), (20, "twenty"), (30, "thirty") }), - ClickHouseTcpColumn.Create("score", new double?[] { 1.25, null, 3.5 }), - ClickHouseTcpColumn.Create("city", new[] { "Amsterdam", "Amsterdam", "Reykjavik" }), - ClickHouseTcpColumn.Create("nick", new string?[] { "ada", null, "ada" }), - ClickHouseTcpColumn.Create("matrix", new[] { new[] { new[] { 1, 2 }, new[] { 3 } }, Array.Empty(), new[] { new[] { 4 } } }), - ClickHouseTcpColumn.Create("tagged", new[] - { - new[] { (1, "a"), (2, "b") }, - Array.Empty<(int, string)>(), - new[] { (3, "c") }, - }), - ClickHouseTcpColumn.Create("buckets", new[] - { - new[] { new KeyValuePair("evens", new[] { 2, 4 }) }, - Array.Empty>(), - new[] { new KeyValuePair("odds", new[] { 1, 3, 5 }) }, - }), - }); - - // Nested has to be created with flatten_nested = 0 to stay one column rather than becoming one - // Array(T) per field, and the client cannot build one from CLR values, so this one is seeded in SQL. - var oneColumnNested = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["flatten_nested"] = "0" }, - }; - - await client.ExecuteAsync( - $@"CREATE TABLE {NestedTable} (id UInt64, items Nested(sku String, qty UInt32)) - ENGINE = MergeTree() ORDER BY id", - oneColumnNested); - - await client.ExecuteAsync($"INSERT INTO {NestedTable} VALUES (1, [('bolt', 2), ('nut', 3)]), (2, []), (3, [('washer', 7)])"); - - Console.WriteLine($"Seeded '{TableName}' with 3 rows of every composite, and '{NestedTable}' with a Nested column."); - } - - private static async Task WhichViewEachCompositeOffers(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. What each composite reads as, and which view it offers\n"); - Console.WriteLine(" ClickHouse type One row is Pattern-matches to"); - Console.WriteLine(" -------------------------------- ------------------------------ ------------------------------"); - - await foreach (Block block in client.StreamAsync($"SELECT {Columns} FROM {TableName} ORDER BY id")) - { - foreach (IColumn column in block.Columns) - { - Console.WriteLine($" {column.TypeName,-32} {Describe(column.ElementType),-30} {View(column)}"); - } - } - - await foreach (Block block in client.StreamAsync($"SELECT items FROM {NestedTable} ORDER BY id")) - { - IColumn column = block["items"]; - Console.WriteLine($" {column.TypeName,-32} {Describe(column.ElementType),-30} {View(column)}"); - } - - Console.WriteLine(); - Console.WriteLine(" The type argument of a view is the wire's element type, which is not always the row's:"); - Console.WriteLine(" Nullable(Float64) reads double?, view INullableColumn"); - Console.WriteLine(" LowCardinality(Nullable(String)) reads string, view ILowCardinalityColumn"); - Console.WriteLine(" ITupleColumn, INestedColumn, IVariantColumn, IDynamicColumn and IQBitColumn are not"); - Console.WriteLine(" generic at all, so those five need no type argument to match on."); - } - - private static async Task MapsAndArrays(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. Map(K, V): two flat columns plus offsets\n"); - Console.WriteLine(" A Map is byte-identical to Array(Tuple(K, V)), so its view is an Array's shape with the"); - Console.WriteLine(" run split in two. Row i's entries are Offsets[i] to Offsets[i + 1] in both columns:\n"); - - await foreach (Block block in client.StreamAsync($"SELECT id, attrs FROM {TableName} ORDER BY id")) - { - if (block["attrs"] is IMapColumn attrs) - { - ReadOnlySpan offsets = attrs.Offsets; - IColumn keys = attrs.KeyColumn; - IColumn values = attrs.ValueColumn; - - Console.WriteLine($" Offsets = [{string.Join(", ", offsets.ToArray())}] ({offsets.Length} entries for {attrs.RowCount} rows)"); - Console.WriteLine($" KeyColumn = [{string.Join(", ", keys.Values.ToArray())}] RowCount {keys.RowCount}, the total entry count"); - Console.WriteLine($" ValueColumn = [{string.Join(", ", values.Values.ToArray())}]"); - Console.WriteLine(); - - for (int row = 0; row < attrs.RowCount; row++) - { - var pairs = new List(); - for (int entry = offsets[row]; entry < offsets[row + 1]; entry++) - { - pairs.Add($"{keys[entry]}={values[entry]}"); - } - - Console.WriteLine($" row {row}: {(pairs.Count == 0 ? "(empty)" : string.Join(", ", pairs))}"); - } - - Console.WriteLine(); - Console.WriteLine(" Row 2 has 'floor' twice. The two columns keep it, entry order and all, which is"); - Console.WriteLine(" what a Dictionary could not do — and the reason the materialized row is a"); - Console.WriteLine($" KeyValuePair[]: attrs[2] = [{string.Join(", ", attrs[2].Select(p => $"{p.Key}={p.Value}"))}]"); - Console.WriteLine(); - Console.WriteLine(" Taking only the keys, or only the values, therefore costs nothing:"); - Console.WriteLine($" distinct keys across every row = {string.Join(", ", keys.Values.ToArray().Distinct())}"); - } - } - } - - private static async Task Tuples(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. Tuple(...): one child column per element, and the names are metadata\n"); - - await foreach (Block block in client.StreamAsync($"SELECT point, named_point FROM {TableName} ORDER BY id")) - { - foreach (IColumn column in block.Columns) - { - var tuple = (ITupleColumn)column; - Console.WriteLine($" {column.TypeName}"); - Console.WriteLine($" Children [{string.Join(", ", tuple.Children.Select(child => $"{child.TypeName} as {Describe(child.ElementType)}"))}]"); - Console.WriteLine($" FieldNames {(tuple.FieldNames is null ? "null — the tuple carries no names at all" : "[" + string.Join(", ", tuple.FieldNames.Select(name => name ?? "(unnamed)")) + "]")}"); - Console.WriteLine($" row 0 {Render(column.GetValue(0))}"); - } - - Console.WriteLine(); - Console.WriteLine(" FieldNames is null for an unnamed tuple, so check it before enumerating; a partly"); - Console.WriteLine(" named tuple gives a list with a null entry per unnamed element."); - Console.WriteLine(); - Console.WriteLine(" The names never reach the value. A named Tuple materializes as a plain ValueTuple, so"); - Console.WriteLine(" read one element without building the pair by going through Children:"); - - var named = (ITupleColumn)block["named_point"]; - IColumn xs = (IColumn)named.Children[0]; - Console.WriteLine($" Children[0].Values = [{string.Join(", ", xs.Values.ToArray())}] (the x of every row, no ValueTuple built)"); - } - } - - private static async Task Nulls(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. Nullable(T): a null map plus a full-height inner column\n"); - - await foreach (Block block in client.StreamAsync($"SELECT score FROM {TableName} ORDER BY id")) - { - // The type argument is double, not double?: it is the inner column's element type. - if (block["score"] is INullableColumn score) - { - ReadOnlySpan nulls = score.NullMap; - IColumn inner = score.Inner; - - Console.WriteLine($" {block["score"].TypeName}, read as {Describe(block["score"].ElementType)}, view INullableColumn"); - Console.WriteLine($" NullMap = [{string.Join(", ", nulls.ToArray())}] one byte per row, 1 means NULL"); - Console.WriteLine($" Inner.Values = [{string.Join(", ", inner.Values.ToArray())}] full height, with a placeholder where the row is NULL"); - Console.WriteLine(); - Console.WriteLine(" The two are indexed by the same row number, so a null-aware read is one branch:"); - - for (int row = 0; row < score.RowCount; row++) - { - Console.WriteLine($" row {row}: {(nulls[row] != 0 ? "NULL" : inner[row].ToString(CultureInfo.InvariantCulture))}"); - } - - Console.WriteLine(); - Console.WriteLine(" Do not read Inner without the null map. The value at a NULL position is the inner"); - Console.WriteLine(" codec's placeholder, not data — here it is 0, which is a perfectly plausible score."); - } - } - } - - private static async Task LowCardinalities(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. LowCardinality(T): a dictionary plus one key per row\n"); - Console.WriteLine(" This is the view that changes what an algorithm costs. The materialized surface resolves"); - Console.WriteLine(" every row to its entry, so a million rows over a five-entry dictionary materializes a"); - Console.WriteLine(" million values; grouping on the keys instead touches each distinct value once.\n"); - - await foreach (Block block in client.StreamAsync($"SELECT city, nick FROM {TableName} ORDER BY id")) - { - foreach (string name in new[] { "city", "nick" }) - { - if (block[name] is ILowCardinalityColumn lc) - { - // The reserved slots hold the inner codec's placeholder, which for a String is the empty - // string — indistinguishable from data unless they are labelled. - string[] slots = lc.Dictionary.Values.ToArray() - .Select((value, slot) => slot < lc.ReservedSlotCount - ? (slot == 0 && lc.ReservedSlotCount == 2 ? "" : "") - : $"'{value}'") - .ToArray(); - - Console.WriteLine($" {block[name].TypeName}"); - Console.WriteLine($" Dictionary [{string.Join(", ", slots)}] RowCount {lc.Dictionary.RowCount}"); - Console.WriteLine($" Keys [{string.Join(", ", lc.Keys.ToArray())}] one per row, an index into it"); - Console.WriteLine($" ReservedSlotCount {lc.ReservedSlotCount} (so data starts at slot {lc.ReservedSlotCount})"); - - for (int row = 0; row < lc.RowCount; row++) - { - bool isNull = lc.ReservedSlotCount == 2 && lc.Keys[row] == 0; - Console.WriteLine($" row {row}: key {lc.Keys[row]} -> {(isNull ? "NULL" : $"'{lc.Dictionary[lc.Keys[row]]}'")}"); - } - } - } - } - - Console.WriteLine(); - Console.WriteLine(" ReservedSlotCount is the whole reason to have that property: the leading dictionary slots"); - Console.WriteLine(" are not data. It is 1 for a non-nullable inner (slot 0 is the inner default) and 2 for a"); - Console.WriteLine(" nullable one (slot 0 is the NULL marker, slot 1 the default). So a key of 0 means NULL for"); - Console.WriteLine(" one shape and an ordinary default for the other, and reading the property is how you tell"); - Console.WriteLine(" them apart without parsing TypeName."); - Console.WriteLine(); - Console.WriteLine(" The dictionary is per block, not per column or per table, so the same value can have a"); - Console.WriteLine(" different key in the next block of the same result."); - } - - private static async Task Nesting(ClickHouseTcpClient client) - { - Console.WriteLine("\n6. Nesting: a child is a column, so you match again\n"); - - await foreach (Block block in client.StreamAsync($"SELECT matrix, tagged, buckets FROM {TableName} ORDER BY id")) - { - Console.WriteLine($" {block["matrix"].TypeName}: an array whose Inner is another array"); - if (block["matrix"] is IArrayColumn matrix && matrix.Inner is IArrayColumn rows) - { - Console.WriteLine($" outer Offsets [{string.Join(", ", matrix.Offsets.ToArray())}]"); - Console.WriteLine($" inner Offsets [{string.Join(", ", rows.Offsets.ToArray())}]"); - Console.WriteLine($" inner InnerValues [{string.Join(", ", rows.InnerValues.ToArray())}] every element of every sub-array, flat"); - Console.WriteLine(" Two offset levels over one flat run, so a sum over the whole column needs no"); - Console.WriteLine($" array at all: total {Sum(rows.InnerValues)}"); - } - - Console.WriteLine(); - Console.WriteLine($" {block["tagged"].TypeName}: an array whose Inner is a tuple"); - if (block["tagged"] is IArrayColumn<(int, string)> tagged && tagged.Inner is ITupleColumn pairs) - { - Console.WriteLine($" Offsets [{string.Join(", ", tagged.Offsets.ToArray())}]"); - Console.WriteLine($" Inner is ITupleColumn with children [{string.Join(", ", pairs.Children.Select(c => c.TypeName))}]"); - Console.WriteLine($" Inner.Children[1].Values = [{string.Join(", ", ((IColumn)pairs.Children[1]).Values.ToArray())}] every tag, no tuple built"); - } - - Console.WriteLine(); - Console.WriteLine($" {block["buckets"].TypeName}: a map whose ValueColumn is an array"); - if (block["buckets"] is IMapColumn buckets && buckets.ValueColumn is IArrayColumn lists) - { - Console.WriteLine($" Offsets [{string.Join(", ", buckets.Offsets.ToArray())}]"); - Console.WriteLine($" KeyColumn.Values [{string.Join(", ", buckets.KeyColumn.Values.ToArray())}]"); - Console.WriteLine($" ValueColumn is IArrayColumn, Offsets [{string.Join(", ", lists.Offsets.ToArray())}], InnerValues [{string.Join(", ", lists.InnerValues.ToArray())}]"); - } - - Console.WriteLine(); - Console.WriteLine(" Composites nest as deep as the server lets them, with no materialization at any level."); - Console.WriteLine(" The one thing to know is that each match needs the child's element type spelled out,"); - Console.WriteLine(" which IColumn.ElementType on the parent tells you: Array(Array(Int32)) reports int[][],"); - Console.WriteLine(" so the outer view is IArrayColumn and the inner one IArrayColumn."); - } - } - - private static async Task NestedColumns(ClickHouseTcpClient client) - { - Console.WriteLine("\n7. Nested(...): named fields over shared offsets\n"); - Console.WriteLine(" A Nested column is byte-identical to Array(Tuple(...)) and differs only in keeping the"); - Console.WriteLine(" field names. Its view is by name rather than by position, and is not generic:\n"); - - await foreach (Block block in client.StreamAsync($"SELECT items FROM {NestedTable} ORDER BY id")) - { - if (block["items"] is INestedColumn items) - { - Console.WriteLine($" {block["items"].TypeName}"); - Console.WriteLine($" FieldCount {items.FieldCount}"); - Console.WriteLine($" FieldNames [{string.Join(", ", items.FieldNames)}]"); - Console.WriteLine($" Offsets [{string.Join(", ", items.Offsets.ToArray())}] shared by every field"); - - var skus = (IColumn)items.GetField("sku"); - var quantities = (IColumn)items.GetField("qty"); - Console.WriteLine($" GetField(\"sku\").Values [{string.Join(", ", skus.Values.ToArray())}]"); - Console.WriteLine($" GetField(\"qty\").Values [{string.Join(", ", quantities.Values.ToArray())}]"); - Console.WriteLine(" GetField(int) takes the same field by position."); - Console.WriteLine(); - - ReadOnlySpan offsets = items.Offsets; - for (int row = 0; row < items.RowCount; row++) - { - var entries = new List(); - for (int entry = offsets[row]; entry < offsets[row + 1]; entry++) - { - entries.Add($"{skus[entry]} x{quantities[entry]}"); - } - - Console.WriteLine($" row {row}: {(entries.Count == 0 ? "(empty)" : string.Join(", ", entries))}"); - } - - Console.WriteLine(); - Console.WriteLine($" The materialized row is an object[][] — one object[] per entry, boxed, so the"); - Console.WriteLine($" field columns are the way to read it: items.GetValue(0) = {Render(block["items"].GetValue(0))}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" A Nested column only exists as one column when the table was created with"); - Console.WriteLine(" flatten_nested = 0. On the default setting the server turns Nested(a T, b U) into an"); - Console.WriteLine(" Array(T) named a and an Array(U) named b, and this view never appears."); - } - - private static async Task GeoAliases(ClickHouseTcpClient client) - { - Console.WriteLine("\n8. The geo aliases resolve to structures you have already seen\n"); - Console.WriteLine(" Each is a name for a shape built out of Tuple and Array, and the wire header carries the"); - Console.WriteLine(" alias rather than the structure — so TypeName is the alias, and the view is the"); - Console.WriteLine(" structure's:\n"); - Console.WriteLine(" TypeName One row is Pattern-matches to"); - Console.WriteLine(" --------------- ---------------------------------- ----------------------------------"); - - const string geoSql = @" - SELECT CAST((1.0, 2.0), 'Point') AS p, - CAST([(0.0, 0.0), (1.0, 0.0), (1.0, 1.0)], 'Ring') AS r, - CAST([(0.0, 0.0), (1.0, 1.0)], 'LineString') AS ls, - CAST([[(0.0, 0.0), (1.0, 0.0), (1.0, 1.0)]], 'Polygon') AS pg, - CAST([[(0.0, 0.0), (1.0, 1.0)]], 'MultiLineString') AS mls, - CAST([[[(0.0, 0.0), (1.0, 0.0), (1.0, 1.0)]]], 'MultiPolygon') AS mp"; - - await foreach (Block block in client.StreamAsync(geoSql)) - { - foreach (IColumn column in block.Columns) - { - Console.WriteLine($" {column.TypeName,-15} {Describe(column.ElementType),-34} {View(column)}"); - } - - Console.WriteLine(); - Console.WriteLine(" Point is a Tuple(Float64, Float64) and the rest are arrays over it, so a Ring's"); - Console.WriteLine(" coordinates are reachable as two flat columns without any tuple being built:"); - - if (block["r"] is IArrayColumn<(double, double)> ring && ring.Inner is ITupleColumn coordinates) - { - var longitudes = (IColumn)coordinates.Children[0]; - var latitudes = (IColumn)coordinates.Children[1]; - Console.WriteLine($" Offsets [{string.Join(", ", ring.Offsets.ToArray())}]"); - Console.WriteLine($" Children[0] [{string.Join(", ", longitudes.Values.ToArray())}]"); - Console.WriteLine($" Children[1] [{string.Join(", ", latitudes.Values.ToArray())}]"); - Console.WriteLine($" row 0 {Render(block["r"].GetValue(0))}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" The point to carry over from the HTTP driver: a coordinate pair here is a ValueTuple,"); - Console.WriteLine(" where that one builds a System.Tuple. So (double, double) and not Tuple,"); - Console.WriteLine(" and .Item1 / .Item2 on a struct rather than on a class."); - Console.WriteLine(); - Console.WriteLine(" Ring and LineString are distinct types to the server and the same structure to this"); - Console.WriteLine(" client, as are Polygon and MultiLineString. Only the name tells them apart."); - } - - private static async Task Geometry(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) - { - Console.WriteLine("\n9. Geometry is the one alias that is not a nested array\n"); - - if (server.Version < GeometryFrom) - { - Console.WriteLine($" Skipped: needs ClickHouse {GeometryFrom} or newer, this server is {server.Version}."); - return; - } - - Console.WriteLine(" It names a Variant over the six above, so one column holds rows of different shapes. The"); - Console.WriteLine(" header carries only 'Geometry', so the client expands the alternatives itself, in the"); - Console.WriteLine(" server's own name-sorted discriminator order:\n"); - - const string sql = @" - SELECT g FROM (SELECT arrayJoin([ - CAST(CAST((1.0, 2.0), 'Point'), 'Geometry'), - CAST(CAST([(1.0, 2.0), (3.0, 4.0)], 'LineString'), 'Geometry'), - CAST(CAST([[[(0.0, 0.0), (1.0, 0.0), (1.0, 1.0)]]], 'MultiPolygon'), 'Geometry')]) AS g)"; - - await foreach (Block block in client.StreamAsync(sql)) - { - if (block["g"] is IVariantColumn geometry) - { - Console.WriteLine($" {block["g"].TypeName}, read as {Describe(block["g"].ElementType)}, view IVariantColumn"); - Console.WriteLine($" TypeCount {geometry.TypeCount}"); - Console.WriteLine($" Discriminators [{string.Join(", ", geometry.Discriminators.ToArray())}]"); - Console.WriteLine($" LocalIndices [{string.Join(", ", geometry.LocalIndices.ToArray())}]"); - Console.WriteLine(); - Console.WriteLine(" Alternative order: 0 LineString, 1 MultiLineString, 2 MultiPolygon, 3 Point,"); - Console.WriteLine(" 4 Polygon, 5 Ring. GetTypeColumn names the shape of each row:"); - - for (int row = 0; row < geometry.RowCount; row++) - { - IColumn child = geometry.GetTypeColumn(geometry.Discriminators[row]); - Console.WriteLine($" row {row}: discriminator {geometry.Discriminators[row]} -> {child.TypeName,-14} value {Render(block["g"].GetValue(row))}"); - } - } - } - - Console.WriteLine(); - Console.WriteLine(" Tcp_014 covers IVariantColumn properly, including the NULL discriminator and how to"); - Console.WriteLine(" dispatch on it without boxing."); - } - - private static double Sum(ReadOnlySpan values) - { - double total = 0; - foreach (int value in values) - { - total += value; - } - - return total; - } - - // Which of the block tier's typed views a column offers, found by pattern-matching rather than by reading - // TypeName. The generic ones each need their element type spelled out, which is what makes this list long. - private static string View(IColumn column) => column switch - { - IVariantColumn => "IVariantColumn", - INestedColumn => "INestedColumn", - ITupleColumn => "ITupleColumn", - IMapColumn => "IMapColumn", - IMapColumn => "IMapColumn", - INullableColumn => "INullableColumn", - ILowCardinalityColumn => "ILowCardinalityColumn", - IArrayColumn => "IArrayColumn", - IArrayColumn => "IArrayColumn", - IArrayColumn<(int, string)> => "IArrayColumn<(int, string)>", - IArrayColumn<(double, double)> => "IArrayColumn<(double, double)>", - IArrayColumn<(double, double)[]> => "IArrayColumn<(double, double)[]>", - IArrayColumn<(double, double)[][]> => "IArrayColumn<(double, double)[][]>", - _ => "- (no composite view)", - }; - - private static string Describe(Type type) - { - if (type.IsArray) - { - return Describe(type.GetElementType()!) + "[]"; - } - - if (type.IsGenericType) - { - Type definition = type.GetGenericTypeDefinition(); - string[] arguments = type.GetGenericArguments().Select(Describe).ToArray(); - if (definition == typeof(Nullable<>)) - { - return arguments[0] + "?"; - } - - if (definition.FullName?.StartsWith("System.ValueTuple`", StringComparison.Ordinal) == true) - { - return "(" + string.Join(", ", arguments) + ")"; - } - - string name = definition.Name[..definition.Name.IndexOf('`')]; - return $"{name}<{string.Join(", ", arguments)}>"; - } - - return type switch - { - _ when type == typeof(byte) => "byte", - _ when type == typeof(int) => "int", - _ when type == typeof(uint) => "uint", - _ when type == typeof(long) => "long", - _ when type == typeof(ulong) => "ulong", - _ when type == typeof(double) => "double", - _ when type == typeof(string) => "string", - _ when type == typeof(object) => "object", - _ => type.Name, - }; - } - - private static string Render(object? value) => value switch - { - null => "NULL", - string text => $"'{text}'", - System.Collections.IEnumerable items => "[" + string.Join(", ", items.Cast().Select(Render)) + "]", - IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), - _ => value.ToString() ?? "NULL", - }; -} diff --git a/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs b/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs deleted file mode 100644 index 1d7c8676d..000000000 --- a/examples/Tcp/Types/Tcp_014_VariantDynamicJson.cs +++ /dev/null @@ -1,431 +0,0 @@ -using System.Globalization; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// The three types whose value is not decided by the type string: Variant(T1, ..., Tn), Dynamic and -/// JSON. -/// -/// -/// Variant and Dynamic are discriminated unions. Both read as IColumn<object>, which -/// loses the static type and boxes every row whose alternative is a value type, and both expose a columnar view -/// instead — a per-row discriminator plus one typed child column per alternative. They differ in where the -/// alternative list comes from: a Variant declares it in the type string, a Dynamic discovers it per -/// block and reports it as -/// . They also differ in how NULL is marked, which is the one detail that -/// will bite you. -/// -/// -/// -/// JSON is a different problem. This client reads and writes it only in the String serialization -/// (version 1), so a value is its compact JSON text. That works in both directions, but the server parses -/// what you write into real paths and re-renders on the way out, so the text you get back is not the text you -/// sent. Section 5 shows exactly what changes. -/// -/// -public static class TcpVariantDynamicJson -{ - private const string VariantTable = "example_tcp_variant"; - private const string DynamicTable = "example_tcp_dynamic"; - private const string JsonTable = "example_tcp_json"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await Seed(client); - await Variants(client); - await Dynamics(client); - await TheTwoCompared(client); - await JsonIsText(client); - await JsonNormalization(client); - } - finally - { - foreach (string table in new[] { VariantTable, DynamicTable, JsonTable }) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); - } - - Console.WriteLine("\nDropped every table this example created."); - } - } - - private static async Task Seed(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {VariantTable}"); - await client.ExecuteAsync($@" - CREATE TABLE {VariantTable} (id UInt64, v Variant(String, UInt64, Array(Int32))) - ENGINE = MergeTree() ORDER BY id"); - - await client.ExecuteAsync($"DROP TABLE IF EXISTS {DynamicTable}"); - await client.ExecuteAsync($@" - CREATE TABLE {DynamicTable} (id UInt64, d Dynamic) - ENGINE = MergeTree() ORDER BY id"); - - await client.ExecuteAsync($"DROP TABLE IF EXISTS {JsonTable}"); - await client.ExecuteAsync($@" - CREATE TABLE {JsonTable} (id UInt64, doc JSON) - ENGINE = MergeTree() ORDER BY id"); - - // Both union types are written from an IColumn: one row per value, of whichever CLR type the - // chosen alternative takes, and null for a NULL row. - await client.InsertAsync( - $"INSERT INTO {VariantTable} (id, v) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3, 4, 5 }), - ClickHouseTcpColumn.Create("v", new object?[] { 42UL, "hi", null, new[] { 1, 2 }, 7UL }), - }); - - await client.InsertAsync( - $"INSERT INTO {DynamicTable} (id, d) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3, 4 }), - ClickHouseTcpColumn.Create("d", new object?[] { 42UL, "hi", null, 1.5 }), - }); - - Console.WriteLine($"Seeded '{VariantTable}' (5 rows), '{DynamicTable}' (4 rows) and '{JsonTable}'."); - } - - private static async Task Variants(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. Variant: the alternatives are declared, and the server sorts them\n"); - - await foreach (Block block in client.StreamAsync($"SELECT v FROM {VariantTable} ORDER BY id")) - { - IColumn column = block["v"]; - Console.WriteLine($" Declared as Variant(String, UInt64, Array(Int32))"); - Console.WriteLine($" Header says {column.TypeName}"); - Console.WriteLine(" The server canonicalizes the alternatives into name-sorted order, and that order is"); - Console.WriteLine(" the discriminator order. So read it from TypeName, never from what you declared.\n"); - - if (column is IVariantColumn variant) - { - Console.WriteLine($" TypeCount {variant.TypeCount}"); - Console.WriteLine($" Discriminators [{string.Join(", ", variant.Discriminators.ToArray())}] one byte per row"); - Console.WriteLine($" LocalIndices [{string.Join(", ", variant.LocalIndices.ToArray())}] -1 for a NULL row"); - Console.WriteLine($" IVariantColumn.NullDiscriminator = {IVariantColumn.NullDiscriminator} a fixed sentinel, not TypeCount"); - Console.WriteLine(); - Console.WriteLine(" One child column per alternative, holding only the rows that chose it:"); - - for (int discriminator = 0; discriminator < variant.TypeCount; discriminator++) - { - IColumn child = variant.GetTypeColumn(discriminator); - Console.WriteLine($" {discriminator} {child.TypeName,-16} {child.RowCount} row(s)"); - } - - Console.WriteLine(); - Console.WriteLine(" Row i's value is GetTypeColumn(Discriminators[i])[LocalIndices[i]], so dispatch"); - Console.WriteLine(" once per alternative and read the child typed rather than boxed:"); - Console.WriteLine(); - - // The typed children are bound once, outside the row loop. Nothing here boxes. - var strings = (IColumn)variant.GetTypeColumn(1); - var numbers = (IColumn)variant.GetTypeColumn(2); - var lists = (IColumn)variant.GetTypeColumn(0); - ReadOnlySpan discriminators = variant.Discriminators; - ReadOnlySpan local = variant.LocalIndices; - - for (int row = 0; row < column.RowCount; row++) - { - byte discriminator = discriminators[row]; - string reading = discriminator == IVariantColumn.NullDiscriminator - ? "NULL" - : discriminator switch - { - 0 => $"Array(Int32) [{string.Join(", ", lists[local[row]])}]", - 1 => $"String '{strings[local[row]]}'", - 2 => $"UInt64 {numbers[local[row]]}", - _ => "?", - }; - - Console.WriteLine($" row {row}: discriminator {discriminator,3}, local {local[row],2} -> {reading}"); - } - - Console.WriteLine(); - Console.WriteLine(" Passing NullDiscriminator to GetTypeColumn throws — it selects no column — so"); - Console.WriteLine(" guard for it before the call, as the loop above does."); - - try - { - _ = variant.GetTypeColumn(IVariantColumn.NullDiscriminator); - } - catch (IndexOutOfRangeException) - { - Console.WriteLine($" GetTypeColumn({IVariantColumn.NullDiscriminator}) -> IndexOutOfRangeException"); - } - } - - Console.WriteLine(); - Console.WriteLine($" The materialized surface is IColumn: ElementType is {column.ElementType.Name}, so"); - Console.WriteLine(" the static type is gone and a value-type alternative is boxed. A String or an Array is"); - Console.WriteLine(" already a reference, so it costs nothing beyond the object[] the caller sees:"); - for (int row = 0; row < column.RowCount; row++) - { - object? value = column.GetValue(row); - Console.WriteLine($" GetValue({row}) -> {(value is null ? "null" : $"{Describe(value.GetType())} {Render(value)}")}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" The server can tell you the same thing in SQL, which is worth knowing for a query you"); - Console.WriteLine(" are debugging:"); - - await foreach (object[] row in client.QueryAsync( - $"SELECT id, variantType(v) FROM {VariantTable} ORDER BY id")) - { - Console.WriteLine($" row {row[0]}: variantType(v) ordinal {row[1]}"); - } - - Console.WriteLine(" variantType returns an Enum8 whose type string spells the whole mapping — and which"); - Console.WriteLine(" this client reads as the bare ordinal, as Tcp_011 section 6 explains."); - } - - private static async Task Dynamics(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. Dynamic: the alternatives are discovered, and they are named\n"); - - await foreach (Block block in client.StreamAsync($"SELECT d FROM {DynamicTable} ORDER BY id")) - { - IColumn column = block["d"]; - Console.WriteLine($" {column.TypeName} — the type string says nothing about what is in it.\n"); - - if (column is IDynamicColumn dynamicColumn) - { - Console.WriteLine($" TypeCount {dynamicColumn.TypeCount}"); - Console.WriteLine($" TypeNames [{string.Join(", ", dynamicColumn.TypeNames)}] read off the wire, in discriminator order"); - Console.WriteLine($" Discriminators [{string.Join(", ", dynamicColumn.Discriminators.ToArray())}] ints here, not bytes"); - Console.WriteLine($" LocalIndices [{string.Join(", ", dynamicColumn.LocalIndices.ToArray())}]"); - Console.WriteLine(); - Console.WriteLine($" NULL is marked with TypeCount ({dynamicColumn.TypeCount}), one past the last type — there is no"); - Console.WriteLine(" fixed sentinel, because the type list is per block rather than declared."); - Console.WriteLine(); - - ReadOnlySpan discriminators = dynamicColumn.Discriminators; - ReadOnlySpan local = dynamicColumn.LocalIndices; - - for (int row = 0; row < column.RowCount; row++) - { - int discriminator = discriminators[row]; - if (discriminator == dynamicColumn.TypeCount) - { - Console.WriteLine($" row {row}: discriminator {discriminator} -> NULL"); - continue; - } - - IColumn child = dynamicColumn.GetTypeColumn(discriminator); - Console.WriteLine( - $" row {row}: discriminator {discriminator} -> {child.TypeName,-8} ({Describe(child.ElementType)}) value {Render(child.GetValue(local[row]))}"); - } - - Console.WriteLine(); - Console.WriteLine(" TypeNames is what makes typed reading possible: the name tells you what to cast a"); - Console.WriteLine(" child to, so a caller can bind IColumn per alternative without inspecting a"); - Console.WriteLine(" single value."); - } - } - - Console.WriteLine(); - Console.WriteLine(" Because the type set is per block, the same value can land on a different discriminator"); - Console.WriteLine(" in the next block of the same result. Read TypeNames inside the loop, not once."); - Console.WriteLine(); - Console.WriteLine(" The client infers the ClickHouse type of each written value from its CLR type, so what"); - Console.WriteLine(" went in as a ulong came back as UInt64 and a double as Float64. A value whose CLR type"); - Console.WriteLine(" has no ClickHouse counterpart cannot be written into a Dynamic at all."); - } - - private static async Task TheTwoCompared(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. Variant against Dynamic, in one table\n"); - Console.WriteLine(" Variant(...) Dynamic"); - Console.WriteLine(" --------------- ---------------------------- ------------------------------"); - Console.WriteLine(" alternatives declared in the type string discovered per block"); - Console.WriteLine(" the list parse it out of TypeName IDynamicColumn.TypeNames"); - Console.WriteLine(" Discriminators ReadOnlySpan ReadOnlySpan"); - Console.WriteLine($" NULL is marked {IVariantColumn.NullDiscriminator} (NullDiscriminator) TypeCount"); - Console.WriteLine(" NULL LocalIndex -1 -1"); - Console.WriteLine(" a row not in the rejected by the server widens the type set"); - Console.WriteLine(" alternative list"); - Console.WriteLine(); - Console.WriteLine(" The asymmetry worth remembering: a Variant tells you nothing about its alternatives"); - Console.WriteLine(" through the interface, and a Geometry column (Tcp_013 section 9) does not even carry"); - Console.WriteLine(" them in its type string. So for a Variant, hard-code the order you declared and check it"); - Console.WriteLine(" against TypeName; for a Dynamic, read TypeNames."); - - Console.WriteLine(); - Console.WriteLine(" dynamicType() is the Dynamic counterpart of variantType(), and unlike it returns the"); - Console.WriteLine(" name rather than an ordinal:"); - - await foreach (object[] row in client.QueryAsync( - $"SELECT id, dynamicType(d) FROM {DynamicTable} ORDER BY id")) - { - Console.WriteLine($" row {row[0]}: dynamicType(d) = '{row[1]}'"); - } - - Console.WriteLine(" 'None' is the NULL row. It is not one of the TypeNames."); - } - - private static async Task JsonIsText(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. JSON: one serialization, and it is text\n"); - Console.WriteLine(" ClickHouse can send a JSON column in several encodings. The per-path binary ones split"); - Console.WriteLine(" the column into one sub-column per JSON path; this client decodes none of them. It reads"); - Console.WriteLine(" and writes only the String serialization, version 1, where a value is its JSON text:\n"); - - await foreach (Block block in client.StreamAsync( - @"SELECT CAST('{""a"": 1}', 'JSON') AS plain, - CAST('{""a"": 1, ""z"": ""s""}', 'JSON(a UInt32)') AS typed_path, - CAST(NULL, 'Nullable(JSON)') AS maybe, - CAST(['{""a"":1}', '{""b"":2}'], 'Array(JSON)') AS several")) - { - foreach (IColumn column in block.Columns) - { - Console.WriteLine($" {column.Name,-11} {column.TypeName,-16} reads as {Describe(column.ElementType),-9} {Render(column.GetValue(0))}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" Every spelling of the type is the same String column, so JSON(a UInt32) and"); - Console.WriteLine(" JSON(max_dynamic_paths=8) need no special handling — the arguments ride in TypeName only."); - Console.WriteLine(" Under a composite the version marker comes first, then the composite's own framing."); - Console.WriteLine(); - Console.WriteLine(" Reading needs the query setting output_format_native_write_json_as_string = 1. The client"); - Console.WriteLine(" sets it on every operation, so this is only your problem if you override it:"); - - try - { - var withoutTheSetting = new ClickHouseTcpQueryOptions - { - Settings = new Dictionary { ["output_format_native_write_json_as_string"] = "0" }, - }; - - // Drained rather than broken out of: the throw is what this demonstrates, and stopping early - // would discard the connection on the way to it. - await foreach (Block _ in client.StreamAsync(@"SELECT CAST('{""a"":1}', 'JSON') AS j", withoutTheSetting)) - { - } - - Console.WriteLine(" accepted, which this example did not expect"); - } - catch (ClickHouseTcpProtocolException ex) - { - Console.WriteLine($" {Wrap(ex.Message)}"); - } - - Console.WriteLine(); - Console.WriteLine(" Writing needs no setting at all: the version marker the client writes tells the server"); - Console.WriteLine(" which encoding it is reading, so version 1 makes it parse the text server-side — into a"); - Console.WriteLine(" JSON(a UInt32) column's typed paths as readily as into an untyped one."); - } - - private static async Task JsonNormalization(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. Text in is not text out\n"); - Console.WriteLine(" A JSON value is parsed into paths and re-rendered, never stored verbatim. So a round trip"); - Console.WriteLine(" through a JSON column is lossy in a way a String column would not be. Written and read"); - Console.WriteLine(" back, unchanged in between:\n"); - - var documents = new (string Text, string What)[] - { - ("{\"b\": 1, \"a\": 2}", "keys are sorted, ordinally"), - ("{ \"x\" : 1 , \"y\": 2 }", "whitespace is dropped"), - ("{\"a\": 1.500, \"b\": 1e3, \"c\": -0.0}", "numbers are re-rendered canonically"), - ("{\"n\": null, \"empty\": {}}", "a JSON null and an empty object contribute no path"), - ("{\"when\": \"2026-06-01T12:00:00Z\"}", "a string the server reads as a DateTime is re-formatted"), - ("{\"B\": 1, \"a\": 2}", "ordinal sorting puts every capital before every lower case"), - }; - - await client.InsertAsync( - $"INSERT INTO {JsonTable} (id, doc) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", Enumerable.Range(0, documents.Length).Select(i => (ulong)i).ToArray()), - ClickHouseTcpColumn.Create("doc", documents.Select(document => document.Text).ToArray()), - }); - - int index = 0; - await foreach (object[] row in client.QueryAsync($"SELECT id, doc FROM {JsonTable} ORDER BY id")) - { - (string text, string what) = documents[index++]; - Console.WriteLine($" {what}"); - Console.WriteLine($" in {text}"); - Console.WriteLine($" out {row[1]}"); - } - - Console.WriteLine(); - Console.WriteLine(" The DateTime row is the one that catches people. \"2026-06-01T12:00:00Z\" was inferred to"); - Console.WriteLine(" be a DateTime path, and a DateTime renders in ClickHouse's own format, so the T and the Z"); - Console.WriteLine(" are gone. The value is not corrupted — but it is no longer the string you wrote, and a"); - Console.WriteLine(" consumer parsing it as ISO 8601 will fail."); - Console.WriteLine(); - Console.WriteLine(" You can see what the server decided each path was:"); - - await foreach (object[] row in client.QueryAsync( - $"SELECT id, toString(JSONAllPathsWithTypes(doc)) FROM {JsonTable} ORDER BY id")) - { - Console.WriteLine($" row {row[0]}: {row[1]}"); - } - - Console.WriteLine(); - Console.WriteLine(" What to do about it:"); - Console.WriteLine(" Do not compare the text you wrote with the text you read. Compare the paths, or the"); - Console.WriteLine(" values at a path, which is what the server can be asked for."); - Console.WriteLine(" Store a timestamp as a real DateTime64 column, not inside a JSON string."); - Console.WriteLine(" Use a String column when you need the bytes back exactly — a JSON column is a set of"); - Console.WriteLine(" typed paths that happens to be spelled as text on this transport."); - } - - private static string Describe(Type type) - { - if (type.IsArray) - { - return Describe(type.GetElementType()!) + "[]"; - } - - return type switch - { - _ when type == typeof(int) => "int", - _ when type == typeof(uint) => "uint", - _ when type == typeof(long) => "long", - _ when type == typeof(ulong) => "ulong", - _ when type == typeof(double) => "double", - _ when type == typeof(string) => "string", - _ when type == typeof(object) => "object", - _ => type.Name, - }; - } - - private static string Render(object? value) => value switch - { - null => "NULL", - string text => $"\"{text}\"", - System.Collections.IEnumerable items => "[" + string.Join(", ", items.Cast().Select(Render)) + "]", - IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), - _ => value.ToString() ?? "NULL", - }; - - // Reflows a long driver message so the console output stays readable. - private static string Wrap(string message) - { - var lines = new List(); - var line = new System.Text.StringBuilder(); - foreach (string word in message.Split(' ')) - { - if (line.Length + word.Length + 1 > 88) - { - lines.Add(line.ToString()); - line.Clear(); - } - - line.Append(line.Length == 0 ? word : " " + word); - } - - lines.Add(line.ToString()); - return string.Join("\n ", lines); - } -} diff --git a/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs b/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs deleted file mode 100644 index 28937f2cb..000000000 --- a/examples/Tcp/Types/Tcp_015_QBitVectorSearch.cs +++ /dev/null @@ -1,491 +0,0 @@ -using System.Globalization; -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// QBit(T, N) and : the one type whose whole point is its storage layout. -/// -/// -/// A QBit row is an N-element vector, but the elements of a row are not stored together. The column -/// holds one bit plane per bit position of the element type, and a plane carries that one bit of every -/// element of every row. So the most significant bits of a whole column sit contiguously, which is what lets a -/// distance be computed at reduced precision by reading only the top few planes — the server's -/// L2DistanceTransposed(vector, query, precision) does exactly that, and -/// is the same access from the client. -/// -/// -/// -/// The default IColumn<T> view undoes the transposition and hands back a float[] (or -/// double[]) per row, which is convenient and throws away the only reason to use the type. This example is -/// about the planes. examples/Http/DataTypes/Vector_001_QBitSimilaritySearch.cs covers the server-side -/// search, which the HTTP transport can do just as well. -/// -/// -public static class TcpQBitVectorSearch -{ - private const string TableName = "example_tcp_qbit"; - private const string WideTable = "example_tcp_qbit_wide"; - - // QBit arrived in 25.10 with limitations, so the driver's own suites gate it at 25.11 and so does this. - private static readonly Version QBitFrom = new(25, 11); - - // Int8 elements and the strided QBit(T, N, stride) form both need a newer server. - private static readonly Version StridedAndInt8From = new(26, 7); - - private static readonly (string Word, float[] Vector)[] Corpus = - { - ("apple", new[] { 0.9f, 0.1f, 0.8f, 0.2f, 0.7f }), - ("banana", new[] { 0.85f, 0.15f, 0.75f, 0.25f, 0.65f }), - ("orange", new[] { 0.88f, 0.12f, 0.78f, 0.22f, 0.68f }), - ("dog", new[] { 0.1f, 0.9f, 0.2f, 0.8f, 0.3f }), - ("horse", new[] { 0.15f, 0.85f, 0.25f, 0.75f, 0.35f }), - ("cat", new[] { 0.12f, 0.88f, 0.22f, 0.78f, 0.32f }), - }; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - ClickHouseTcpServerInfo server = await client.GetServerInfoAsync(); - - if (server.Version < QBitFrom) - { - Console.WriteLine($"QBit needs ClickHouse {QBitFrom} or newer, and this server is {server.Version}."); - Console.WriteLine("Nothing here runs on it: the CREATE TABLE is the first thing that would fail."); - return; - } - - try - { - await Seed(client); - await TheGeometry(client); - await ReadingAPlane(client); - await ByteOrderWithinABitmap(client); - await ReducedPrecision(client); - await ElementTypes(client, server); - await Strided(client, server); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {WideTable}"); - Console.WriteLine("\nDropped every table this example created."); - } - } - - private static async Task Seed(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} (word String, vec QBit(Float32, 5)) - ENGINE = MergeTree() ORDER BY word"); - - // A QBit column is written from one float[] per row. The client transposes it into planes. - await client.InsertAsync( - $"INSERT INTO {TableName} (word, vec) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("word", Corpus.Select(entry => entry.Word).ToArray()), - ClickHouseTcpColumn.Create("vec", Corpus.Select(entry => entry.Vector).ToArray()), - }); - - Console.WriteLine($"Seeded '{TableName}' with {Corpus.Length} words as QBit(Float32, 5) embeddings."); - } - - private static async Task TheGeometry(ClickHouseTcpClient client) - { - Console.WriteLine("\n1. What the column reports about its layout\n"); - - await foreach (Block block in client.StreamAsync($"SELECT vec FROM {TableName} ORDER BY word")) - { - IColumn column = block["vec"]; - Console.WriteLine($" {column.TypeName}, reads as {Describe(column.ElementType)}, {column.RowCount} rows\n"); - - if (column is IQBitColumn qbit) - { - Console.WriteLine($" Dimension {qbit.Dimension} the N of QBit(T, N) — elements per vector"); - Console.WriteLine($" BitWidth {qbit.BitWidth} the stored element's bit width, so the number of planes"); - Console.WriteLine($" Stride {qbit.Stride} elements one group of planes covers"); - Console.WriteLine($" GroupCount {qbit.GroupCount} Dimension / Stride"); - Console.WriteLine($" BytesPerRow {qbit.BytesPerRow} ceil(Stride / 8) — one row's bitmap within one plane"); - Console.WriteLine(); - Console.WriteLine(" The body is plane-major and every row is the same width, so its size is exact:"); - - int body = qbit.BitWidth * qbit.RowCount * qbit.BytesPerRow; - int flat = qbit.Dimension * (qbit.BitWidth / 8) * qbit.RowCount; - Console.WriteLine($" BitWidth * RowCount * BytesPerRow = {qbit.BitWidth} * {qbit.RowCount} * {qbit.BytesPerRow} = {body} bytes"); - Console.WriteLine($" the same values as {qbit.Dimension} Float32 per row = {flat} bytes"); - Console.WriteLine(); - Console.WriteLine($" The extra is padding. A plane's row is a whole number of bytes, so {qbit.BytesPerRow * 8} bit slots"); - Console.WriteLine($" carry {qbit.Stride} elements and {(qbit.BytesPerRow * 8) - qbit.Stride} slots go unused in every one of the {qbit.BitWidth} planes. A Stride"); - Console.WriteLine($" that is a multiple of 8 wastes nothing; {qbit.Stride} is not, so this column is the wider one."); - Console.WriteLine(); - Console.WriteLine(" BitWidth is the width of the STORED element, not of the CLR one: a"); - Console.WriteLine(" QBit(BFloat16, N) has 16 planes and still reads as float[]. Section 5."); - } - } - } - - private static async Task ReadingAPlane(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. A plane is one bit of every element of every row\n"); - - await foreach (Block block in client.StreamAsync($"SELECT word, vec FROM {TableName} ORDER BY word")) - { - if (block["vec"] is IQBitColumn qbit) - { - var words = (IColumn)block["word"]; - Console.WriteLine($" GetPlane(bit) returns RowCount * BytesPerRow = {qbit.RowCount} * {qbit.BytesPerRow} = {qbit.RowCount * qbit.BytesPerRow} bytes."); - Console.WriteLine($" Row r's bitmap is the slice [r * BytesPerRow, (r + 1) * BytesPerRow)."); - Console.WriteLine(); - Console.WriteLine($" bit is the SIGNIFICANCE within the stored element, so {qbit.BitWidth - 1} is the sign bit and 0 the"); - Console.WriteLine(" least significant mantissa bit. (The wire stores the planes the other way round,"); - Console.WriteLine(" most significant first; the accessor hides that.)"); - Console.WriteLine(); - Console.WriteLine($" The top {qbit.BitWidth - 24} planes of this corpus, most significant first:"); - Console.WriteLine(); - Console.WriteLine(" plane bitmaps, one byte per row"); - Console.WriteLine(" ------ -------------------------"); - - for (int bit = qbit.BitWidth - 1; bit >= 24; bit--) - { - ReadOnlySpan plane = qbit.GetPlane(bit); - Console.WriteLine($" bit {bit,2} {string.Join(" ", plane.ToArray().Select(value => value.ToString("X2", CultureInfo.InvariantCulture)))}"); - } - - // The highest plane that is not identical across every row. Found from the data rather than - // asserted, because it depends entirely on what the vectors are. - int firstDifference = qbit.BitWidth - 1; - while (firstDifference >= 0 && Uniform(qbit.GetPlane(firstDifference))) - { - firstDifference--; - } - - Console.WriteLine(); - Console.WriteLine($" The top {qbit.BitWidth - 1 - firstDifference} planes are identical in every row: bit {qbit.BitWidth - 1} is the sign and every vector"); - Console.WriteLine($" here is positive, and the exponent's high bits agree because every element is in"); - Console.WriteLine($" [0.1, 0.9]. The first plane that separates the corpus is bit {firstDifference}. That is the type's"); - Console.WriteLine(" bargain: precision costs planes, and how many you can drop depends on the data."); - - Console.WriteLine(); - Console.WriteLine($" Within a row's bitmap, element i is bit i % 8 of byte BytesPerRow - 1 - i / 8. With"); - Console.WriteLine($" BytesPerRow = {qbit.BytesPerRow} there is one byte per row, so element i is simply bit i:"); - Console.WriteLine(); - Console.WriteLine($" word bit {firstDifference} bitmap elements with bit {firstDifference} set"); - Console.WriteLine(" ------- ------------- ------------------------"); - - ReadOnlySpan interesting = qbit.GetPlane(firstDifference); - for (int row = 0; row < qbit.RowCount; row++) - { - byte bitmap = interesting[(row * qbit.BytesPerRow) + qbit.BytesPerRow - 1]; - var set = new List(); - for (int element = 0; element < qbit.Dimension; element++) - { - if ((bitmap & (1 << element)) != 0) - { - set.Add(element); - } - } - - Console.WriteLine($" {words[row],-7} {Convert.ToString(bitmap, 2).PadLeft(8, '0')} {(set.Count == 0 ? "none" : string.Join(", ", set))}"); - } - - Console.WriteLine(); - Console.WriteLine(" A bit index outside the planes, or a group outside the groups, is refused:"); - foreach (Action attempt in new Action[] { () => qbit.GetPlane(qbit.BitWidth), () => qbit.GetPlane(0, 1) }) - { - try - { - attempt(); - } - catch (ArgumentOutOfRangeException ex) - { - Console.WriteLine($" {ex.Message.Split(" (Parameter")[0]}"); - } - } - } - } - } - - private static async Task ByteOrderWithinABitmap(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. Past 8 elements the bytes run backwards\n"); - Console.WriteLine(" The bits within a byte run least significant first, but the bytes run in the reverse of"); - Console.WriteLine(" the element order — element 0 is in the LAST byte. Equivalently, a row's bitmap is the"); - Console.WriteLine(" big-endian encoding of a BytesPerRow-byte integer whose bit i is element i. That is"); - Console.WriteLine(" invisible at 5 elements and not at 12:\n"); - - await client.ExecuteAsync($"DROP TABLE IF EXISTS {WideTable}"); - await client.ExecuteAsync($@" - CREATE TABLE {WideTable} (v QBit(Float32, 12)) - ENGINE = MergeTree() ORDER BY tuple()"); - - // Elements 0 and 8 negative, the rest positive, so the sign plane says exactly where they sit. - float[] signs = Enumerable.Range(0, 12).Select(i => i is 0 or 8 ? -1.0f : 1.0f).ToArray(); - await client.InsertAsync( - $"INSERT INTO {WideTable} (v) VALUES", - new[] { ClickHouseTcpColumn.Create("v", new[] { signs }) }); - - Console.WriteLine($" One row of QBit(Float32, 12): [{string.Join(", ", signs.Select(value => value.ToString(CultureInfo.InvariantCulture)))}]"); - Console.WriteLine(" Only elements 0 and 8 are negative.\n"); - - await foreach (Block block in client.StreamAsync($"SELECT v FROM {WideTable}")) - { - if (block["v"] is IQBitColumn wide) - { - ReadOnlySpan sign = wide.GetPlane(wide.BitWidth - 1); - Console.WriteLine($" BytesPerRow {wide.BytesPerRow} (ceil(12 / 8))"); - Console.WriteLine($" sign plane {string.Join(" ", sign.ToArray().Select(value => Convert.ToString(value, 2).PadLeft(8, '0')))}"); - Console.WriteLine(" ^ byte 0 ^ byte 1"); - Console.WriteLine(" Byte 1 holds elements 0-7 and byte 0 elements 8-11, so the bit set in byte 1 is"); - Console.WriteLine(" element 0 and the bit set in byte 0 is element 8."); - Console.WriteLine(); - Console.WriteLine(" The formula covers both: element i is bit i % 8 of byte BytesPerRow - 1 - i / 8."); - Console.WriteLine($" element 0 -> bit 0 of byte {wide.BytesPerRow - 1 - (0 / 8)}"); - Console.WriteLine($" element 8 -> bit 0 of byte {wide.BytesPerRow - 1 - (8 / 8)}"); - Console.WriteLine(); - Console.WriteLine($" With Stride not a multiple of 8, the {(wide.BytesPerRow * 8) - wide.Dimension} unused bits are the high bits of byte 0."); - } - } - } - - private static async Task ReducedPrecision(ClickHouseTcpClient client) - { - Console.WriteLine("\n4. Why the planes exist: a distance at reduced precision\n"); - Console.WriteLine(" Reading only the top K planes and treating the rest of each element's bits as zero gives"); - Console.WriteLine(" a truncated float — a quarter of the bytes at K = 8. Rebuilt from its planes, the vector"); - Console.WriteLine(" of 'apple':\n"); - Console.WriteLine(" Planes read Bytes per row Reconstructed vector"); - Console.WriteLine(" ----------- ------------- ----------------------------------------------"); - - await foreach (Block block in client.StreamAsync($"SELECT word, vec FROM {TableName} ORDER BY word")) - { - if (block["vec"] is IQBitColumn qbit) - { - var words = (IColumn)block["word"]; - int apple = 0; - for (int row = 0; row < words.RowCount; row++) - { - if (words[row] == "apple") - { - apple = row; - } - } - - foreach (int keep in new[] { 32, 16, 12, 8 }) - { - float[] rebuilt = Reconstruct(qbit, apple, keep); - Console.WriteLine( - $" top {keep,2} {keep * qbit.BytesPerRow,3} [{string.Join(", ", rebuilt.Select(value => value.ToString("0.####", CultureInfo.InvariantCulture)))}]"); - } - - // The materialized view, for comparison: it reads every plane, which is what "top 32" did. - float[] materialized = ((IColumn)block["vec"])[apple]; - Console.WriteLine(); - Console.WriteLine($" The top row is exact, and equals what the IColumn view hands back:"); - Console.WriteLine($" [{string.Join(", ", materialized.Select(value => value.ToString("0.####", CultureInfo.InvariantCulture)))}]"); - } - } - - Console.WriteLine(); - Console.WriteLine(" The server does the same arithmetic in L2DistanceTransposed's third argument, which is a"); - Console.WriteLine(" count of planes. Ranking 'apple' against the corpus at three precisions:\n"); - Console.WriteLine(" word precision 32 precision 12 precision 8"); - Console.WriteLine(" ------- ------------ ------------ -----------"); - - const string query = "[0.9, 0.1, 0.8, 0.2, 0.7]"; - await foreach (object[] row in client.QueryAsync($@" - SELECT word, - L2DistanceTransposed(vec, {query}, 32) AS d32, - L2DistanceTransposed(vec, {query}, 12) AS d12, - L2DistanceTransposed(vec, {query}, 8) AS d8 - FROM {TableName} - ORDER BY d32")) - { - Console.WriteLine($" {row[0],-7} {Number(row[1]),-12} {Number(row[2]),-12} {Number(row[3])}"); - } - - Console.WriteLine(); - Console.WriteLine(" Read the columns, not just the numbers. At precision 12 the ranking is unchanged and the"); - Console.WriteLine(" distances are already wrong in the second digit. At precision 8 apple and orange tie and"); - Console.WriteLine(" banana comes out ahead of both — the ranking has broken, while the two clusters are still"); - Console.WriteLine(" cleanly separated. That is the trade the type is for: shortlist cheaply at low precision,"); - Console.WriteLine(" then re-rank the shortlist exactly. How low you can go is a property of your vectors, so"); - Console.WriteLine(" measure it rather than picking a number."); - Console.WriteLine(); - Console.WriteLine(" Where the client's plane access earns its keep is the work the server has no function"); - Console.WriteLine(" for: a custom metric, a quantizer, or an index built over the top planes only. Reading"); - Console.WriteLine(" GetPlane(bit) for the few bits you want touches only those bytes, whereas the"); - Console.WriteLine(" IColumn view materializes every element of every row."); - } - - private static async Task ElementTypes(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) - { - Console.WriteLine("\n5. The element types, and the CLR type each reads as\n"); - Console.WriteLine(" Type BitWidth Reads as Note"); - Console.WriteLine(" ------------------ -------- --------- --------------------------------------"); - - foreach (string element in new[] { "BFloat16", "Float32", "Float64", "Int8" }) - { - if (element == "Int8" && server.Version < StridedAndInt8From) - { - Console.WriteLine($" QBit(Int8, 5) - - skipped: needs ClickHouse {StridedAndInt8From} or newer,"); - Console.WriteLine($" this server is {server.Version}"); - continue; - } - - string table = $"{TableName}_{element}"; - try - { - await client.ExecuteAsync($"CREATE TABLE {table} (v QBit({element}, 5)) ENGINE = MergeTree() ORDER BY tuple()"); - await client.ExecuteAsync($"INSERT INTO {table} VALUES ([1.0, 2.0, 0.5, -1.0, 0.25])"); - - await foreach (Block block in client.StreamAsync($"SELECT v FROM {table}")) - { - var qbit = (IQBitColumn)block["v"]; - string note = element switch - { - "BFloat16" => "16 planes, widened to float on the way out", - "Float64" => "the only one that reads as double[]", - "Int8" => "since ClickHouse 26.7", - _ => "the common case", - }; - Console.WriteLine($" {block["v"].TypeName,-18} {qbit.BitWidth,-8} {Describe(block["v"].ElementType),-9} {note}"); - Console.WriteLine($" row 0 = [{string.Join(", ", ((System.Collections.IEnumerable)block["v"].GetValue(0)!).Cast().Select(Number))}]"); - } - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" QBit({element}, 5) refused by this server:"); - Console.WriteLine($" {FirstLine(ex.Message)}"); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); - } - } - - Console.WriteLine(); - Console.WriteLine(" A BFloat16 element's plane positions are those of the 16-bit brain-float, not of the"); - Console.WriteLine(" float you read: bit 15 is its sign, and there are only 16 planes to ask for."); - } - - private static async Task Strided(ClickHouseTcpClient client, ClickHouseTcpServerInfo server) - { - Console.WriteLine("\n6. The strided form, and why Stride and GroupCount exist\n"); - Console.WriteLine(" ClickHouse 26.7 added an optional third argument, QBit(T, N, stride), which splits a row"); - Console.WriteLine(" into N / stride independent groups, each carrying its own full set of planes. A plane is"); - Console.WriteLine(" then GroupCount disjoint runs, so GetPlane(bit) cannot name it and GetPlane(bit, group)"); - Console.WriteLine(" is the accessor.\n"); - - if (server.Version < StridedAndInt8From) - { - Console.WriteLine($" Skipped: needs ClickHouse {StridedAndInt8From} or newer, this server is {server.Version}."); - Console.WriteLine(" Confirming the server's own answer rather than assuming it:"); - - try - { - await client.ExecuteAsync($"CREATE TABLE {WideTable}_strided (v QBit(Float32, 8, 4)) ENGINE = MergeTree() ORDER BY tuple()"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {WideTable}_strided"); - Console.WriteLine(" accepted, which this example did not expect"); - } - catch (ClickHouseTcpServerException ex) - { - Console.WriteLine($" {FirstLine(ex.Message)}"); - } - } - else - { - Console.WriteLine(" This server is new enough to declare one. Note that this client does not decode the"); - Console.WriteLine(" strided body yet, so reading such a column reports a NotSupportedException:"); - - try - { - await client.ExecuteAsync($"CREATE TABLE {WideTable}_strided (v QBit(Float32, 8, 4)) ENGINE = MergeTree() ORDER BY tuple()"); - await client.ExecuteAsync($"INSERT INTO {WideTable}_strided VALUES ([1, 2, 3, 4, 5, 6, 7, 8])"); - - // Drained: the read is expected to fail, and the failure is the point. - await foreach (Block _ in client.StreamAsync($"SELECT v FROM {WideTable}_strided")) - { - } - - Console.WriteLine(" read, which this example did not expect"); - } - catch (Exception ex) when (ex is NotSupportedException or ClickHouseTcpServerException) - { - Console.WriteLine($" {ex.GetType().Name}: {FirstLine(ex.Message)}"); - } - finally - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {WideTable}_strided"); - } - } - - Console.WriteLine(); - Console.WriteLine(" So on any server this client reads, GroupCount is 1 and Stride equals Dimension. Both"); - Console.WriteLine(" properties are there so plane-reading code is written against the general layout —"); - Console.WriteLine(" GetPlane(bit, group), BytesPerRow from Stride — and needs no change when it is not."); - } - - // True when every byte of a plane is the same, so the plane separates no row from any other. - private static bool Uniform(ReadOnlySpan plane) - { - for (int i = 1; i < plane.Length; i++) - { - if (plane[i] != plane[0]) - { - return false; - } - } - - return true; - } - - // Rebuilds one row's vector from its top `keep` planes, leaving the rest of each element's bits zero. This is - // the client-side equivalent of L2DistanceTransposed's precision argument. - private static float[] Reconstruct(IQBitColumn column, int row, int keep) - { - var rebuilt = new float[column.Dimension]; - for (int element = 0; element < column.Dimension; element++) - { - uint bits = 0; - for (int bit = column.BitWidth - 1; bit >= column.BitWidth - keep; bit--) - { - ReadOnlySpan plane = column.GetPlane(bit); - byte bitmap = plane[(row * column.BytesPerRow) + column.BytesPerRow - 1 - (element / 8)]; - if ((bitmap & (1 << (element % 8))) != 0) - { - bits |= 1u << bit; - } - } - - rebuilt[element] = BitConverter.UInt32BitsToSingle(bits); - } - - return rebuilt; - } - - private static string Describe(Type type) => type switch - { - _ when type == typeof(float[]) => "float[]", - _ when type == typeof(double[]) => "double[]", - _ when type == typeof(sbyte[]) => "sbyte[]", - _ => type.Name, - }; - - private static string Number(object? value) - => value is IFormattable formattable ? formattable.ToString("0.######", CultureInfo.InvariantCulture) : "-"; - - private static string FirstLine(string message) - { - int newline = message.IndexOf('\n'); - string line = newline < 0 ? message : message[..newline]; - if (line.StartsWith("DB::Exception: ", StringComparison.Ordinal)) - { - line = line["DB::Exception: ".Length..]; - } - - int scope = line.IndexOf(": In scope", StringComparison.Ordinal); - return scope < 0 ? line : line[..scope]; - } -} diff --git a/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs b/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs new file mode 100644 index 000000000..82835407b --- /dev/null +++ b/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs @@ -0,0 +1,66 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Inserts data that is already organized into typed columns. +public static class TcpColumnarInsert +{ + private const string TableName = "example_tcp_columnar_insert"; + + public static async Task Run() + { + await using var client = ExampleConfig.CreateTcpClient(); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + + try + { + await client.ExecuteAsync($""" + CREATE TABLE {TableName} + ( + id UInt64, + name String, + score Float64, + region String DEFAULT 'unknown' + ) + ENGINE = MergeTree + ORDER BY id + """); + + var ids = new ulong[] { 1, 2, 3 }; + var names = new[] { "Ada", "Grace", "Alan" }; + var scores = new[] { 99.5, 97.25, 91.0 }; + + // Column names, not argument order, determine the target column. + var columns = new IColumn[] + { + ClickHouseTcpColumn.Create("score", scores), + ClickHouseTcpColumn.Create("id", ids), + ClickHouseTcpColumn.Create("name", names), + }; + + await client.InsertAsync( + $"INSERT INTO {TableName} (id, name, score) VALUES", + columns, + new ClickHouseTcpInsertOptions { MaxRowsPerBlock = 2 }); + + // region is absent from the statement, so ClickHouse applies its default. + await client.InsertAsync( + $"INSERT INTO {TableName} (id, name) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("name", new[] { "Edsger" }), + ClickHouseTcpColumn.Create("id", new ulong[] { 4 }), + }); + + await foreach (object[] row in client.QueryAsync( + $"SELECT id, name, score, region FROM {TableName} ORDER BY id")) + { + Console.WriteLine($"{row[0]}: {row[1]}, score {row[2]}, region {row[3]}"); + } + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } + } +} diff --git a/examples/Tcp/Write/Tcp_002_CompositeWrites.cs b/examples/Tcp/Write/Tcp_002_CompositeWrites.cs new file mode 100644 index 000000000..3b62aee49 --- /dev/null +++ b/examples/Tcp/Write/Tcp_002_CompositeWrites.cs @@ -0,0 +1,108 @@ +using ClickHouse.Driver.Tcp; + +namespace ClickHouse.Driver.Examples; + +/// Writes arrays, maps, tuples, nullable values, and low-cardinality strings. +public static class TcpCompositeWrites +{ + private const string SourceTable = "example_tcp_composite_writes"; + private const string CopyTable = "example_tcp_composite_writes_copy"; + private const string Columns = "id, readings, attributes, point, score, city"; + + public static async Task Run() + { + ClickHouseTcpClientOptions options = ExampleConfig.TcpBuilder().ToOptions(); + + // Streaming holds one connection while InsertAsync uses another, so this needs two pool slots. + options = options with { MaxPoolSize = Math.Max(2, options.MaxPoolSize) }; + + await using var client = new ClickHouseTcpClient(options); + + foreach (string table in new[] { SourceTable, CopyTable }) + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + + try + { + await client.ExecuteAsync(CreateTable(SourceTable)); + await client.ExecuteAsync(CreateTable(CopyTable)); + + var readings = new[] + { + new[] { 0.5, 0.75, 1.0 }, + Array.Empty(), + }; + var attributes = new[] + { + new[] + { + new KeyValuePair("floor", 3), + new KeyValuePair("room", 12), + }, + Array.Empty>(), + }; + + // Use one array or map per row, ValueTuple for Tuple, and nullable CLR values for Nullable. + await client.InsertAsync( + $"INSERT INTO {SourceTable} ({Columns}) VALUES", + new IColumn[] + { + ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2 }), + ClickHouseTcpColumn.Create("readings", readings), + ClickHouseTcpColumn.Create("attributes", attributes), + ClickHouseTcpColumn.Create("point", new[] { (1, "one"), (2, "two") }), + ClickHouseTcpColumn.Create("score", new double?[] { 1.25, null }), + ClickHouseTcpColumn.Create("city", new[] { "Amsterdam", "Amsterdam" }), + }); + + await PrintRows(client, SourceTable); + + // Read columns already use the native layout. Reinsert them before the borrowed block expires. + await foreach (Block block in client.StreamAsync( + $"SELECT {Columns} FROM {SourceTable} ORDER BY id")) + { + await client.InsertAsync( + $"INSERT INTO {CopyTable} ({Columns}) VALUES", + block.Columns.ToArray()); + } + + Console.WriteLine("Copied directly from result blocks:"); + await PrintRows(client, CopyTable); + } + finally + { + foreach (string table in new[] { SourceTable, CopyTable }) + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); + } + } + } + + private static string CreateTable(string table) => $""" + CREATE TABLE {table} + ( + id UInt64, + readings Array(Float64), + attributes Map(String, Int64), + point Tuple(x Int32, y String), + score Nullable(Float64), + city LowCardinality(String) + ) + ENGINE = MergeTree + ORDER BY id + """; + + private static async Task PrintRows(ClickHouseTcpClient client, string table) + { + await foreach (object[] row in client.QueryAsync($""" + SELECT id, toString(readings), toString(attributes), toString(point), + toString(score), city + FROM {table} + ORDER BY id + """)) + { + Console.WriteLine(string.Join(" | ", row.Select(value => value?.ToString() ?? "NULL"))); + } + } +} diff --git a/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs b/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs deleted file mode 100644 index 4f6e38572..000000000 --- a/examples/Tcp/Write/Tcp_009_ColumnarInsert.cs +++ /dev/null @@ -1,389 +0,0 @@ -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// The columnar insert tier: one .Create call per target column, then -/// InsertAsync(sql, columns). Columns are matched to the target by name, so their order is free and a -/// named subset is allowed; the ClickHouse type is never stated, because the server sends the target's schema -/// before any row data. -/// -/// -/// Tcp_006_BlocksAndColumns is the read side of this tier. This is the write side, and the two meet: -/// a column read out of a is a valid insert column. Tcp_010_CompositeWrites covers -/// the composite types and that round trip. -/// -/// -public static class TcpColumnarInsert -{ - // These examples are not the test suite, so fixed names are fine. All four are dropped even if a step throws. - private const string TableName = "example_tcp_columnar_insert"; - private const string DefaultsTable = "example_tcp_columnar_insert_defaults"; - private const string InstantsTable = "example_tcp_columnar_insert_instants"; - private const string BulkTable = "example_tcp_columnar_insert_bulk"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await OneColumnPerTargetColumn(client); - await MatchedByName(client); - await ANamedSubset(client); - await TheServerStatesTheType(client); - await BlockGeometry(client); - await TheRowTierForComparison(client); - RulesWorthKnowing(); - } - finally - { - foreach (string table in new[] { TableName, DefaultsTable, InstantsTable, BulkTable }) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); - } - - Console.WriteLine("\nDropped every table this example created."); - } - } - - private static async Task OneColumnPerTargetColumn(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); - await client.ExecuteAsync($@" - CREATE TABLE {TableName} - ( - id UInt64, - name String, - score Float64 - ) - ENGINE = MergeTree() - ORDER BY id"); - - Console.WriteLine($"1. One column per target column\n"); - Console.WriteLine($" Created '{TableName}' (id UInt64, name String, score Float64)\n"); - - // The data is already grouped by column, which is how the wire wants it. Nothing is transposed and no - // value is boxed, so this is the shape to reach for when the data is columnar to begin with: a parsed - // file, a computed series, an ETL stage. - // - // Create takes the array over rather than copying it, so treat it as handed away: do not write to ids, - // names or scores until the insert has completed. - var ids = new ulong[] { 1, 2, 3, 4 }; - var names = new[] { "Ada", "Grace", "Alan", "Edsger" }; - var scores = new[] { 99.5, 97.25, 91.0, 94.75 }; - - await client.InsertAsync( - $"INSERT INTO {TableName} (id, name, score) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", ids), - ClickHouseTcpColumn.Create("name", names), - ClickHouseTcpColumn.Create("score", scores), - }); - - Console.WriteLine(" InsertAsync(\"INSERT INTO ... (id, name, score) VALUES\", [three columns])"); - Console.WriteLine(" The statement ends at VALUES. The rows travel after it as native blocks, never as SQL text.\n"); - await Show(client, $"SELECT id, name, score FROM {TableName} ORDER BY id", "id", "name", "score"); - - // The generic argument is the CLR type of one row's value, and the factory reports it back as ElementType. - // TypeName is null: an inserted column has no header of its own, so there is no ClickHouse type to report. - IColumn column = ClickHouseTcpColumn.Create("id", ids); - Console.WriteLine($"\n A built column reports: RowCount {column.RowCount}, ElementType {column.ElementType.Name}, TypeName {column.TypeName ?? "null"}"); - Console.WriteLine(" TypeName is null because the ClickHouse type is the server's to state, which is section 4."); - } - - private static async Task MatchedByName(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. Matched by name, not by position\n"); - - // Both orders differ from the table's and from each other, and the insert still lands correctly: the - // server's schema block names its columns, and each supplied column is looked up by its own name. - await client.InsertAsync( - $"INSERT INTO {TableName} (score, id, name) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("name", new[] { "Barbara", "Frances" }), - ClickHouseTcpColumn.Create("score", new[] { 96.5, 98.0 }), - ClickHouseTcpColumn.Create("id", new ulong[] { 5, 6 }), - }); - - Console.WriteLine(" The statement lists (score, id, name); the columns are supplied as name, score, id."); - Console.WriteLine(" Neither order is the table's, and both rows are still correct:\n"); - await Show(client, $"SELECT id, name, score FROM {TableName} WHERE id > 4 ORDER BY id", "id", "name", "score"); - - Console.WriteLine("\n Every column the statement lists must be supplied, and nothing else. Both mistakes are"); - Console.WriteLine(" caught before a single row is written, and both messages name the columns involved:\n"); - - await ShowRejection( - client, - "score not supplied", - $"INSERT INTO {TableName} (id, name, score) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 7 }), - ClickHouseTcpColumn.Create("name", new[] { "Katherine" }), - }); - - // The lookup is ordinal, so 'ID' is not 'id': the target reports id as missing and ID as unexpected. - await ShowRejection( - client, - "'ID' for 'id'", - $"INSERT INTO {TableName} (id) VALUES", - new IColumn[] { ClickHouseTcpColumn.Create("ID", new ulong[] { 7 }) }); - - Console.WriteLine("\n The second is why names are worth getting exactly right: the comparison is ordinal, as"); - Console.WriteLine(" ClickHouse's own is, so a case difference is a different column and not a near miss."); - } - - private static async Task ANamedSubset(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {DefaultsTable}"); - await client.ExecuteAsync($@" - CREATE TABLE {DefaultsTable} - ( - id UInt64, - name String, - region String DEFAULT 'unknown', - attempts UInt8 DEFAULT 1 - ) - ENGINE = MergeTree() - ORDER BY id"); - - Console.WriteLine("\n3. A named subset, and the server fills the rest\n"); - Console.WriteLine($" '{DefaultsTable}' has four columns, two of them with a DEFAULT."); - Console.WriteLine(" The statement lists two, so the schema block describes two, so two columns are enough:\n"); - - await client.InsertAsync( - $"INSERT INTO {DefaultsTable} (id, name) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2 }), - ClickHouseTcpColumn.Create("name", new[] { "north", "south" }), - }); - - await Show(client, $"SELECT id, name, region, attempts FROM {DefaultsTable} ORDER BY id", "id", "name", "region", "attempts"); - - Console.WriteLine("\n region and attempts were never sent, and hold the DEFAULT the table declares."); - Console.WriteLine(" It is the statement's column list that decides the subset, not the columns you pass:"); - Console.WriteLine(" omit the list and the server describes every column, so every column must be supplied."); - } - - private static async Task TheServerStatesTheType(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {InstantsTable}"); - await client.ExecuteAsync($@" - CREATE TABLE {InstantsTable} - ( - seconds DateTime('UTC'), - millis DateTime64(3, 'UTC'), - micros DateTime64(6, 'UTC') - ) - ENGINE = MergeTree() - ORDER BY seconds"); - - Console.WriteLine("\n4. You never state the ClickHouse type\n"); - Console.WriteLine(" An INSERT over this protocol has two phases. The client sends the statement, the server"); - Console.WriteLine(" answers with a schema block naming and typing the target columns, and only then does the"); - Console.WriteLine(" client serialize. So the target type is known before a byte of data is encoded, and the"); - Console.WriteLine(" caller supplies CLR values only.\n"); - Console.WriteLine(" One DateTime[] into three columns of different precision, with no type stated anywhere:\n"); - - var instants = new[] - { - new DateTime(2026, 6, 1, 10, 0, 0, 125, DateTimeKind.Utc), - new DateTime(2026, 6, 1, 10, 0, 0, 875, DateTimeKind.Utc), - }; - - await client.InsertAsync( - $"INSERT INTO {InstantsTable} (seconds, millis, micros) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("seconds", instants), - ClickHouseTcpColumn.Create("millis", instants), - ClickHouseTcpColumn.Create("micros", instants), - }); - - await Show( - client, - $"SELECT toString(seconds), toString(millis), toString(micros) FROM {InstantsTable} ORDER BY millis", - 26, - "seconds", "millis", "micros"); - - Console.WriteLine("\n Same values, three encodings: whole seconds, milliseconds, microseconds."); - Console.WriteLine(" The 125 and 875 milliseconds are gone from the DateTime column because DateTime holds"); - Console.WriteLine(" seconds, which is the target's decision and not the client's.\n"); - - Console.WriteLine(" What the CLR type must satisfy is the target codec, and a mismatch is rejected before"); - Console.WriteLine(" any row is written:"); - - await ShowRejection( - client, - "long into a DateTime column", - $"INSERT INTO {InstantsTable} (seconds) VALUES", - new IColumn[] { ClickHouseTcpColumn.Create("seconds", new[] { 1780308000L }) }); - - Console.WriteLine("\n And note what is not here: no DESCRIBE, no probe query. The HTTP client's"); - Console.WriteLine(" InsertBinaryAsync has to learn the schema itself, with a SELECT ... WHERE 1=0 per call"); - Console.WriteLine(" unless you pass InsertOptions.ColumnTypes or turn on InsertOptions.UseSchemaCache."); - Console.WriteLine(" Here the schema arrives inside the insert, so there is nothing to cache or skip."); - } - - private static async Task BlockGeometry(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {BulkTable}"); - await client.ExecuteAsync($@" - CREATE TABLE {BulkTable} - ( - id UInt64, - name String, - score Float64 - ) - ENGINE = MergeTree() - ORDER BY id"); - - Console.WriteLine("\n5. ClickHouseTcpInsertOptions.MaxRowsPerBlock\n"); - Console.WriteLine(" One InsertAsync call is one statement, but not necessarily one wire block. The cap"); - Console.WriteLine(" splits the rows into blocks of at most that many. It is block geometry, not a memory"); - Console.WriteLine(" bound: it defaults to 1,000,000 rows, and null writes one block of any height.\n"); - - Console.WriteLine(" The same six rows, once split into three blocks and once written as one:\n"); - Console.WriteLine(" MaxRowsPerBlock Rows stored Active parts"); - Console.WriteLine(" --------------- ----------- ------------"); - - await SixRowsAndCountParts(client, maxRowsPerBlock: 2); - await SixRowsAndCountParts(client, maxRowsPerBlock: null); - - Console.WriteLine(); - Console.WriteLine(" The cap is a client-side concern only. This server recombines the blocks of one insert"); - Console.WriteLine(" before it writes, so the six rows land as one part either way: lowering the cap does not"); - Console.WriteLine(" create parts and raising it does not remove them."); - Console.WriteLine(" Leave it alone unless you want a particular block height. To bound the memory a large"); - Console.WriteLine(" insert costs, set ClickHouseTcpClientOptions.MaxSendBufferBytes: it flushes the buffered"); - Console.WriteLine(" bytes to the socket whenever they pass the cap, independent of block height. One column"); - Console.WriteLine(" larger than the cap still buffers in full."); - } - - private static async Task SixRowsAndCountParts(ClickHouseTcpClient client, int? maxRowsPerBlock) - { - await client.ExecuteAsync($"TRUNCATE TABLE {BulkTable}"); - - await client.InsertAsync( - $"INSERT INTO {BulkTable} (id, name, score) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3, 4, 5, 6 }), - ClickHouseTcpColumn.Create("name", new[] { "a", "b", "c", "d", "e", "f" }), - ClickHouseTcpColumn.Create("score", new[] { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 }), - }, - new ClickHouseTcpInsertOptions { MaxRowsPerBlock = maxRowsPerBlock }); - - object rows = await client.ExecuteScalarAsync($"SELECT count() FROM {BulkTable}"); - object parts = await client.ExecuteScalarAsync( - $"SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = '{BulkTable}' AND active"); - - Console.WriteLine($" {maxRowsPerBlock?.ToString() ?? "null",-15} {rows,11} {parts,12}"); - } - - private static async Task TheRowTierForComparison(ClickHouseTcpClient client) - { - Console.WriteLine("\n6. The row tier, for comparison\n"); - Console.WriteLine(" InsertRowsAsync takes one object[] per row and the same statement. It is the right"); - Console.WriteLine(" call when the data really is row-shaped, and it differs in three ways:\n"); - Console.WriteLine(" values are matched to the target columns by POSITION, not by name;"); - Console.WriteLine(" each row is an object[], so every value-type value is boxed as the caller builds it;"); - Console.WriteLine(" the client then transposes those rows into one typed column per target.\n"); - - await client.ExecuteAsync($"TRUNCATE TABLE {BulkTable}"); - - var rows = new List - { - new object[] { 1UL, "Ada", 99.5 }, - new object[] { 2UL, "Grace", 97.25 }, - }; - - await client.InsertRowsAsync($"INSERT INTO {BulkTable} (id, name, score) VALUES", rows); - await Show(client, $"SELECT id, name, score FROM {BulkTable} ORDER BY id", "id", "name", "score"); - - // Positional matching is the trap: the values are the right types for the row, just not for the columns - // in the order the statement names them. The message names the position, the column and both CLR types. - try - { - await client.InsertRowsAsync( - $"INSERT INTO {BulkTable} (id, name, score) VALUES", - new List { new object[] { "Alan", 3UL, 91.0 } }); - } - catch (InvalidOperationException ex) - { - Console.WriteLine($"\n Values in the wrong order: {ex.Message}"); - } - - // The shaping cost is measurable and synchronous, so this number is exact rather than a benchmark: it is - // what the two shapes of the same 50,000 rows allocate before either call is made. - const int Rows = 50_000; - - long before = GC.GetAllocatedBytesForCurrentThread(); - var ids = new ulong[Rows]; - var names = new string[Rows]; - var scores = new double[Rows]; - long columnar = GC.GetAllocatedBytesForCurrentThread() - before; - - before = GC.GetAllocatedBytesForCurrentThread(); - var boxed = new object[Rows][]; - for (int i = 0; i < Rows; i++) - { - boxed[i] = new object[] { ids[i], names[i], scores[i] }; - } - - long rowwise = GC.GetAllocatedBytesForCurrentThread() - before; - - Console.WriteLine($"\n Holding the same {Rows:N0} rows of (UInt64, String, Float64):"); - Console.WriteLine($" three typed arrays {columnar,10:N0} bytes"); - Console.WriteLine($" one object[] per row {rowwise,10:N0} bytes ({boxed.Length:N0} arrays, plus a box per number)"); - Console.WriteLine(" The columnar tier's saving is mostly this: the arrays are usually the shape the data is"); - Console.WriteLine(" already in, so neither the boxes nor the per-row arrays are ever created."); - } - - private static void RulesWorthKnowing() - { - Console.WriteLine("\n7. Rules worth knowing\n"); - Console.WriteLine(" Create takes your array over, it does not copy it. Do not write to an array after"); - Console.WriteLine(" handing it to Create, until the insert has completed. The IEnumerable overload"); - Console.WriteLine(" enumerates once into an array, and takes over a T[] passed to it as is."); - Console.WriteLine(" Every column must hold the same number of rows, and each name must be unique."); - Console.WriteLine(" Zero rows is a no-op that still validates: the statement is sent, the schema is"); - Console.WriteLine(" matched, and no data block follows. An empty column list is a no-op too."); - Console.WriteLine(" The columns you build are yours. The insert does not dispose them, and disposing one"); - Console.WriteLine(" before the insert empties it, so keep them alive until the call returns."); - Console.WriteLine(" InsertAsync is a ValueTask: await it once, and do not await it twice."); - } - - // Prints a small result set with a header, so each section can show what the server actually stored. - private static Task Show(ClickHouseTcpClient client, string sql, params string[] headers) - => Show(client, sql, 12, headers); - - private static async Task Show(ClickHouseTcpClient client, string sql, int width, params string[] headers) - { - Console.WriteLine(" " + string.Join(" ", headers.Select(h => h.PadRight(width)))); - Console.WriteLine(" " + string.Join(" ", headers.Select(_ => new string('-', width)))); - await foreach (object[] row in client.QueryAsync(sql)) - { - Console.WriteLine(" " + string.Join(" ", row.Select(v => (v?.ToString() ?? "NULL").PadRight(width)))); - } - } - - // Runs an insert that is expected to be rejected client-side and prints the reason. The client closes the row - // stream cleanly before throwing, so the connection goes back to the pool usable. - private static async Task ShowRejection(ClickHouseTcpClient client, string what, string sql, IReadOnlyList columns) - { - try - { - await client.InsertAsync(sql, columns); - Console.WriteLine($" {what}: accepted, which this example did not expect"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" {what}: {ex.Message.Split(" (Parameter")[0]}"); - } - } -} diff --git a/examples/Tcp/Write/Tcp_010_CompositeWrites.cs b/examples/Tcp/Write/Tcp_010_CompositeWrites.cs deleted file mode 100644 index 2190060d9..000000000 --- a/examples/Tcp/Write/Tcp_010_CompositeWrites.cs +++ /dev/null @@ -1,354 +0,0 @@ -using ClickHouse.Driver.Tcp; - -namespace ClickHouse.Driver.Examples; - -/// -/// Writing composite columns on the columnar tier: the two array shapes, and then Map, Tuple, -/// Nullable and LowCardinality. -/// -/// -/// An Array(T) column is accepted in two shapes. Jagged is one T[] per row, which is what -/// .Create builds. Dense is a flat inner column -/// plus per-row offsets, which is the wire's own layout and what a read produces, so a column read out of a -/// re-inserts with nothing rebuilt. Section 3 is that round trip, and it is the reason the -/// tier exists. -/// -/// -/// -/// Tcp_009_ColumnarInsert covers the tier itself: matching by name, the subset rule, and why no ClickHouse -/// type is ever stated. Tcp_006_BlocksAndColumns covers reading the same shapes. -/// -/// -public static class TcpCompositeWrites -{ - // These examples are not the test suite, so fixed names are fine. All three are dropped even if a step throws. - private const string ArraysTable = "example_tcp_composite_writes_arrays"; - private const string DenseTable = "example_tcp_composite_writes_dense"; - private const string OthersTable = "example_tcp_composite_writes_others"; - private const string OthersDenseTable = "example_tcp_composite_writes_others_dense"; - - private const string ArrayColumns = "id, readings, tags, maybe"; - private const string OtherColumns = "id, attrs, point, score, city, nick"; - - public static async Task Run() - { - await using var client = ExampleConfig.CreateTcpClient(); - - try - { - await JaggedArrays(client); - await TheNonNullableRowRule(client); - await DenseArraysAndTheRoundTrip(client); - await TheOtherComposites(client); - await WhatEachTargetAccepts(client); - WhatToRemember(); - } - finally - { - foreach (string table in new[] { ArraysTable, DenseTable, OthersTable, OthersDenseTable }) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {table}"); - } - - Console.WriteLine("\nDropped every table this example created."); - } - } - - private static async Task JaggedArrays(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {ArraysTable}"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {DenseTable}"); - await client.ExecuteAsync(ArrayDdl(ArraysTable)); - await client.ExecuteAsync(ArrayDdl(DenseTable)); - - Console.WriteLine("1. The jagged shape: one array per row\n"); - Console.WriteLine($" '{ArraysTable}' (id UInt64, readings Array(Float64), tags Array(String),"); - Console.WriteLine(" maybe Array(Nullable(Int32)))\n"); - Console.WriteLine(" Create builds an IColumn, so the CLR type of one row is the array type the"); - Console.WriteLine(" target's element type maps to: double[] for Array(Float64), string[] for Array(String),"); - Console.WriteLine(" int?[] for Array(Nullable(Int32)).\n"); - - // Array.Empty is an empty row, which is a value. It is not a null row: see section 2. - var readings = new[] { new[] { 0.5, 0.75, 1.0 }, new[] { 1.25, 1.5 }, Array.Empty() }; - var tags = new[] { new[] { "north", "roof" }, Array.Empty(), new[] { "south" } }; - var maybe = new[] { new int?[] { 7, null, 9 }, new int?[] { null }, Array.Empty() }; - - await client.InsertAsync( - $"INSERT INTO {ArraysTable} ({ArrayColumns}) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2, 3 }), - ClickHouseTcpColumn.Create("readings", readings), - ClickHouseTcpColumn.Create("tags", tags), - ClickHouseTcpColumn.Create("maybe", maybe), - }); - - await ShowArrays(client, ArraysTable); - - Console.WriteLine("\n Nothing was flattened up front. The codec walks the rows once to build the offsets"); - Console.WriteLine(" the wire needs, then writes each row's elements straight from its own array, so the"); - Console.WriteLine(" only extra buffer is the offsets. Where the element type is itself composite the"); - Console.WriteLine(" elements go through a lazy concatenated view rather than a copy."); - } - - private static async Task TheNonNullableRowRule(ClickHouseTcpClient client) - { - Console.WriteLine("\n2. A row of Array(T) may not be null\n"); - Console.WriteLine(" ClickHouse has no such value: an Array(T) row is a run of elements, possibly of length"); - Console.WriteLine(" zero, and there is no bit on the wire that could say 'absent' instead. So a null row is"); - Console.WriteLine(" refused rather than quietly turned into an empty one:\n"); - - // The offsets pass reaches this row and refuses it. That happens while the block is being encoded, after - // the statement has gone out, so this failure costs the connection: the client cannot leave a half-written - // block on the wire, and drops it instead. Validate your rows before you hand them over. - try - { - await client.InsertAsync( - $"INSERT INTO {ArraysTable} (id, readings) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 4 }), - ClickHouseTcpColumn.Create("readings", new double[]?[] { null }), - }); - } - catch (ArgumentException ex) - { - Console.WriteLine($" {ex.Message.Split(" (Parameter")[0]}"); - } - - Console.WriteLine("\n The two ways out are in the message, and they mean different things:"); - Console.WriteLine(" Array.Empty() is a row that exists and holds nothing."); - Console.WriteLine(" Array(Nullable(T)) is a row whose ELEMENTS may be null, which is the 'maybe' column"); - Console.WriteLine(" above: row 2 is [NULL], one element long, and row 3 is [], zero elements long."); - Console.WriteLine(); - Console.WriteLine(" Unlike the name and type checks in Tcp_009, this one fires while the block is being"); - Console.WriteLine(" encoded, so it is worth checking your rows before the call rather than after it."); - } - - private static async Task DenseArraysAndTheRoundTrip(ClickHouseTcpClient client) - { - Console.WriteLine("\n3. The dense shape, and the round trip it makes free\n"); - Console.WriteLine(" The wire does not carry one array per row. It carries every row's elements end to end"); - Console.WriteLine(" plus one cumulative offset per row, and that is exactly what a read hands back: an"); - Console.WriteLine(" IArrayColumn over the server's own layout. Handed back to an insert, it is written"); - Console.WriteLine(" from that layout with no arrays rebuilt.\n"); - - int blocks = 0; - await foreach (Block block in client.StreamAsync($"SELECT {ArrayColumns} FROM {ArraysTable} ORDER BY id")) - { - blocks++; - - if (block["readings"] is IArrayColumn dense) - { - Console.WriteLine($" readings, as read: {block["readings"].TypeName}, {dense.RowCount} rows"); - Console.WriteLine($" InnerValues = [{string.Join(", ", dense.InnerValues.ToArray())}] (every row's elements, flat)"); - Console.WriteLine($" Offsets = [{string.Join(", ", dense.Offsets.ToArray())}] (cumulative ends, one more entry than rows)"); - Console.WriteLine(" Row i is InnerValues.Slice(Offsets[i], Offsets[i + 1] - Offsets[i]), so row 2's"); - Console.WriteLine($" slice is [{string.Join(", ", dense.InnerValues[dense.Offsets[2]..dense.Offsets[3]].ToArray())}] and it is empty because both offsets are {dense.Offsets[2]}."); - } - - // The block is borrowed, so the re-insert happens inside this iteration. It runs on a second pooled - // connection, because the first is busy streaming this result. - await client.InsertAsync($"INSERT INTO {DenseTable} ({ArrayColumns}) VALUES", block.Columns.ToArray()); - } - - Console.WriteLine($"\n Re-inserted {blocks} block into '{DenseTable}' with no column rebuilt at all:\n"); - await ShowArrays(client, DenseTable); - - Console.WriteLine("\n Every value survived, including the empty rows and the null elements. That is the"); - Console.WriteLine(" whole point of the tier: a copy, a filter, or a backfill can read a block and write it"); - Console.WriteLine(" again without ever materializing a row.\n"); - Console.WriteLine(" Two things to know about it:"); - Console.WriteLine(" A read column carries the name the SELECT gave it, and an insert matches by name, so"); - Console.WriteLine(" rename in the query when the target column is named differently: SELECT readings AS"); - Console.WriteLine(" other_name. There is no way to rename a column object."); - Console.WriteLine(" The dense shape is what you receive, not something you can build. Create only makes"); - Console.WriteLine(" the jagged shape, so a caller that already holds flat values and offsets has to"); - Console.WriteLine(" slice them into per-row arrays first."); - } - - private static async Task TheOtherComposites(ClickHouseTcpClient client) - { - await client.ExecuteAsync($"DROP TABLE IF EXISTS {OthersTable}"); - await client.ExecuteAsync($"DROP TABLE IF EXISTS {OthersDenseTable}"); - await client.ExecuteAsync(OthersDdl(OthersTable)); - await client.ExecuteAsync(OthersDdl(OthersDenseTable)); - - Console.WriteLine("\n4. Map, Tuple, Nullable and LowCardinality\n"); - - // A Map row is a pair array rather than a dictionary: the wire carries keys and values in order, so a - // pair array can express what a Dictionary cannot. - var attrs = new[] - { - new[] { new KeyValuePair("floor", 3), new KeyValuePair("room", 12) }, - Array.Empty>(), - }; - - await client.InsertAsync( - $"INSERT INTO {OthersTable} ({OtherColumns}) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2 }), - ClickHouseTcpColumn.Create("attrs", attrs), - ClickHouseTcpColumn.Create("point", new[] { (1, "one"), (2, "two") }), - ClickHouseTcpColumn.Create("score", new double?[] { 1.25, null }), - ClickHouseTcpColumn.Create("city", new[] { "Amsterdam", "Amsterdam" }), - ClickHouseTcpColumn.Create("nick", new string?[] { "ada", null }), - }); - - Console.WriteLine(" Map(String, Int64) is KeyValuePair[] per row, not a"); - Console.WriteLine(" Dictionary: the wire carries the keys and the values"); - Console.WriteLine(" as two columns in order, which a pair array matches."); - Console.WriteLine(" Tuple(x Int32, y String) is (int, string) per row. The element names live in"); - Console.WriteLine(" the type string only, so an unnamed ValueTuple is"); - Console.WriteLine(" what a named tuple takes."); - Console.WriteLine(" Nullable(Float64) is double? per row. A reference type is already"); - Console.WriteLine(" nullable, so Nullable(String) takes string."); - Console.WriteLine(" LowCardinality(String) is plain string per row. The client works out the"); - Console.WriteLine(" block's dictionary and its key width; you never"); - Console.WriteLine(" build either."); - Console.WriteLine(" LowCardinality(Nullable(String)) is string per row, null allowed.\n"); - - await ShowOthers(client, OthersTable); - - Console.WriteLine("\n All five take section 3's round trip too. A Map arrives as its key and value columns,"); - Console.WriteLine(" a Tuple as its element columns, a Nullable as a null map plus its inner column, a"); - Console.WriteLine(" LowCardinality as a dictionary plus its keys, and each of those is the layout its codec"); - Console.WriteLine($" writes from. Re-inserted into '{OthersDenseTable}' straight from the read:\n"); - - await foreach (Block block in client.StreamAsync($"SELECT {OtherColumns} FROM {OthersTable} ORDER BY id")) - { - await client.InsertAsync($"INSERT INTO {OthersDenseTable} ({OtherColumns}) VALUES", block.Columns.ToArray()); - } - - await ShowOthers(client, OthersDenseTable); - } - - private static async Task WhatEachTargetAccepts(ClickHouseTcpClient client) - { - Console.WriteLine("\n5. What a composite refuses, and what it says\n"); - - // Both of these are type checks against the target's schema, so they are decided before any row is - // written and the message is the only cost. - await ShowRejection( - client, - "a Dictionary for a Map row", - $"INSERT INTO {OthersTable} (id, attrs) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 3 }), - ClickHouseTcpColumn.Create("attrs", new[] { new Dictionary { ["floor"] = 3 } }), - }); - - await ShowRejection( - client, - "a double for a Nullable(Float64) row", - $"INSERT INTO {OthersTable} (id, score) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 3 }), - ClickHouseTcpColumn.Create("score", new[] { 1.0 }), - }); - - Console.WriteLine(); - Console.WriteLine(" Both messages name the target type, which is the useful half. The CLR half is spelled as"); - Console.WriteLine(" the internal column class, so read its type argument (System.Double here) and compare"); - Console.WriteLine(" that with the list in section 4.\n"); - - // A Map row has the same non-nullable rule as an Array row, and is checked at the same point: while the - // block is being encoded, not before it. - await ShowRejection( - client, - "a null Map row", - $"INSERT INTO {OthersTable} (id, attrs) VALUES", - new IColumn[] - { - ClickHouseTcpColumn.Create("id", new ulong[] { 3 }), - ClickHouseTcpColumn.Create("attrs", new KeyValuePair[]?[] { null }), - }); - - Console.WriteLine("\n Same rule as Array(T), same two ways out: an empty pair array for an empty map, or"); - Console.WriteLine(" Map(K, Nullable(V)) to carry null values. And like section 2's, it is a check on the"); - Console.WriteLine(" values rather than on the types, so it fires later than the two above."); - } - - private static void WhatToRemember() - { - Console.WriteLine("\n6. What to remember\n"); - Console.WriteLine(" Pick the CLR type from the target, not from what is convenient: one array per row for"); - Console.WriteLine(" Array(T), one pair array per row for Map(K, V), a ValueTuple for Tuple(...), T? for"); - Console.WriteLine(" Nullable(T), and the plain value for LowCardinality(T)."); - Console.WriteLine(" A row of Array(T) or Map(K, V) is never null. Use an empty array, or make the elements"); - Console.WriteLine(" nullable."); - Console.WriteLine(" A column read out of a block is a valid insert column, and re-inserts without its"); - Console.WriteLine(" composite layout being rebuilt: it is already in the layout the codec writes from."); - Console.WriteLine(" Re-insert it inside the iteration that yielded it, because the block is borrowed."); - Console.WriteLine(" Match the column's name to the target, in the SELECT if need be."); - } - - private static string OthersDdl(string table) => $@" - CREATE TABLE {table} - ( - id UInt64, - attrs Map(String, Int64), - point Tuple(x Int32, y String), - score Nullable(Float64), - city LowCardinality(String), - nick LowCardinality(Nullable(String)) - ) - ENGINE = MergeTree() - ORDER BY id"; - - private static string ArrayDdl(string table) => $@" - CREATE TABLE {table} - ( - id UInt64, - readings Array(Float64), - tags Array(String), - maybe Array(Nullable(Int32)) - ) - ENGINE = MergeTree() - ORDER BY id"; - - private static async Task ShowArrays(ClickHouseTcpClient client, string table) - { - Console.WriteLine(" id readings tags maybe"); - Console.WriteLine(" -- ---------------- ----------------- ----------------"); - await foreach (object[] row in client.QueryAsync( - $"SELECT id, toString(readings), toString(tags), toString(maybe) FROM {table} ORDER BY id")) - { - Console.WriteLine($" {row[0],2} {row[1],-16} {row[2],-17} {row[3],-16}"); - } - } - - private static async Task ShowOthers(ClickHouseTcpClient client, string table) - { - Console.WriteLine(" id attrs point score city nick"); - Console.WriteLine(" -- ------------------------ ---------- ----- --------- ----"); - await foreach (object[] row in client.QueryAsync( - $@"SELECT id, toString(attrs), toString(point), toString(score), city, toString(nick) - FROM {table} ORDER BY id")) - { - Console.WriteLine($" {row[0],2} {row[1],-24} {row[2],-10} {Text(row[3]),-5} {row[4],-9} {Text(row[5])}"); - } - } - - // toString of a NULL is NULL, not the empty string: toString keeps the argument's nullability, so the row - // tier hands back a null reference here. - private static string Text(object value) => value?.ToString() ?? "NULL"; - - // Runs an insert that is expected to be rejected client-side and prints the reason. - private static async Task ShowRejection(ClickHouseTcpClient client, string what, string sql, IReadOnlyList columns) - { - try - { - await client.InsertAsync(sql, columns); - Console.WriteLine($" {what}: accepted, which this example did not expect"); - } - catch (ArgumentException ex) - { - Console.WriteLine($" {what}:"); - Console.WriteLine($" {ex.Message.Split(" (Parameter")[0]}"); - } - } -} From 1fae9ef001acbac3570a6d2c46901068ba2bdd73 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sat, 29 Aug 2026 20:29:04 +0200 Subject: [PATCH 14/16] Explain TCP allocation tradeoffs --- examples/Tcp/Read/Tcp_001_ReadTiers.cs | 6 +++--- examples/Tcp/Read/Tcp_004_Poco.cs | 2 ++ examples/Tcp/Write/Tcp_001_ColumnarInsert.cs | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/Tcp/Read/Tcp_001_ReadTiers.cs b/examples/Tcp/Read/Tcp_001_ReadTiers.cs index 95484c28f..44eae514f 100644 --- a/examples/Tcp/Read/Tcp_001_ReadTiers.cs +++ b/examples/Tcp/Read/Tcp_001_ReadTiers.cs @@ -35,21 +35,21 @@ await client.InsertRowsAsync( new object[] { 3UL, "Singapore", 28.0 }, }); - // Row reads need no model and expose the values in their wire representation. + // object[] rows need no model, but allocate an array per row and box value types. Console.WriteLine("QueryAsync: flexible object[] rows"); await foreach (object[] row in client.QueryAsync(Sql)) { Console.WriteLine($" {row[0]}: {row[1]}, {row[2]} °C"); } - // POCO reads map column names to properties and convert compatible values. + // POCO reads map and convert by name, allocating one Reading object per row. Console.WriteLine("QueryAsync: strongly typed objects"); await foreach (Reading row in client.QueryAsync(Sql)) { Console.WriteLine($" {row.Id}: {row.City}, {row.Temperature} °C"); } - // Block reads are best for column-oriented work and avoid materializing each row. + // Blocks expose borrowed column buffers without per-row materialization; prefer them for throughput. Console.WriteLine("StreamAsync: columnar blocks"); await foreach (Block block in client.StreamAsync(Sql)) { diff --git a/examples/Tcp/Read/Tcp_004_Poco.cs b/examples/Tcp/Read/Tcp_004_Poco.cs index 9a2eb4146..cb0ec608a 100644 --- a/examples/Tcp/Read/Tcp_004_Poco.cs +++ b/examples/Tcp/Read/Tcp_004_Poco.cs @@ -49,6 +49,8 @@ await client.InsertRowsAsync( $"INSERT INTO {TableName} (id, full_name, signal_count, recorded_at) VALUES", rows); + // POCO mapping is usually slower than block iteration: it allocates and fills one object per row. + // StreamAsync exposes borrowed column buffers and avoids those per-row object allocations. await foreach (Observation row in client.QueryAsync( $"SELECT id, full_name, signal_count, recorded_at, internal_notes " + $"FROM {TableName} ORDER BY id")) diff --git a/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs b/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs index 82835407b..2c4108797 100644 --- a/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs +++ b/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs @@ -30,6 +30,7 @@ ORDER BY id var names = new[] { "Ada", "Grace", "Alan" }; var scores = new[] { 99.5, 97.25, 91.0 }; + // Typed columns avoid the row-to-column projection performed by InsertRowsAsync. // Column names, not argument order, determine the target column. var columns = new IColumn[] { From e9269d153caac44c9d9eeb6bcc4a56ac3356d87e Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Sun, 30 Aug 2026 10:03:57 +0200 Subject: [PATCH 15/16] Use the block-tier projection surface in the examples Five native examples and the two indexes now teach what the block tier can do: - Tcp_001_ScalarTypes writes an Enum from a label and a String from raw bytes, then reads both back through IEnumColumn, IStringColumn, and Block.ReadAs. - Tcp_002_DateTimeAndTimezones reads a whole DateTime column as DateTimeOffset with Block.ReadAs. - Tcp_003_CompositeRead walks every column through the non-generic composite views, with no element type known. - Tcp_004_VariantDynamicJson names each row's alternative from IVariantColumn.TypeNames instead of printing bare discriminators. - Tcp_002_CompositeWrites builds the flat array shape with ClickHouseTcpColumn.CreateArray and asks ClickHouseTcpTypes.CanWrite which CLR types a target column accepts. Co-Authored-By: Claude Opus 5 (1M context) --- examples/README.md | 6 ++--- examples/Tcp/README.md | 6 +++-- examples/Tcp/Types/Tcp_001_ScalarTypes.cs | 26 +++++++++++++++--- .../Tcp/Types/Tcp_002_DateTimeAndTimezones.cs | 4 +++ examples/Tcp/Types/Tcp_003_CompositeRead.cs | 20 +++++++++++++- .../Tcp/Types/Tcp_004_VariantDynamicJson.cs | 13 ++++++--- examples/Tcp/Write/Tcp_002_CompositeWrites.cs | 27 ++++++++++++++----- 7 files changed, 83 insertions(+), 19 deletions(-) diff --git a/examples/README.md b/examples/README.md index 0129164bf..ac7dc5512 100644 --- a/examples/README.md +++ b/examples/README.md @@ -116,13 +116,13 @@ These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/READ ### Native Protocol: Writing Data - [Tcp_001_ColumnarInsert.cs](Tcp/Write/Tcp_001_ColumnarInsert.cs) - Insert typed columns and let ClickHouse fill defaults -- [Tcp_002_CompositeWrites.cs](Tcp/Write/Tcp_002_CompositeWrites.cs) - Insert composite values and reuse a column from a block +- [Tcp_002_CompositeWrites.cs](Tcp/Write/Tcp_002_CompositeWrites.cs) - Insert composite values, build the flat array shape, and reuse a column from a block ### Native Protocol: Data Types -- [Tcp_001_ScalarTypes.cs](Tcp/Types/Tcp_001_ScalarTypes.cs) - Inspect representative ClickHouse-to-CLR scalar mappings +- [Tcp_001_ScalarTypes.cs](Tcp/Types/Tcp_001_ScalarTypes.cs) - Inspect scalar type mappings, enum labels, and raw `String` bytes - [Tcp_002_DateTimeAndTimezones.cs](Tcp/Types/Tcp_002_DateTimeAndTimezones.cs) - Read and write date, time, and timezone-aware values -- [Tcp_003_CompositeRead.cs](Tcp/Types/Tcp_003_CompositeRead.cs) - Read arrays, maps, tuples, nullable values, and nested data +- [Tcp_003_CompositeRead.cs](Tcp/Types/Tcp_003_CompositeRead.cs) - Read composites through their typed views, or without knowing the element type - [Tcp_004_VariantDynamicJson.cs](Tcp/Types/Tcp_004_VariantDynamicJson.cs) - Read `Variant`, `Dynamic`, and `JSON` columns - [Tcp_005_QBitVectorSearch.cs](Tcp/Types/Tcp_005_QBitVectorSearch.cs) - Inspect `QBit` planes and run approximate vector search diff --git a/examples/Tcp/README.md b/examples/Tcp/README.md index 4a03d940a..55d71ab85 100644 --- a/examples/Tcp/README.md +++ b/examples/Tcp/README.md @@ -40,5 +40,7 @@ Use the HTTP client when you need: - a per-query database or role. One type detail is easy to miss: row reads return `DateTime`, `DateTime64`, `Time`, and `Time64` as -their wire integer values. `QueryAsync` converts mapped POCO properties, while block reads expose -`IDateTimeColumn` and `ITimeColumn` for typed conversion. +their wire integer values. `QueryAsync` converts mapped POCO properties. On the block tier, +`Block.Column` is a cast to the type the column decoded to, while `Block.ReadAs` converts to +any other type the ClickHouse type offers, and `IDateTimeColumn` and `ITimeColumn` convert per row. +`ClickHouseTcpTypes.CanRead` and `CanWrite` answer which CLR types a given ClickHouse type accepts. diff --git a/examples/Tcp/Types/Tcp_001_ScalarTypes.cs b/examples/Tcp/Types/Tcp_001_ScalarTypes.cs index 491986335..d06d78ec3 100644 --- a/examples/Tcp/Types/Tcp_001_ScalarTypes.cs +++ b/examples/Tcp/Types/Tcp_001_ScalarTypes.cs @@ -82,7 +82,9 @@ await client.InsertAsync( 20), }), ClickHouseTcpColumn.Create("flag", new[] { true }), - ClickHouseTcpColumn.Create("text", new[] { "hello" }), + + // String is byte-oriented, so it takes raw bytes as well as text. 0xFF is not valid UTF-8. + ClickHouseTcpColumn.Create("text", new[] { new byte[] { 0x68, 0x69, 0xFF } }), // FixedString is binary data; byte[] preserves zeros and non-UTF-8 bytes. ClickHouseTcpColumn.Create( @@ -94,8 +96,8 @@ await client.InsertAsync( ClickHouseTcpColumn.Create("ip4", new[] { IPAddress.Parse("192.168.0.1") }), ClickHouseTcpColumn.Create("ip6", new[] { IPAddress.Parse("2001:db8::1") }), - // Enum columns use their signed integer storage type on the block tier. - ClickHouseTcpColumn.Create("colour", new sbyte[] { 2 }), + // An Enum stores a signed integer, and accepts either that ordinal or a declared label. + ClickHouseTcpColumn.Create("colour", new[] { "green" }), }); await foreach (Block block in client.StreamAsync($"SELECT {Columns} FROM {TableName}")) @@ -106,6 +108,24 @@ await client.InsertAsync( $"{column.Name,-7} {column.TypeName,-28} " + $"-> {FriendlyName(column.ElementType),-20} {Render(column.GetValue(0))}"); } + + Console.WriteLine(); + + // ElementType above is the default reading, not the only one a type offers. A String decodes as + // UTF-8, so the table lost 0xFF; IStringColumn returns the bytes the server sent. + var text = (IStringColumn)block["text"]; + Console.WriteLine($"text as raw bytes: 0x{Convert.ToHexString(text.GetBytes(0))}"); + + // An Enum decodes as its ordinal; IEnumColumn carries the labels the type string declares. + var colour = (IEnumColumn)block["colour"]; + Console.WriteLine( + "colour members: " + + string.Join(", ", colour.Members.Select(member => $"{member.Key}={member.Value}"))); + Console.WriteLine($"colour label of row 0: {colour.GetLabel(0)}"); + + // ReadAs asks for a reading by CLR type, so the whole column arrives as labels. + IColumn labels = block.ReadAs("colour"); + Console.WriteLine($"colour read as string: {labels[0]}"); } } finally diff --git a/examples/Tcp/Types/Tcp_002_DateTimeAndTimezones.cs b/examples/Tcp/Types/Tcp_002_DateTimeAndTimezones.cs index 9c2f0b685..d258ac335 100644 --- a/examples/Tcp/Types/Tcp_002_DateTimeAndTimezones.cs +++ b/examples/Tcp/Types/Tcp_002_DateTimeAndTimezones.cs @@ -66,6 +66,10 @@ await client.InsertAsync( PrintTimestamp((IDateTimeColumn)block["precise_at"]); PrintTime((ITimeColumn)block["elapsed"]); PrintTime((ITimeColumn)block["precise_elapsed"]); + + // ReadAs converts a whole column to a reading its type offers, applying the same zone and scale. + IColumn captured = block.ReadAs("captured_at"); + Console.WriteLine($"captured_at as DateTimeOffset: {captured[0]:O}"); } var utc = new ClickHouseTcpQueryOptions diff --git a/examples/Tcp/Types/Tcp_003_CompositeRead.cs b/examples/Tcp/Types/Tcp_003_CompositeRead.cs index 47e93126d..5ca6c5b95 100644 --- a/examples/Tcp/Types/Tcp_003_CompositeRead.cs +++ b/examples/Tcp/Types/Tcp_003_CompositeRead.cs @@ -86,13 +86,21 @@ await client.InsertAsync( $"LowCardinality dictionary [{string.Join(", ", city.Dictionary.Values.ToArray())}], " + $"keys [{string.Join(", ", city.Keys.ToArray())}]"); } + + // A typed view needs the element type, and a wrong type argument compiles and never matches. The + // non-generic views walk the same storage when the types are not known in advance. + Console.WriteLine("Shapes read without a type argument:"); + foreach (IColumn column in block.Columns) + { + Console.WriteLine($" {column.Name}: {Describe(column)}"); + } } // Geo aliases use the same column shape as their underlying tuple types. await foreach (Block block in client.StreamAsync( "SELECT CAST((1.0, 2.0), 'Point') AS point")) { - Console.WriteLine($"Point uses {block["point"].GetType().Name} and materializes as " + + Console.WriteLine($"Point is a {Describe(block["point"])} and materializes as " + $"{block["point"].GetValue(0)}"); } } @@ -101,4 +109,14 @@ await client.InsertAsync( await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); } } + + private static string Describe(IColumn column) => column switch + { + IArrayColumn array => $"Array of {Describe(array.Inner)}", + IMapColumn map => $"Map of {Describe(map.KeyColumn)} to {Describe(map.ValueColumn)}", + INullableColumn nullable => $"Nullable {Describe(nullable.Inner)}", + ILowCardinalityColumn dictionary => $"LowCardinality of {Describe(dictionary.Dictionary)}", + ITupleColumn tuple => $"Tuple of ({string.Join(", ", tuple.Children.Select(Describe))})", + _ => column.ElementType.Name, + }; } diff --git a/examples/Tcp/Types/Tcp_004_VariantDynamicJson.cs b/examples/Tcp/Types/Tcp_004_VariantDynamicJson.cs index 9c3e2c540..4c02bfd79 100644 --- a/examples/Tcp/Types/Tcp_004_VariantDynamicJson.cs +++ b/examples/Tcp/Types/Tcp_004_VariantDynamicJson.cs @@ -102,13 +102,18 @@ private static async Task PrintVariant(ClickHouseTcpClient client) $"SELECT value FROM {VariantTable} ORDER BY id")) { var column = (IVariantColumn)block["value"]; - Console.WriteLine( - $"Variant: {column.TypeCount} alternatives, discriminators " + - $"[{string.Join(", ", column.Discriminators.ToArray())}]"); + + // TypeNames lists the alternatives in discriminator order. The server sorts them by name, so the + // declared order says nothing, and it is the only route to what a discriminator selects. + Console.WriteLine($"Variant alternatives: [{string.Join(", ", column.TypeNames)}]"); for (int row = 0; row < column.RowCount; row++) { - Console.WriteLine($" {FormatValue(block["value"].GetValue(row))}"); + byte discriminator = column.Discriminators[row]; + string alternative = discriminator == IVariantColumn.NullDiscriminator + ? "no alternative" + : column.TypeNames[discriminator]; + Console.WriteLine($" {alternative}: {FormatValue(block["value"].GetValue(row))}"); } } } diff --git a/examples/Tcp/Write/Tcp_002_CompositeWrites.cs b/examples/Tcp/Write/Tcp_002_CompositeWrites.cs index 3b62aee49..1d1cb5298 100644 --- a/examples/Tcp/Write/Tcp_002_CompositeWrites.cs +++ b/examples/Tcp/Write/Tcp_002_CompositeWrites.cs @@ -28,11 +28,26 @@ public static async Task Run() await client.ExecuteAsync(CreateTable(SourceTable)); await client.ExecuteAsync(CreateTable(CopyTable)); - var readings = new[] + // Which CLR type a target column takes is a question the client answers, not a table to maintain. + foreach ((string Type, Type Element, string Label) candidate in new[] { - new[] { 0.5, 0.75, 1.0 }, - Array.Empty(), - }; + ("Array(Float64)", typeof(double[]), "double[]"), + ("Map(String, Int64)", typeof(KeyValuePair[]), "KeyValuePair[]"), + ("Map(String, Int64)", typeof(Dictionary), "Dictionary"), + }) + { + Console.WriteLine( + $"{candidate.Type} from {candidate.Label}: " + + $"{ClickHouseTcpTypes.CanWrite(candidate.Type, candidate.Element)}"); + } + + // CreateArray builds the flat native shape: every row's values end to end, plus the per-row offsets + // into them. Row 0 takes the first three values and row 1 is empty. + IArrayColumn readings = ClickHouseTcpColumn.CreateArray( + "readings", + ClickHouseTcpColumn.Create("readings", new[] { 0.5, 0.75, 1.0 }), + new[] { 0, 3, 3 }); + var attributes = new[] { new[] @@ -43,13 +58,13 @@ public static async Task Run() Array.Empty>(), }; - // Use one array or map per row, ValueTuple for Tuple, and nullable CLR values for Nullable. + // Create takes one array or map per row, ValueTuple for Tuple, and nullable CLR values for Nullable. await client.InsertAsync( $"INSERT INTO {SourceTable} ({Columns}) VALUES", new IColumn[] { ClickHouseTcpColumn.Create("id", new ulong[] { 1, 2 }), - ClickHouseTcpColumn.Create("readings", readings), + readings, ClickHouseTcpColumn.Create("attributes", attributes), ClickHouseTcpColumn.Create("point", new[] { (1, "one"), (2, "two") }), ClickHouseTcpColumn.Create("score", new double?[] { 1.25, null }), From da30261804e8bf977fe23e84ac68a388030b6bdb Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Mon, 31 Aug 2026 17:27:17 +0200 Subject: [PATCH 16/16] Use the surface the public-API pass added Each of these hand-rolled something the client can now be asked for, or asserted something it can now show. - ITupleColumn.FieldNames is empty rather than null for an unnamed tuple, so the null-coalesce is dead. - ClickHouseTcpClientOptions.ResolvedPort replaces two hand-written derivations of the default port, one of which hard-coded 9000 and 9440. - ClickHouseTcpInsertOptions.DeduplicationToken replaces the raw setting. - ClickHouseTcpServerInfo separates the three protocol revisions, so the server info example prints which one is in force rather than one unlabelled number. - A query with no QueryId gets one from the client, which the log lines carry. - OnBlockWritten shows the block sizing MaxRowsPerBlock produced, and measures that the client's codec governs what an insert writes: 160,023 bytes go out as 80,083 under LZ4 and 33,850 under ZSTD. Co-Authored-By: Claude Opus 5 (1M context) --- examples/README.md | 2 +- .../Advanced/Tcp_001_SettingsAndQueryId.cs | 5 +++ .../Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs | 18 ++++---- examples/Tcp/Advanced/Tcp_005_Compression.cs | 44 +++++++++++++++++-- examples/Tcp/Advanced/Tcp_006_ServerInfo.cs | 4 +- examples/Tcp/Connection/Tcp_003_Tls.cs | 6 +-- examples/Tcp/Core/Tcp_002_ConnectionString.cs | 2 +- examples/Tcp/Types/Tcp_003_CompositeRead.cs | 3 +- examples/Tcp/Write/Tcp_001_ColumnarInsert.cs | 12 ++++- 9 files changed, 74 insertions(+), 22 deletions(-) diff --git a/examples/README.md b/examples/README.md index ac7dc5512..3ed4ea65f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -138,7 +138,7 @@ These use `ClickHouseTcpClient` and need port 9000. See [Tcp/README.md](Tcp/READ - [Tcp_001_SettingsAndQueryId.cs](Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs) - Apply settings and assign a query ID - [Tcp_002_ProgressAndStatistics.cs](Tcp/Advanced/Tcp_002_ProgressAndStatistics.cs) - Receive progress and profile callbacks - [Tcp_003_Cancellation.cs](Tcp/Advanced/Tcp_003_Cancellation.cs) - Cancel row, block, and command operations -- [Tcp_004_ErrorsAndRetries.cs](Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs) - Handle errors and retry transient reads safely +- [Tcp_004_ErrorsAndRetries.cs](Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs) - Handle errors and retry a failed read safely - [Tcp_005_Compression.cs](Tcp/Advanced/Tcp_005_Compression.cs) - Select a native compression codec - [Tcp_006_ServerInfo.cs](Tcp/Advanced/Tcp_006_ServerInfo.cs) - Read handshake metadata and gate optional features diff --git a/examples/Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs b/examples/Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs index a20c6bb80..94f272be7 100644 --- a/examples/Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs +++ b/examples/Tcp/Advanced/Tcp_001_SettingsAndQueryId.cs @@ -39,5 +39,10 @@ public static async Task Run() object nextSetting = await client.ExecuteScalarAsync("SELECT getSetting('max_threads')"); Console.WriteLine($"Next query uses the client default again: {nextSetting}"); + + // With no QueryId the client generates one, so every operation is correlatable with + // system.query_log. The protocol never sends the server's own id back. + object generated = await client.ExecuteScalarAsync("SELECT currentQueryID()"); + Console.WriteLine($"Client-generated query ID: {generated}"); } } diff --git a/examples/Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs b/examples/Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs index 48673f7d1..8ffd0dcfe 100644 --- a/examples/Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs +++ b/examples/Tcp/Advanced/Tcp_004_ErrorsAndRetries.cs @@ -2,7 +2,7 @@ namespace ClickHouse.Driver.Examples; -/// Handles server, transport, and protocol errors and retries transient reads. +/// Handles server, transport, and protocol errors and retries a read that failed to connect. public static class TcpErrorsAndRetries { private const string TableName = "example_tcp_retry_deduplication"; @@ -18,8 +18,7 @@ public static async Task Run() } catch (ClickHouseTcpServerException ex) { - Console.WriteLine( - $"Server error: {ex.Code} ({ex.RawCode}), transient={ex.IsTransient}"); + Console.WriteLine($"Server error: {ex.Code} ({ex.RawCode})"); } await using var unreachable = new ClickHouseTcpClient( @@ -29,7 +28,7 @@ public static async Task Run() DialTimeout = TimeSpan.FromSeconds(1), }); - // A transient read is safe to retry. A failed write may already have reached the server. + // A read is safe to retry. A failed write may already have reached the server. int attempts = 0; object result = await RetryRead(async () => { @@ -54,10 +53,7 @@ ORDER BY id // Reuse one token for retries of the same logical batch. The table must enable deduplication. var insertOptions = new ClickHouseTcpInsertOptions { - Settings = new Dictionary - { - ["insert_deduplication_token"] = "example-logical-batch-1", - }, + DeduplicationToken = "example-logical-batch-1", }; await client.InsertRowsAsync( @@ -88,9 +84,11 @@ private static async Task RetryRead(Func> operation) { return await operation(); } - catch (ClickHouseTcpException ex) when (ex.IsTransient && attempt < MaxAttempts) + // The connection failed, so the read never ran. Which failures are worth a retry is the + // caller's policy: a server rejection of the query itself would repeat, so it is not caught. + catch (ClickHouseTcpTransportException ex) when (attempt < MaxAttempts) { - Console.WriteLine($"Transient {ex.GetType().Name}; retrying."); + Console.WriteLine($"{ex.GetType().Name}; retrying."); await Task.Delay(TimeSpan.FromMilliseconds(100 * attempt)); } } diff --git a/examples/Tcp/Advanced/Tcp_005_Compression.cs b/examples/Tcp/Advanced/Tcp_005_Compression.cs index 7a2a4e1a6..9cbf4c139 100644 --- a/examples/Tcp/Advanced/Tcp_005_Compression.cs +++ b/examples/Tcp/Advanced/Tcp_005_Compression.cs @@ -5,9 +5,18 @@ namespace ClickHouse.Driver.Examples; /// Selects LZ4, Zstandard, or no native block compression. public static class TcpCompression { + private const string TableName = "example_tcp_compression"; + public static async Task Run() { - // Compression applies to native data blocks; it does not change query results. + // Compression governs what an insert writes. It does not choose what the server sends: the request + // carries a flag and no codec name, so the server frames its blocks with network_compression_method. + var rows = new ulong[20000]; + for (int i = 0; i < rows.Length; i++) + { + rows[i] = (ulong)i; + } + foreach (string codec in new[] { "lz4", "zstd", "none" }) { var builder = ExampleConfig.TcpBuilder(); @@ -15,12 +24,39 @@ public static async Task Run() ClickHouseTcpClientOptions options = builder.ToOptions(); await using var client = new ClickHouseTcpClient(options); - object rows = await client.ExecuteScalarAsync("SELECT count() FROM numbers(10000)"); + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + try + { + await client.ExecuteAsync($"CREATE TABLE {TableName} (id UInt64) ENGINE = Memory"); + + long uncompressed = 0; + long compressed = 0; + await client.InsertAsync( + $"INSERT INTO {TableName} (id) VALUES", + new IColumn[] { ClickHouseTcpColumn.Create("id", rows) }, + new ClickHouseTcpInsertOptions + { + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnBlockWritten = block => + { + uncompressed += block.UncompressedBytes; + compressed += block.CompressedBytes; + }, + }, + }); - string compressor = options.Compressor?.GetType().Name ?? "none"; - Console.WriteLine($"Compression={codec,-4} -> {compressor,-20}; rows={rows}"); + string compressor = options.Compressor?.GetType().Name ?? "none"; + Console.WriteLine( + $"Compression={codec,-4} -> {compressor,-20}; insert wrote {uncompressed} bytes as {compressed}"); + } + finally + { + await client.ExecuteAsync($"DROP TABLE IF EXISTS {TableName}"); + } } + // Set network_compression_method to change what a query result is compressed with. Console.WriteLine("LZ4 is the default. Zstandard usually trades more CPU for smaller payloads."); Console.WriteLine("Choose a codec with measurements from your workload and network."); } diff --git a/examples/Tcp/Advanced/Tcp_006_ServerInfo.cs b/examples/Tcp/Advanced/Tcp_006_ServerInfo.cs index 95d928c7a..e6dbb39ce 100644 --- a/examples/Tcp/Advanced/Tcp_006_ServerInfo.cs +++ b/examples/Tcp/Advanced/Tcp_006_ServerInfo.cs @@ -15,7 +15,9 @@ public static async Task Run() Console.WriteLine($"Name: {server.Name}"); Console.WriteLine($"Version: {server.Version}"); - Console.WriteLine($"Protocol revision: {server.ProtocolRevision}"); + // The negotiated revision is the lower of the two the client and the server support. + Console.WriteLine($"Protocol revision: {server.ProtocolRevision} in force"); + Console.WriteLine($" server advertised {server.ServerProtocolRevision}, client supports {server.ClientProtocolRevision}"); Console.WriteLine($"Timezone: {server.Timezone}"); Console.WriteLine($"Display name: {server.DisplayName}"); diff --git a/examples/Tcp/Connection/Tcp_003_Tls.cs b/examples/Tcp/Connection/Tcp_003_Tls.cs index 4a78280d2..34f3b4513 100644 --- a/examples/Tcp/Connection/Tcp_003_Tls.cs +++ b/examples/Tcp/Connection/Tcp_003_Tls.cs @@ -18,9 +18,9 @@ public static async Task Run() ConfigureTls = tls => tls.EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, }; - Console.WriteLine($"Plain endpoint: {plain.Host}:{plain.Port ?? 9000}"); - Console.WriteLine($"Secure endpoint: {secure.Host}:{secure.Port ?? 9440}"); - Console.WriteLine("With Port unset, TLS uses the native secure port 9440."); + // ResolvedPort is the port a connection dials: Port when set, otherwise derived from UseTls. + Console.WriteLine($"Plain endpoint: {plain.Host}:{plain.ResolvedPort}"); + Console.WriteLine($"Secure endpoint: {secure.Host}:{secure.ResolvedPort}"); // Use TlsCaCertificatePath for a private CA. It replaces the host trust store. // TlsAllowInvalidCertificates is intended only for local development. diff --git a/examples/Tcp/Core/Tcp_002_ConnectionString.cs b/examples/Tcp/Core/Tcp_002_ConnectionString.cs index 8d36e618c..e11921744 100644 --- a/examples/Tcp/Core/Tcp_002_ConnectionString.cs +++ b/examples/Tcp/Core/Tcp_002_ConnectionString.cs @@ -17,7 +17,7 @@ public static async Task Run() builder["set_max_threads"] = 2; ClickHouseTcpClientOptions options = builder.ToOptions(); - Console.WriteLine($"Endpoint: {options.Host}:{options.Port ?? 9000}/{options.Database}"); + Console.WriteLine($"Endpoint: {options.Host}:{options.ResolvedPort}/{options.Database}"); Console.WriteLine($"Compression: {options.Compressor?.GetType().Name ?? "none"}"); Console.WriteLine($"Max pool size: {options.MaxPoolSize}"); diff --git a/examples/Tcp/Types/Tcp_003_CompositeRead.cs b/examples/Tcp/Types/Tcp_003_CompositeRead.cs index 5ca6c5b95..298e91129 100644 --- a/examples/Tcp/Types/Tcp_003_CompositeRead.cs +++ b/examples/Tcp/Types/Tcp_003_CompositeRead.cs @@ -70,7 +70,8 @@ await client.InsertAsync( if (block["point"] is ITupleColumn point) { Console.WriteLine( - $"Tuple fields [{string.Join(", ", point.FieldNames ?? Array.Empty())}]"); + // FieldNames is empty, not null, for an unnamed tuple. + $"Tuple fields [{string.Join(", ", point.FieldNames)}]"); } if (block["score"] is INullableColumn score) diff --git a/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs b/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs index 2c4108797..ff56aa3a4 100644 --- a/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs +++ b/examples/Tcp/Write/Tcp_001_ColumnarInsert.cs @@ -39,10 +39,20 @@ ORDER BY id ClickHouseTcpColumn.Create("name", names), }; + // An insert gets no server progress packets, so OnBlockWritten is its only progress. It also + // reports the block sizing MaxRowsPerBlock produced: 3 rows capped at 2 is a block of 2 then 1. await client.InsertAsync( $"INSERT INTO {TableName} (id, name, score) VALUES", columns, - new ClickHouseTcpInsertOptions { MaxRowsPerBlock = 2 }); + new ClickHouseTcpInsertOptions + { + MaxRowsPerBlock = 2, + Callbacks = new ClickHouseTcpQueryCallbacks + { + OnBlockWritten = block => Console.WriteLine( + $"Block {block.BlockIndex}: {block.RowCount} rows, {block.UncompressedBytes} bytes"), + }, + }); // region is absent from the statement, so ClickHouse applies its default. await client.InsertAsync(