From 68ade5afc89dcf8b08760c78e926a7ba47e39e3d Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:43:44 +0200 Subject: [PATCH 1/4] Add the json-observer-masking skill and a test project that runs every skill snippet Every csharp block under skills/ is compiled against the libraries and run: programs must print their // Output: comment, test classes run their [Fact]/[Theory] methods. --- ...tem.Text.Json.Observer.Skills.Tests.csproj | 13 + .../SkillDocuments.cs | 22 ++ .../SkillSnippetTests.cs | 249 +++++++++++++++ .../SkillStructureTests.cs | 70 ++++ .../xunit.runner.json | 3 + DragoAnt.System.Text.Json.slnx | 1 + skills/json-observer-masking/SKILL.md | 65 ++++ skills/json-observer-masking/examples.md | 301 ++++++++++++++++++ .../migrating-from-1x.md | 106 ++++++ skills/json-observer-masking/pitfalls.md | 144 +++++++++ skills/json-observer-masking/recipes.md | 202 ++++++++++++ 11 files changed, 1176 insertions(+) create mode 100644 DragoAnt.System.Text.Json.Observer.Skills.Tests/DragoAnt.System.Text.Json.Observer.Skills.Tests.csproj create mode 100644 DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillDocuments.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillSnippetTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillStructureTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Skills.Tests/xunit.runner.json create mode 100644 skills/json-observer-masking/SKILL.md create mode 100644 skills/json-observer-masking/examples.md create mode 100644 skills/json-observer-masking/migrating-from-1x.md create mode 100644 skills/json-observer-masking/pitfalls.md create mode 100644 skills/json-observer-masking/recipes.md diff --git a/DragoAnt.System.Text.Json.Observer.Skills.Tests/DragoAnt.System.Text.Json.Observer.Skills.Tests.csproj b/DragoAnt.System.Text.Json.Observer.Skills.Tests/DragoAnt.System.Text.Json.Observer.Skills.Tests.csproj new file mode 100644 index 0000000..4cd24c7 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Skills.Tests/DragoAnt.System.Text.Json.Observer.Skills.Tests.csproj @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillDocuments.cs b/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillDocuments.cs new file mode 100644 index 0000000..b2341f9 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillDocuments.cs @@ -0,0 +1,22 @@ +namespace DragoAnt.System.Text.Json.Observer.Skills.Tests; + +internal static class SkillDocuments +{ + public static string Root { get; } = Path.Combine(AppContext.BaseDirectory, "skills"); + + /// Every markdown file under skills/, as a path relative to it with forward slashes. + public static IReadOnlyList All { get; } = Directory.Exists(Root) + ? Directory.GetFiles(Root, "*.md", SearchOption.AllDirectories) + .Select(f => Path.GetRelativePath(Root, f).Replace('\\', '/')) + .Order(StringComparer.Ordinal) + .ToArray() + : []; + + /// Skill folder names: the directories holding a SKILL.md. + public static IReadOnlyList Skills { get; } = All + .Where(d => d.EndsWith("/SKILL.md", StringComparison.Ordinal)) + .Select(d => d[..d.IndexOf('/')]) + .ToArray(); + + public static string[] ReadLines(string document) => File.ReadAllLines(Path.Combine(Root, document)); +} diff --git a/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillSnippetTests.cs b/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillSnippetTests.cs new file mode 100644 index 0000000..8be4646 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillSnippetTests.cs @@ -0,0 +1,249 @@ +using System.Reflection; +using System.Runtime.ExceptionServices; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +namespace DragoAnt.System.Text.Json.Observer.Skills.Tests; + +/// +/// Every ```csharp block under skills/ is compiled against the current libraries and run. +/// A block with [Fact] or [Theory] is an xUnit test class: each test method runs and must pass. +/// Any other block is a program: it must run without throwing and print what its // Output: comment says. +/// A block preceded by <!-- doc-test: skip --> is a fragment and is not compiled. +/// +public sealed partial class SkillSnippetTests +{ + private static readonly Lazy References = new(LoadReferences); + + public static TheoryData Snippets() + { + var data = new TheoryData(); + foreach (var document in SkillDocuments.All) + { + var count = Extract(document).Count; + for (var i = 0; i < count; i++) + { + data.Add(document, i); + } + } + + return data; + } + + [Fact] + public void EverySkill_HasRunnableSnippets() + { + foreach (var skill in SkillDocuments.Skills) + { + SkillDocuments.All.Where(d => d.StartsWith(skill + "/", StringComparison.Ordinal)) + .Sum(d => Extract(d).Count) + .Should().BeGreaterThan(0, skill); + } + } + + [Theory] + [MemberData(nameof(Snippets))] + public async Task Snippet_CompilesAndRuns(string document, int index) + { + var snippet = Extract(document)[index]; + var where = $"{document} snippet at line {snippet.Line}"; + + var assembly = Compile(snippet, $"{document.Replace('/', '_').Replace('.', '_').Replace('-', '_')}_{index}", out var diagnostics); + assembly.Should().NotBeNull($"{where} must compile:{Environment.NewLine}{diagnostics}"); + + if (snippet.IsTestClass) + { + var ran = await RunTests(assembly!, where); + ran.Should().BeGreaterThan(0, $"{where} declares at least one test"); + return; + } + + var printed = await RunProgram(assembly!); + if (snippet.ExpectedOutput is { } expected) + { + printed.TrimEnd().Should().Be(expected, $"{where} prints its // Output: comment"); + } + } + + private sealed record Snippet(int Line, string Code, string? ExpectedOutput, bool IsTestClass); + + private static List Extract(string document) + { + var lines = SkillDocuments.ReadLines(document); + var snippets = new List(); + for (var i = 0; i < lines.Length; i++) + { + if (lines[i].Trim() != "```csharp") + { + continue; + } + + var skip = i > 0 && lines[i - 1].Contains("doc-test: skip", StringComparison.Ordinal); + var start = i + 1; + var end = start; + while (end < lines.Length && lines[end].Trim() != "```") + { + end++; + } + + if (!skip) + { + var code = lines[start..end]; + var isTestClass = code.Any(l => TestAttribute().IsMatch(l)); + snippets.Add(new Snippet(start + 1, string.Join('\n', code), isTestClass ? null : ExpectedOutput(code), isTestClass)); + } + + i = end; + } + + return snippets; + } + + private static string? ExpectedOutput(string[] code) + { + var at = Array.FindIndex(code, l => l.Trim() == "// Output:"); + if (at < 0) + { + return null; + } + + var output = code.Skip(at + 1).TakeWhile(l => l.TrimStart().StartsWith("//", StringComparison.Ordinal)) + .Select(l => l.TrimStart()[2..].TrimStart()); + return string.Join('\n', output); + } + + private static Assembly? Compile(Snippet snippet, string name, out string diagnostics) + { + var tree = CSharpSyntaxTree.ParseText(ImplicitUsings + snippet.Code, new CSharpParseOptions(LanguageVersion.Latest)); + var compilation = CSharpCompilation.Create( + name, + [tree], + References.Value, + new CSharpCompilationOptions( + snippet.IsTestClass ? OutputKind.DynamicallyLinkedLibrary : OutputKind.ConsoleApplication, + nullableContextOptions: NullableContextOptions.Enable)); + + using var stream = new MemoryStream(); + var result = compilation.Emit(stream); + diagnostics = string.Join(Environment.NewLine, result.Diagnostics.Where(d => d.Severity >= DiagnosticSeverity.Warning)); + return result.Success ? Assembly.Load(stream.ToArray()) : null; + } + + private static async Task RunProgram(Assembly assembly) + { + var entry = assembly.EntryPoint!; + var original = Console.Out; + var printed = new StringWriter(); + Console.SetOut(printed); + try + { + await Await(Invoke(entry, null, entry.GetParameters().Length == 0 ? null : [Array.Empty()])); + } + finally + { + Console.SetOut(original); + } + + return NewLines().Replace(printed.ToString(), "\n"); + } + + private static async Task RunTests(Assembly assembly, string where) + { + var ran = 0; + foreach (var type in assembly.GetTypes().Where(t => t is { IsClass: true, IsAbstract: false, IsPublic: true })) + { + foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static)) + { + var attributes = method.GetCustomAttributes().ToList(); + var test = attributes.FirstOrDefault(a => a.GetType().Name is "FactAttribute" or "TheoryAttribute"); + if (test is null || test.GetType().GetProperty("Skip")?.GetValue(test) is string) + { + continue; + } + + List rows = test.GetType().Name == "TheoryAttribute" + ? attributes.Where(a => a.GetType().Name == "InlineDataAttribute") + .Select(a => (object?[]?)a.GetType().GetProperty("Data")!.GetValue(a)) + .ToList() + : [null]; + rows.Should().NotBeEmpty($"{where}: {type.Name}.{method.Name} is a theory, so it needs [InlineData] rows"); + + foreach (var row in rows) + { + var instance = method.IsStatic ? null : Activator.CreateInstance(type); + try + { + await Await(Invoke(method, instance, row)); + ran++; + } + catch (Exception ex) + { + throw new InvalidOperationException( + $"{where}: {type.Name}.{method.Name}({string.Join(", ", row ?? [])}) failed: {ex.Message}", ex); + } + finally + { + switch (instance) + { + case IAsyncDisposable asyncDisposable: + await asyncDisposable.DisposeAsync(); + break; + case IDisposable disposable: + disposable.Dispose(); + break; + } + } + } + } + } + + return ran; + } + + private static object? Invoke(MethodInfo method, object? instance, object?[]? arguments) + { + try + { + return method.Invoke(instance, arguments); + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + ExceptionDispatchInfo.Capture(ex.InnerException).Throw(); + throw; + } + } + + private static async Task Await(object? result) + { + switch (result) + { + case Task task: + await task; + break; + case ValueTask valueTask: + await valueTask; + break; + } + } + + private const string ImplicitUsings = + "global using System;\nglobal using System.Collections.Generic;\nglobal using System.IO;\nglobal using System.Linq;\n" + + "global using System.Net.Http;\nglobal using System.Threading;\nglobal using System.Threading.Tasks;\n"; + + private static MetadataReference[] LoadReferences() + { + var platform = ((string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!).Split(Path.PathSeparator); + var local = Directory.GetFiles(AppContext.BaseDirectory, "*.dll"); + return platform.Concat(local) + .GroupBy(Path.GetFileName, StringComparer.OrdinalIgnoreCase) + .Select(g => (MetadataReference)MetadataReference.CreateFromFile(g.First())) + .ToArray(); + } + + [GeneratedRegex(@"^\s*\[(Fact|Theory)\b")] + private static partial Regex TestAttribute(); + + [GeneratedRegex("\r\n?")] + private static partial Regex NewLines(); +} diff --git a/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillStructureTests.cs b/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillStructureTests.cs new file mode 100644 index 0000000..b0fd828 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Skills.Tests/SkillStructureTests.cs @@ -0,0 +1,70 @@ +using System.Text.RegularExpressions; + +namespace DragoAnt.System.Text.Json.Observer.Skills.Tests; + +/// +/// Keeps the skills installable by any agent: open SKILL.md frontmatter only, and every relative link resolves. +/// +public sealed partial class SkillStructureTests +{ + private static readonly string[] ExpectedSkills = ["json-observer-http-logging", "json-observer-masking", "json-observer-testing"]; + + [Fact] + public void Skills_AreTheExpectedSet() => + SkillDocuments.Skills.Order(StringComparer.Ordinal).Should().Equal(ExpectedSkills); + + public static TheoryData SkillNames() => new(ExpectedSkills); + + [Theory] + [MemberData(nameof(SkillNames))] + public void SkillMd_HasOnlyNameAndDescriptionFrontmatter(string skill) + { + var lines = SkillDocuments.ReadLines($"{skill}/SKILL.md"); + lines[0].Should().Be("---"); + var end = Array.IndexOf(lines, "---", 1); + end.Should().BeGreaterThan(0, "the frontmatter is closed"); + + var fields = lines[1..end].ToDictionary(l => l[..l.IndexOf(':')], l => l[(l.IndexOf(':') + 1)..].Trim()); + fields.Keys.Should().BeEquivalentTo(["name", "description"]); + fields["name"].Should().Be(skill); + fields["name"].Should().MatchRegex("^[a-z0-9]+(-[a-z0-9]+)*$"); + fields["name"].Length.Should().BeLessThanOrEqualTo(64); + fields["description"].Length.Should().BeInRange(1, 1024); + lines.Length.Should().BeLessThan(500, "SKILL.md stays short; detail goes to companions"); + } + + public static TheoryData Documents() + { + var data = new TheoryData(); + foreach (var document in SkillDocuments.All) + { + data.Add($"skills/{document}"); + } + + data.Add("docs/skills.md"); + return data; + } + + [Theory] + [MemberData(nameof(Documents))] + public void RelativeLinks_Resolve(string document) + { + var path = Path.Combine(AppContext.BaseDirectory, document); + var text = File.ReadAllText(path); + foreach (Match link in RelativeLink().Matches(text)) + { + var target = link.Groups["target"].Value; + var file = target.Split('#')[0]; + var resolved = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(path)!, file)); + var inRepo = resolved.StartsWith(Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "skills")), StringComparison.OrdinalIgnoreCase) + || resolved.StartsWith(Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "docs")), StringComparison.OrdinalIgnoreCase); + if (inRepo) + { + File.Exists(resolved).Should().BeTrue($"{document} links to {target}"); + } + } + } + + [GeneratedRegex(@"\]\((?\.{1,2}/[^)\s]+\.md(#[^)\s]*)?)\)")] + private static partial Regex RelativeLink(); +} diff --git a/DragoAnt.System.Text.Json.Observer.Skills.Tests/xunit.runner.json b/DragoAnt.System.Text.Json.Observer.Skills.Tests/xunit.runner.json new file mode 100644 index 0000000..86c7ea0 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Skills.Tests/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json" +} diff --git a/DragoAnt.System.Text.Json.slnx b/DragoAnt.System.Text.Json.slnx index 8ef29b4..0c02e57 100644 --- a/DragoAnt.System.Text.Json.slnx +++ b/DragoAnt.System.Text.Json.slnx @@ -18,6 +18,7 @@ + diff --git a/skills/json-observer-masking/SKILL.md b/skills/json-observer-masking/SKILL.md new file mode 100644 index 0000000..3e7366f --- /dev/null +++ b/skills/json-observer-masking/SKILL.md @@ -0,0 +1,65 @@ +--- +name: json-observer-masking +description: Mask or extract values in JSON with DragoAnt.System.Text.Json.Observer before logging it — passwords, card numbers, tokens, emails, PII — in one streaming Utf8JsonReader to Utf8JsonWriter pass, without deserializing. Covers JsonObserver.Obj/Array/Any rules, absolute vs Relative property paths, PropMatches (EndsWith, Contains, OneOf, Regex), MaskAny/MaskStr/MaskTag (Full, Last4, Hash, Omit), default policies (AllowList, BlockList, NullList), allow-lists from DTOs with JsonShape.FromTypeInfo, extracting fields into a context with Read rules, the never-throw string and UTF-8 APIs with MaskResult/MaskStatus for cut-off or invalid bodies, JsonObserverOptions limits, and allocation-free hot paths. Use when redacting a JSON payload, writing or fixing JsonObserver rules, choosing a default policy, upgrading from 1.x, or when masked output looks wrong ("everything became ***", "the secret is still visible", "booleans are masked", "output is empty"). +--- + +# Masking JSON with DragoAnt.System.Text.Json.Observer + +`JsonObserver` rewrites a JSON payload token by token: values a rule names are masked (or handed to a context object), everything else follows a **default policy**. It never builds objects or a DOM, and it **never throws** — a cut-off or invalid payload yields its masked prefix, closed into valid JSON. + +Package: `DragoAnt.System.Text.Json.Observer` (net8.0, net9.0, net10.0). Namespaces: `DragoAnt.System.Text.Json.Observer`, `.Strategies` (`MaskTag`, `PropMatches`) and `using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies;` (`BlockList`, `AllowList`, `NullList`, `Relative`). + +## Quick start + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(Relative(rules => rules + .Match("password").MaskAny("***") + .Match("card", "number").MaskAny(MaskTag.Last4), + BlockList)); + +Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"number":"4111111111111111"},"active":true}""")); +// Output: +// {"user":"alice","password":"***","card":{"number":"***1111"},"active":true} +``` + +## Decision path + +1. **Do you have the DTO the JSON comes from?** Build an allow-list from it: `JsonShape.FromTypeInfo(typeInfo, classify)` + `JsonObserver.FromShape(shape)`. Known fields pass, the ones you classify are masked by tag, anything new is masked. Safest choice for third-party payloads → [examples.md#allow-list-from-a-type](./examples.md#allow-list-from-a-type). +2. **You only know which names are sensitive** ("mask every `password`, wherever it is") → **block-list**: `JsonObserver.Obj(Relative(rules => …, BlockList))`. +3. **You know which fields are safe to show** and want everything else hidden → **allow-list rules**: absolute rules with `.Unmasked()` under the default `AllowList`. +4. **You also need values out** (an order id for a log scope) → `JsonObserver.Obj(…)` with `Read*` rules; `Read(json, ctx)` extracts without writing → [examples.md#extract-values-while-masking](./examples.md#extract-values-while-masking). +5. **Hot path** (every request body) → the UTF-8 API `Mask(ReadOnlySpan, IBufferWriter)` with a reused `ArrayBufferWriter` → [recipes.md#hot-path-utf-8-api](./recipes.md#hot-path-utf-8-api). +6. **Body may be cut off** (a size-capped log, a stream prefix) → use the overload with `MaskResult` and look at `Status` → [recipes.md#cut-off-or-invalid-bodies](./recipes.md#cut-off-or-invalid-bodies). + +## Rules that matter + +1. **Build once, share everywhere.** An observer is immutable and thread-safe; keep it in a `static readonly` field. Building one per call costs far more than masking. +2. **The default policy is `AllowList`.** A factory without a policy, and `Relative(rules)` without its second argument, write every string, number **and boolean** no rule names as `"***"` (`null` stays). Pass `BlockList` to keep unnamed values. +3. **Absolute vs relative.** Rules on a builder (`Obj(root => root.Match("order").Obj(…))`) follow the path from the root. Rules inside `Relative(…)` match the **end** of a path at any depth: `Match("card", "number")` hits every `…card.number`. An **array item is one path level**, so reach `{"lines":[{"qty":…}]}` with `Match("lines", anyItem, "qty")` where `anyItem = new PropMatchingStrategy(_ => true)`. The first rule that matches wins. +4. **Names match exactly and case-insensitively.** Use `PropMatches.EndsWith/StartsWith/Contains/OneOf/Regex` for anything else. +5. **Prefer `MaskAny` for secrets.** Every `Mask*` rule masks the whole value whatever its JSON type (a number, a boolean, an object), but `MaskAny` keeps `null` as `null` without calling your function, while `MaskStr` passes `null` to it. +6. **Tags for standard masks:** `MaskAny(MaskTag.Full)` → `"***"`, `Last4` → `"***1111"` (shorter than 8 characters → `"***"`), `Hash` → `"hash:<16 hex>"`, `Omit` → `null`. Set `JsonObserverOptions.HashKey` for hashes that correlate across processes; the default key is random per process. +7. **Never throws, always safe.** Both APIs return the masked prefix of cut-off or invalid input, closed into valid JSON, and never write a masked value in clear. `MaskStatus` is `Masked`, `Truncated`, `Invalid` or `NotJson` (empty output: empty input, or a root that is not an object or array). +8. **Measure allocations, never assume zero.** With constant or tag rules, the UTF-8 API allocates a small constant amount per call; a masking function receives a `string`, which allocates. + +## Pitfalls (details in [pitfalls.md](./pitfalls.md)) + +- "Everything became `***`" → you used the default `AllowList`; pass `BlockList` (to the factory, or as `Relative`'s second argument). +- "The secret is still visible" → the rule is absolute but the field is nested, or the name differs (`Password` vs `passwd`); use `Relative` and a `PropMatches`. +- **Known 2.0.0 issue:** rules of an `Obj(...)` inside a property's `Array(...)` never match — under `BlockList` that value stays in clear. Use `Match("lines", anyItem, "qty")` instead ([pitfalls.md](./pitfalls.md#rules-inside-a-nested-array-do-not-match)). +- A rule written before an `Obj(...)` rule for the same name wins and masks the whole object. +- `JsonObserver.Obj(...)` on a root array returns `Invalid`; use `JsonObserver.Any(...)` when the root can be either. +- Comments in the input are accepted and never written; a UTF-8 BOM is skipped. + +## Companions + +- [examples.md](./examples.md) — runnable examples: rule kinds, policies, matchers, arrays, tags, shapes, extraction. +- [recipes.md](./recipes.md) — logging a request body, hot path, cut-off bodies, limits, hashing, custom strategies. +- [pitfalls.md](./pitfalls.md) — symptoms and their causes. +- [migrating-from-1x.md](./migrating-from-1x.md) — breaking changes from 1.x with before/after. + +Every C# block in these files is a complete program (top-level statements, implicit usings) that compiles against the library and prints what its `// Output:` comment shows. diff --git a/skills/json-observer-masking/examples.md b/skills/json-observer-masking/examples.md new file mode 100644 index 0000000..3ce75dc --- /dev/null +++ b/skills/json-observer-masking/examples.md @@ -0,0 +1,301 @@ +# Examples — json-observer-masking + +Each block is a complete program: create a console project, reference `DragoAnt.System.Text.Json.Observer`, paste it into `Program.cs`. The `// Output:` comment is what it prints. + +## Default policies + +The default policy decides what happens to a value no rule names. It is the last argument of `JsonObserver.Obj/Array/Any`, of `Relative(...)`, and of nested `Obj(...)`/`Array(...)` rules (a nested rule inherits the enclosing one when it is omitted). + +| Policy | `{"s":"x","n":1,"b":true,"z":null}` becomes | +| --- | --- | +| `AllowList` (the default) | `{"s":"***","n":"***","b":"***","z":null}` | +| `BlockList` | unchanged — only values a rule names are masked | +| `NullList` | `{"s":null,"n":null,"b":null,"z":null}` | + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +const string json = """{"s":"x","n":1,"b":true,"z":null}"""; + +Console.WriteLine(JsonObserver.Obj(AllowList).Mask(json)); +Console.WriteLine(JsonObserver.Obj(BlockList).Mask(json)); +Console.WriteLine(JsonObserver.Obj(NullList).Mask(json)); +Console.WriteLine(JsonObserver.Obj(root => root.Match("n").Unmasked()).Mask(json)); +// Output: +// {"s":"***","n":"***","b":"***","z":null} +// {"s":"x","n":1,"b":true,"z":null} +// {"s":null,"n":null,"b":null,"z":null} +// {"s":"***","n":1,"b":"***","z":null} +``` + +## Absolute rules + +Absolute rules follow the path from the root: one `Match` per level with a nested `Obj(...)`, or several names in one `Match`. Under the default `AllowList` this is an allow-list: name what may be shown with `Unmasked()`. + +```csharp +using DragoAnt.System.Text.Json.Observer; + +var masker = JsonObserver.Obj(root => root + .Match("id").Unmasked() + .Match("order").Obj(order => order + .Match("status").Unmasked() + .Match("total").Unmasked()) + .Match("customer", "country").Unmasked()); + +Console.WriteLine(masker.Mask(""" + {"id":7,"order":{"status":"paid","total":9.5,"note":"leave at door"},"customer":{"name":"Alice","country":"NL"}} + """)); +// Output: +// {"id":7,"order":{"status":"paid","total":9.5,"note":"***"},"customer":{"name":"***","country":"NL"}} +``` + +## Relative rules and name matchers + +`Relative(rules, defaultPolicy)` is a policy whose rules match the **end** of a property path at any depth. A plain string is an exact, case-insensitive name; `PropMatches` tests names differently. The first matching rule wins. + +```csharp +using System.Text.RegularExpressions; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(Relative(rules => rules + .Match(PropMatches.EndsWith("token")).MaskAny("***") + .Match(PropMatches.Contains("email")).MaskAny("***") + .Match(PropMatches.OneOf("pwd", "passwd", "password")).MaskAny("***") + .Match(PropMatches.Regex(new Regex("^x-api-", RegexOptions.IgnoreCase))).MaskAny("***") + .Match("card", "cvv").MaskAny("***"), + BlockList)); + +Console.WriteLine(masker.Mask(""" + {"auth":{"AccessToken":"a1","refresh_token":"r2"},"user":{"WorkEmail":"a@b.c","PWD":"p"},"headers":{"X-Api-Key":"k"},"payment":{"card":{"cvv":123,"brand":"visa"}}} + """)); +// Output: +// {"auth":{"AccessToken":"***","refresh_token":"***"},"user":{"WorkEmail":"***","PWD":"***"},"headers":{"X-Api-Key":"***"},"payment":{"card":{"cvv":"***","brand":"visa"}}} +``` + +Absolute and relative rules combine: absolute rules first, then a `Relative(...)` policy for everything they do not name. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj( + root => root.Match("debug").MaskAny("[removed]"), + Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + +Console.WriteLine(masker.Mask("""{"debug":{"trace":"…"},"login":{"user":"bob","password":"p"}}""")); +// Output: +// {"debug":"[removed]","login":{"user":"bob","password":"***"}} +``` + +## Rule kinds + +Every `Mask*` rule masks the whole value, whatever its JSON type; an object or array under a mask rule is skipped unread. + +| Rule | The function receives | A `null` value | +| --- | --- | --- | +| `MaskAny("***")` / `MaskAny((value, ctx) => …)` | a string decoded; a number or boolean as its literal (`"12.50"`, `"true"`); `null` for an object or array | stays `null`; the function is not called | +| `MaskStr(...)` | the same as `MaskAny` | may reach the function as `null` | +| `MaskRawValue(...)` | the same, but a string as raw JSON text, escapes kept | may reach the function as `null` | +| `MaskInt` / `MaskLong` / `MaskDecimal` | the number when it fits, otherwise `null` | may reach the function as `null` | +| `MaskBool` | `true`/`false`, otherwise `null` | may reach the function as `null` | +| `MaskAny(MaskTag)` | — written by the tag strategy | stays `null` | +| `Unmasked()` | — a string, number, boolean or `null` written unchanged | stays `null` | + +A strategy is a constant string, a `Regex` whose matches become `*`, or a function; a function returning `null` writes `null`. Write functions so that a `null` input returns `null` (or a constant): whether a JSON `null` reaches a `MaskStr`-family function differs between absolute and relative rules. + +```csharp +using System.Text.RegularExpressions; +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(Relative(rules => rules + .Match("pin").MaskAny("***") + .Match("phone").MaskStr(new Regex("[0-9](?=[0-9]{2})")) + .Match("amount").MaskAny((value, _) => value is null ? null : $"<{value.Length} chars>") + .Match("age").MaskInt((age, _) => age >= 18 ? "adult" : "minor") + .Match("address").MaskAny("***") + .Match("note").MaskAny("***"), + BlockList)); + +Console.WriteLine(masker.Mask(""" + {"pin":1234,"phone":"+31612345678","amount":12.50,"age":41,"address":{"street":"Main 1","city":"Delft"},"note":null} + """)); +// Output: +// {"pin":"***","phone":"+*********78","amount":"<5 chars>","age":"adult","address":"***","note":null} +``` + +## Arrays and the root type + +`JsonObserver.Obj(...)` expects a root object and `JsonObserver.Array(...)` a root array; the other root is `Invalid`. `JsonObserver.Any(obj, array, policy)` accepts both. In an array builder every rule applies to every item; `Obj(...)` handles the items that are objects. + +**An array item is one level of a property path.** A multi-name `Match` crosses one level per name, so `Match("lines", "sku")` never reaches `{"lines":[{"sku":…}]}`; put a match-anything test where the item is: `Match("lines", AnyItem, "sku")` with `AnyItem = new PropMatchingStrategy(_ => true)`. Use that form for **objects inside a nested array**: in 2.0.0, rules of an `Obj(...)` placed inside a property's `Array(...)` do not match (see [pitfalls.md](./pitfalls.md#rules-inside-a-nested-array-do-not-match)). + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var anyItem = new PropMatchingStrategy(_ => true); + +var rootArray = JsonObserver.Array(items => items.Obj(item => item.Match("sku").Unmasked())); + +var scalarItems = JsonObserver.Obj(root => root.Match("tags").Array(tags => tags.Unmasked())); + +var objectItems = JsonObserver.Obj(root => root + .Match("lines", anyItem, "sku").Unmasked() + .Match("lines", anyItem, "qty").Unmasked()); + +var either = JsonObserver.Any(_ => { }, _ => { }, Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + +Console.WriteLine(rootArray.Mask("""[{"sku":"A1","price":3},{"sku":"B2","price":4}]""")); +Console.WriteLine(scalarItems.Mask("""{"tags":["vip",3],"customer":"Alice"}""")); +Console.WriteLine(objectItems.Mask("""{"lines":[{"sku":"A1","qty":2,"price":3}],"customer":"Alice"}""")); +Console.WriteLine(either.Mask("""[{"password":"p"},{"user":"u"}]""")); +Console.WriteLine(either.Mask("""{"password":"p"}""")); + +JsonObserver.Obj(BlockList).Mask("[1,2]", out var result); +Console.WriteLine(result.Status); +// Output: +// [{"sku":"A1","price":"***"},{"sku":"B2","price":"***"}] +// {"tags":["vip",3],"customer":"***"} +// {"lines":[{"sku":"A1","qty":2,"price":"***"}],"customer":"***"} +// [{"password":"***"},{"user":"u"}] +// {"password":"***"} +// Invalid +``` + +## Tags + +`MaskAny(MaskTag.X)` masks with the call's `Utf8MaskStrategy` (the built-in one unless `JsonObserverOptions.MaskStrategy` sets another). `Hash` is an HMAC-SHA256 keyed by `JsonObserverOptions.HashKey`; with no key, a random key is used for the lifetime of the process. + +```csharp +using System.Text; +using System.Text.RegularExpressions; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(Relative(rules => rules + .Match("full").MaskAny(MaskTag.Full) + .Match("card").MaskAny(MaskTag.Last4) + .Match("short").MaskAny(MaskTag.Last4) + .Match("email").MaskAny(MaskTag.Hash) + .Match("ssn").MaskAny(MaskTag.Omit), + BlockList)); + +var options = new JsonObserverOptions(HashKey: Encoding.UTF8.GetBytes("a key shared by every instance")); +var masked = masker.Mask("""{"full":true,"card":"4111111111111111","short":"1234567","email":"a@b.c","ssn":"123-45-6789"}""", options)!; +var hash = Regex.Match(masked, "hash:[0-9a-f]{16}").Value; + +Console.WriteLine(masked.Replace(hash, "hash:…")); +Console.WriteLine(masker.Mask("""{"email":"a@b.c"}""", options) == $$"""{"email":"{{hash}}"}"""); +// Output: +// {"full":"***","card":"***1111","short":"***","email":"hash:…","ssn":null} +// True +``` + +## Allow-list from a type + +`JsonShape.FromTypeInfo(typeInfo, classify)` builds the expected structure from System.Text.Json metadata: names after the naming policy and `[JsonPropertyName]`, lists, dictionaries and recursive types. `classify` returns a `MaskTag` for a sensitive property and `null` for one shown as is. `JsonObserver.FromShape(shape, options)` then writes known values as they are, masks the classified ones, and handles unknown members by `JsonShapeOptions.Unknown`: + +| `UnknownMemberPolicy` | An unknown member | +| --- | --- | +| `MaskWhole` (default) | is written as `"***"`, whatever its type | +| `Descend` | objects and arrays are walked, names stay visible, every value is `"***"` | +| `PassThrough` | is written as is; only classified members are masked | + +```csharp +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; + +var jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) { TypeInfoResolver = new DefaultJsonTypeInfoResolver() }; +var shape = JsonShape.FromTypeInfo( + jsonOptions.GetTypeInfo(typeof(Customer)), + property => property.AttributeProvider?.IsDefined(typeof(SensitiveAttribute), inherit: true) == true ? MaskTag.Last4 : null); + +var maskWhole = JsonObserver.FromShape(shape); +var descend = JsonObserver.FromShape(shape, new JsonShapeOptions(UnknownMemberPolicy.Descend)); + +const string json = """{"name":"Alice","card_no":"4111111111111111","tags":["vip"],"extra":{"risk":"high"}}"""; +Console.WriteLine(maskWhole.Mask(json)); +Console.WriteLine(descend.Mask(json)); +// Output: +// {"name":"Alice","card_no":"***1111","tags":["vip"],"extra":"***"} +// {"name":"Alice","card_no":"***1111","tags":["vip"],"extra":{"risk":"***"}} + +[AttributeUsage(AttributeTargets.Property)] +sealed class SensitiveAttribute : Attribute; + +sealed class Customer +{ + public string? Name { get; set; } + + [Sensitive] + [JsonPropertyName("card_no")] + public string? CardNumber { get; set; } + + public List? Tags { get; set; } +} +``` + +On .NET 8, metadata from a source-generated `JsonSerializerContext` carries no attributes; classify by `property.Name` there. A shape can also be written by hand, and is frozen once an observer is built from it: + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; + +var shape = JsonShape.Object( + ("id", JsonShape.Scalar), + ("token", JsonShape.Masked(MaskTag.Full)), + ("items", JsonShape.Array(JsonShape.Object(("sku", JsonShape.Scalar)))), + ("labels", JsonShape.Map(JsonShape.Scalar))); + +Console.WriteLine(JsonObserver.FromShape(shape).Mask(""" + {"id":1,"token":"t","items":[{"sku":"A","price":2}],"labels":{"env":"prod"},"other":true} + """)); +// Output: +// {"id":1,"token":"***","items":[{"sku":"A","price":"***"}],"labels":{"env":"prod"},"other":"***"} +``` + +## Extract values while masking + +`JsonObserver.Obj(...)` adds `Read*` rules that hand a value to a context object and write it unchanged. The context-aware policies live in `JsonObserverValuePolicies`. `Mask(json, context)` masks and extracts in one pass; `Read(json, context)` only extracts and returns a `MaskResult`. + +```csharp +using DragoAnt.System.Text.Json.Observer; + +var observer = JsonObserver.Obj( + root => root + .Match("orderId").ReadLong((id, info) => info.OrderId = id) + .Match("total").ReadDecimal((total, info) => info.Total = total) + .Match("customer", "email").MaskAny("***"), + JsonObserverValuePolicies.BlockList); + +const string json = """{"orderId":1001,"total":19.90,"customer":{"email":"a@b.c","tier":"gold"}}"""; + +var info = new OrderInfo(); +Console.WriteLine(observer.Mask(json, info)); +Console.WriteLine(FormattableString.Invariant($"{info.OrderId} {info.Total}")); + +var readOnly = new OrderInfo(); +var result = observer.Read(json, readOnly); +Console.WriteLine($"{result.Status} {readOnly.OrderId}"); +// Output: +// {"orderId":1001,"total":19.90,"customer":{"email":"***","tier":"gold"}} +// 1001 19.90 +// Masked 1001 + +sealed class OrderInfo +{ + public long? OrderId { get; set; } + public decimal? Total { get; set; } +} +``` + +A number that does not fit the read type (a fraction for `ReadInt`, a 30-digit integer) reaches the callback as `null`; the token is still written unchanged. diff --git a/skills/json-observer-masking/migrating-from-1x.md b/skills/json-observer-masking/migrating-from-1x.md new file mode 100644 index 0000000..4e83c20 --- /dev/null +++ b/skills/json-observer-masking/migrating-from-1x.md @@ -0,0 +1,106 @@ +# Migrating from 1.x to 2.0 — json-observer-masking + +2.0 keeps the builder API (`JsonObserver.Obj/Array/Any`, `Match`, `Relative`, `Mask*`, `Read*`) and changes what the defaults produce. Work through this list; the "before" blocks are 1.x code and do not compile against 2.0. + +## 1. The default policy masks booleans and uses one token + +`AllowList` (still the default) now writes every string, number **and boolean** as `"***"`. 1.x wrote `"#str#*****"` / `"#number#*****"` and kept booleans; that output survives as the obsolete `LegacyAllowList`. Update golden strings in tests, and log parsers that looked for `#str#`. + +```csharp +using DragoAnt.System.Text.Json.Observer; + +Console.WriteLine(JsonObserver.Obj(root => root.Match("id").Unmasked()).Mask("""{"id":1,"name":"x","vip":true}""")); +// Output: +// {"id":1,"name":"***","vip":"***"} +``` + +## 2. `Mask(string)` never throws, and its options moved + +1.x threw on invalid JSON and took `JsonReaderOptions`, `JsonWriterOptions`, `ignoreNulls` and `ignoreComments`. 2.0 never throws, always skips comments, accepts trailing commas, writes non-ASCII unescaped, and takes one `JsonObserverOptions`. Drop the `try/catch` around `Mask`, and read `MaskResult` when you need to know what happened. + + +```csharp +// 1.x +try +{ + var masked = observer.Mask(json, ignoreNulls: true, writerOptions: new JsonWriterOptions { Indented = true }); +} +catch (JsonException) +{ + // invalid body +} +``` + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var observer = JsonObserver.Obj(BlockList); +var masked = observer.Mask("""{"a":1,"b":null,}""", out var result, new JsonObserverOptions(IgnoreNulls: true)); +Console.WriteLine($"{result.Status} {masked}"); +// Output: +// Masked {"a":1} +``` + +## 3. `Read(...)` returns a `MaskResult` + +`Read` used to return `void` and throw; it now returns `MaskResult` and never throws. `Read(byte[])` became `Read(ReadOnlySpan)` (a `byte[]` converts implicitly). + +```csharp +using System.Text; +using DragoAnt.System.Text.Json.Observer; + +var observer = JsonObserver.Obj(root => root.Match("id").ReadInt((id, h) => h.Id = id)); +var holder = new Holder(); +MaskResult result = observer.Read(Encoding.UTF8.GetBytes("""{"id":5}"""), holder); +Console.WriteLine($"{result.Status} {holder.Id}"); +// Output: +// Masked 5 + +sealed class Holder +{ + public int? Id { get; set; } +} +``` + +## 4. Every `Mask*` rule masks the whole value, whatever its type + +`MaskStr`, `MaskRawValue`, `MaskInt`, `MaskLong`, `MaskDecimal` and `MaskBool` used to hand a value of another type to the default policy (so under `BlockList` a numeric `cvv` under `MaskStr` stayed visible) and descended into objects. Now they mask any value and skip containers unread; `MaskStr` receives a number or boolean as its literal. Because they also match containers, a mask rule placed before an `Obj(...)`/`Array(...)` rule for the same name now wins over it — reorder such rules. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var observer = JsonObserver.Obj(Relative(rules => rules.Match("cvv").MaskStr("***").Match("address").MaskStr("***"), BlockList)); +Console.WriteLine(observer.Mask("""{"cvv":123,"address":{"street":"Main 1"}}""")); +// Output: +// {"cvv":"***","address":"***"} +``` + +## 5. A string cut by `MaxValueBytes` reports `Truncated` + +It used to report success. `FailedAtByte` is -1 in that case (the whole document was read). Masking functions now receive such a value cut to `MaxValueBytes`. + +## 6. Number read rules no longer fail the body + +`ReadInt`, `ReadLong` and `ReadDecimal` receive `null` for a number that does not fit, and the token is written unchanged. + +## 7. `PropertyPath` is a `ref struct` + +Custom `MaskValue` rules that keep a `PropertyPath` beyond the call, construct one, or use `MaxLength`/`Dispose` must change: only `GetPropertyName`, `GetPropertyNameReverse`, `Length` and `ToString` remain. Rebuild custom rules against 2.0. + +## 8. `JsonWriter` is sealed to the library + +It cannot be derived from outside; `JsonWriter.FromUtf8JsonWriter`, `JsonWriter.Empty` and `WriteCommentValue` are gone. + +## 9. Internal types + +`JsonObserverException`, `PropertyPathMatch`, `JsonPropertyMatchDelegate`, `JsonPropertyPathMatchDelegate` and the builder constructors are internal. Start rules with `Match(...)` on the builder you are given. + +## 10. A UTF-8 byte order mark is skipped + +## New in 2.0, worth adopting while you migrate + +- The UTF-8 API with a reused `IBufferWriter` ([recipes.md](./recipes.md#hot-path-utf-8-api)). +- `MaskAny(MaskTag.Last4/Hash/Omit)` instead of hand-written masking functions ([examples.md](./examples.md#tags)). +- `JsonShape.FromTypeInfo` + `JsonObserver.FromShape` for structure-aware allow-lists ([examples.md](./examples.md#allow-list-from-a-type)). diff --git a/skills/json-observer-masking/pitfalls.md b/skills/json-observer-masking/pitfalls.md new file mode 100644 index 0000000..d0d1753 --- /dev/null +++ b/skills/json-observer-masking/pitfalls.md @@ -0,0 +1,144 @@ +# Pitfalls — json-observer-masking + +Symptom first, then the cause and the fix. Each block is a complete program. + +## Everything became `"***"` + +**Cause:** the default policy is `AllowList`. `JsonObserver.Obj(rules)` without a second argument, and `Relative(rules)` without its second argument, mask every string, number and boolean no rule names. + +**Fix:** pass `BlockList` when only the named values are sensitive. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +const string json = """{"user":"alice","password":"p","active":true}"""; + +Console.WriteLine(JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("x"))).Mask(json)); +Console.WriteLine(JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("x"), BlockList)).Mask(json)); +// Output: +// {"user":"***","password":"x","active":"***"} +// {"user":"alice","password":"x","active":true} +``` + +## The secret is still visible + +Check, in order: + +1. **The rule is absolute, the field is nested.** `JsonObserver.Obj(root => root.Match("password")…)` only matches a top-level `password`. Use `Relative(...)` to match at any depth. +2. **The name differs.** Matching is exact (case-insensitive): `password` does not match `newPassword` or `passwd`. Use `PropMatches.Contains("password")` or `PropMatches.OneOf(...)`. +3. **The path crosses an array.** An array item is a path level: `Match("users", "password")` does not reach `{"users":[{"password":…}]}`. Use a single name in `Relative(...)`, or `Match("users", AnyItem, "password")` with `AnyItem = new PropMatchingStrategy(_ => true)`. +4. **The value is not a string and the rule only reads.** `Read*` rules write the value unchanged; use a `Mask*` rule. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +const string json = """{"login":{"newPassword":"p1"},"users":[{"password":"p2"}]}"""; + +var tooNarrow = JsonObserver.Obj(root => root.Match("password").MaskAny("***"), BlockList); +var fixedRules = JsonObserver.Obj(Relative(rules => rules.Match(PropMatches.Contains("password")).MaskAny("***"), BlockList)); + +Console.WriteLine(tooNarrow.Mask(json)); +Console.WriteLine(fixedRules.Mask(json)); +// Output: +// {"login":{"newPassword":"p1"},"users":[{"password":"p2"}]} +// {"login":{"newPassword":"***"},"users":[{"password":"***"}]} +``` + +## Rules inside a nested array do not match + +**Known issue in 2.0.0:** the rules of an `Obj(...)` placed inside a property's `Array(...)` — `root.Match("lines").Array(l => l.Obj(line => line.Match("qty")…))` — never match. Under `BlockList` the value stays in clear; under `AllowList` everything in the item is masked. An `Obj(...)` directly under a root `JsonObserver.Array(...)` works. + +**Fix:** address the items with a path that names the item level, or with relative rules. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var anyItem = new PropMatchingStrategy(_ => true); +const string json = """{"lines":[{"sku":"A1","qty":2}]}"""; + +var broken = JsonObserver.Obj(root => root.Match("lines").Array(lines => lines.Obj(line => line.Match("qty").MaskAny("***"))), BlockList); +var byPath = JsonObserver.Obj(root => root.Match("lines", anyItem, "qty").MaskAny("***"), BlockList); +var byRelative = JsonObserver.Obj(Relative(rules => rules.Match("lines", anyItem, "qty").MaskAny("***"), BlockList)); + +Console.WriteLine(broken.Mask(json)); +Console.WriteLine(byPath.Mask(json)); +Console.WriteLine(byRelative.Mask(json)); +// Output: +// {"lines":[{"sku":"A1","qty":2}]} +// {"lines":[{"sku":"A1","qty":"***"}]} +// {"lines":[{"sku":"A1","qty":"***"}]} +``` + +## A whole object was replaced by `"***"` + +**Cause:** every `Mask*` rule matches containers too and masks them whole. A mask rule written **before** an `Obj(...)`/`Array(...)` rule for the same name wins, because the first matching rule wins. + +**Fix:** put the `Obj(...)` rule first, or use a more specific name. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +const string json = """{"card":{"brand":"visa","number":"4111"}}"""; + +var maskFirst = JsonObserver.Obj(root => root + .Match("card").MaskAny("***") + .Match("card").Obj(card => card.Match("number").MaskAny("***")), BlockList); + +var objFirst = JsonObserver.Obj(root => root + .Match("card").Obj(card => card.Match("number").MaskAny("***")) + .Match("card").MaskAny("***"), BlockList); + +Console.WriteLine(maskFirst.Mask(json)); +Console.WriteLine(objFirst.Mask(json)); +// Output: +// {"card":"***"} +// {"card":{"brand":"visa","number":"***"}} +``` + +## The output is empty + +**Cause:** `MaskStatus.NotJson` — the input is empty, or its root is a string or number. Or `Invalid` with nothing read: a root array given to `JsonObserver.Obj(...)` (or a root object to `JsonObserver.Array(...)`). + +**Fix:** check `MaskResult.Status`; use `JsonObserver.Any(...)` when the root can be an object or an array. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var objOnly = JsonObserver.Obj(BlockList); +var any = JsonObserver.Any(_ => { }, _ => { }, BlockList); + +Console.WriteLine($"[{objOnly.Mask("[1]", out var r1)}] {r1.Status}"); +Console.WriteLine($"[{objOnly.Mask("\"text\"", out var r2)}] {r2.Status}"); +Console.WriteLine($"[{any.Mask("[1]", out var r3)}] {r3.Status}"); +// Output: +// [] Invalid +// [] NotJson +// [[1]] Masked +``` + +## `Hash` values differ between services or restarts + +**Cause:** with no `JsonObserverOptions.HashKey`, the key is random per process. **Fix:** set the same `HashKey` everywhere ([recipes.md](./recipes.md#correlate-masked-values-across-services)). + +## `Last4` shows `"***"` instead of the last digits + +**Cause:** values shorter than 8 characters are masked fully, so a short value is not narrowed down. Booleans, objects and arrays are always `"***"`. + +## A masking function sees a shortened value + +**Cause:** `MaxValueBytes` also cuts the value handed to a function. The result reports `Truncated`. + +## Building an observer per call + +An observer compiles its rules when built and caches path buffers across calls. Building one per message costs far more than masking; keep it in a `static readonly` field or a singleton and share it across threads. + +## Expecting zero allocations + +With constant or tag rules the UTF-8 API allocates a small constant amount per call (a few hundred bytes, whatever the body size). Masking functions receive a `string`, and the string API allocates the input and output strings. Set allocation budgets from measurements, not from "0 B". diff --git a/skills/json-observer-masking/recipes.md b/skills/json-observer-masking/recipes.md new file mode 100644 index 0000000..fd8271a --- /dev/null +++ b/skills/json-observer-masking/recipes.md @@ -0,0 +1,202 @@ +# Recipes — json-observer-masking + +Task-shaped solutions. Each block is a complete program; the `// Output:` comment is what it prints. + +## Log a request body safely + +A body headed for a log is usually size-capped, so it may be cut mid-document. Mask with a `MaxOutputBytes` limit and log the status next to the body — the masked text is valid JSON whatever the status. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var logged = BodyLog.Describe("""{"user":"alice","password":"s3cret","card":{"number":"4111111111111111","cvv":123}}"""); +Console.WriteLine(logged); +// Output: +// Masked {"user":"alice","password":"***","card":{"number":"***1111","cvv":"***"}} + +static class BodyLog +{ + private static readonly JsonObserver Masker = JsonObserver.Obj(Relative(rules => rules + .Match(PropMatches.OneOf("password", "secret", "cvv")).MaskAny(MaskTag.Full) + .Match("card", "number").MaskAny(MaskTag.Last4) + .Match(PropMatches.EndsWith("token")).MaskAny(MaskTag.Full), + BlockList)); + + private static readonly JsonObserverOptions Options = new(MaxOutputBytes: 4096, MaxValueBytes: 512); + + public static string Describe(string body) + { + var masked = Masker.Mask(body, out var result, Options); + return $"{result.Status} {masked}"; + } +} +``` + +## Hot path: UTF-8 API + +`Mask(ReadOnlySpan, IBufferWriter, JsonObserverOptions?)` reads bytes and writes bytes, with no string conversion either way. Reuse the output buffer (`ResetWrittenCount()`), and keep it per thread — an `ArrayBufferWriter` is not thread-safe, the observer is. + +```csharp +using System.Buffers; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var first = Utf8Masking.Mask("""{"user":"alice","password":"p1"}"""u8); +var second = Utf8Masking.Mask("""{"user":"bob","password":"p2"}"""u8); +Console.WriteLine(first); +Console.WriteLine(second); +// Output: +// Masked {"user":"alice","password":"***"} +// Masked {"user":"bob","password":"***"} + +static class Utf8Masking +{ + private static readonly JsonObserver Masker = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + + [ThreadStatic] + private static ArrayBufferWriter? _output; + + public static string Mask(ReadOnlySpan utf8) + { + var output = _output ??= new ArrayBufferWriter(4096); + output.ResetWrittenCount(); + var result = Masker.Mask(utf8, output); + return $"{result.Status} {Encoding.UTF8.GetString(output.WrittenSpan)}"; + } +} +``` + +In a real hot path, write `output.WrittenSpan` straight to the log sink instead of decoding it to a string. With constant or tag rules, each call allocates a small constant amount (a few hundred bytes, whatever the body size); a rule with a masking function allocates the `string` it receives. Measure with `GC.GetAllocatedBytesForCurrentThread()` rather than assuming. + +## Cut-off or invalid bodies + +Both APIs never throw. Branch on `MaskResult.Status`: + +| `MaskStatus` | Meaning | Output | +| --- | --- | --- | +| `Masked` | the whole payload was read and masked | the masked JSON | +| `Truncated` | the payload ended inside the document, or the output reached `MaxOutputBytes` (`FailedAtByte` = where reading stopped), or a string was cut to `MaxValueBytes` (`FailedAtByte` = -1) | valid JSON: the masked part, open objects and arrays closed | +| `Invalid` | not valid JSON (including plain text), deeper than `MaxDepth`, the root type the observer does not accept, or a rule threw | valid JSON: the masked part read before the failure (may be empty) | +| `NotJson` | empty input, or valid JSON whose root is not an object or array (`"text"`, `42`) | empty string, `BytesWritten` 0 | + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + +foreach (var body in new[] + { + """{"user":"alice","password":"s3cret"}""", + """{"user":{"login":"alice","password":"s3cret","roles":["admin","dev""", + """{"user":"alice",,}""", + "42", + "not json", + }) +{ + var masked = masker.Mask(body, out var result); + Console.WriteLine($"{result.Status,-9} at={result.FailedAtByte,3} {masked}".TrimEnd()); +} +// Output: +// Masked at= -1 {"user":"alice","password":"***"} +// Truncated at= 61 {"user":{"login":"alice","password":"***","roles":["admin"]}} +// Invalid at= 16 {"user":"alice"} +// NotJson at= 0 +// Invalid at= 0 +``` + +A value that is cut never leaks: a string being written when the input ends is dropped, not written in part. + +## Limits and output settings + +`JsonObserverOptions` applies to both APIs. Build it once and reuse it. + +| Option | Default | Effect | +| --- | --- | --- | +| `MaxOutputBytes` | unlimited | output cap in UTF-8 bytes; reaching it closes the output → `Truncated` | +| `MaxValueBytes` | unlimited | longest string written; a longer one is cut and ends with `…` → `Truncated`; masking functions also receive the cut value | +| `MaxDepth` | 64 | deeper nesting → `Invalid` | +| `RelaxedEscaping` | `true` | non-ASCII and HTML characters written unescaped | +| `HashKey` | random per process | key of `MaskTag.Hash` | +| `MaskStrategy` | built-in | writes `MaskTag` rules | +| `IgnoreNulls` | `false` | drops `null` properties and items, and objects and arrays left empty by that | +| `Indented` | `false` | indented output | + +```csharp +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(BlockList); +const string json = """{"note":"a very long free-text note","empty":null,"tags":[null],"city":"Zürich"}"""; + +Console.WriteLine(masker.Mask(json, new JsonObserverOptions(MaxValueBytes: 10, IgnoreNulls: true))); +Console.WriteLine(masker.Mask(json, new JsonObserverOptions(RelaxedEscaping: false))); +// Output: +// {"note":"a very lon…","city":"Zürich"} +// {"note":"a very long free-text note","empty":null,"tags":[null],"city":"Z\u00FCrich"} +``` + +## Correlate masked values across services + +`MaskTag.Hash` writes `hash:` and 16 hex characters of an HMAC-SHA256. Give every instance the same `HashKey` (from configuration or a secret store, never from source) and the same value hashes the same everywhere, so you can follow one customer through logs without seeing their email. Without a key, hashes only correlate inside one process. + +```csharp +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(Relative(rules => rules.Match("email").MaskAny(MaskTag.Hash), BlockList)); +var keyFromConfiguration = Encoding.UTF8.GetBytes("load-me-from-configuration"); + +var serviceA = new JsonObserverOptions(HashKey: keyFromConfiguration); +var serviceB = new JsonObserverOptions(HashKey: keyFromConfiguration.ToArray()); + +var a = masker.Mask("""{"email":"alice@example.com"}""", serviceA); +var b = masker.Mask("""{"email":"alice@example.com"}""", serviceB); +Console.WriteLine(a == b); +Console.WriteLine(a!.Contains("alice", StringComparison.Ordinal)); +// Output: +// True +// False +``` + +## Custom mask strategy + +`MaskTag` rules are written by a `Utf8MaskStrategy`. Replace it per call with `JsonObserverOptions.MaskStrategy` — one strategy serves every tag; delegate the kinds you do not change to `Utf8MaskStrategy.Default`. `value` is the unescaped string, the literal of a number or boolean, or empty for an object or array; write exactly one value. + +```csharp +using System.Text.Json; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(Relative(rules => rules + .Match("password").MaskAny(MaskTag.Full) + .Match("card").MaskAny(MaskTag.Last4), + BlockList)); + +var options = new JsonObserverOptions(MaskStrategy: new RedactedStrategy()); +Console.WriteLine(masker.Mask("""{"password":"p","card":"4111111111111111"}""", options)); +// Output: +// {"password":"[redacted]","card":"***1111"} + +sealed class RedactedStrategy : Utf8MaskStrategy +{ + public override void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) + { + if (tag.Kind == MaskKind.Full) + { + writer.WriteStringValue("[redacted]"u8); + return; + } + + Default.Mask(value, tokenType, tag, writer, options); + } +} +``` + +A strategy that throws turns the result into `Invalid` with the masked prefix before that value — it never writes the value in clear. From dfc6a4eda670277c901f8e96b01691f0cb9fd235 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:02:22 +0200 Subject: [PATCH 2/4] Add the json-observer-http-logging skill Registration, masker providers, sinks, options, body markers and the asynchronous response entry, each with a runnable example. --- skills/json-observer-http-logging/SKILL.md | 118 ++++++++ skills/json-observer-http-logging/examples.md | 250 ++++++++++++++++ skills/json-observer-http-logging/pitfalls.md | 160 ++++++++++ skills/json-observer-http-logging/recipes.md | 281 ++++++++++++++++++ 4 files changed, 809 insertions(+) create mode 100644 skills/json-observer-http-logging/SKILL.md create mode 100644 skills/json-observer-http-logging/examples.md create mode 100644 skills/json-observer-http-logging/pitfalls.md create mode 100644 skills/json-observer-http-logging/recipes.md diff --git a/skills/json-observer-http-logging/SKILL.md b/skills/json-observer-http-logging/SKILL.md new file mode 100644 index 0000000..d338d79 --- /dev/null +++ b/skills/json-observer-http-logging/SKILL.md @@ -0,0 +1,118 @@ +--- +name: json-observer-http-logging +description: Log HttpClient request and response JSON bodies with secrets masked, using DragoAnt.System.Text.Json.Observer.Http — AddJsonBodyLogging on IHttpClientBuilder, JsonBodyLoggingHandler (a DelegatingHandler), per-request model types with WithBodyLogging or SendWithBodyLoggingAsync, When (Never, OnFailure, Always), MaxBodyBytes, IncludeSensitive, IJsonBodyMaskerProvider to pick a JsonObserver per body model, IJsonBodyLogSink for custom sinks, JsonBodyLogEntry fields, JsonBodyStatus and JsonBodyOutcome (Success, Failure, Exception, Canceled). Use when adding body logging to a named or typed HttpClient, deciding what is logged on failure, wiring maskers per request and response model, sending entries to your own sink, or debugging "[body withheld]", "[body not buffered]", "[body not logged: …]", "[body not JSON]", every logged value being "***", or a log entry that appears late or not at all. +--- + +# Logging HttpClient JSON bodies with DragoAnt.System.Text.Json.Observer.Http + +`JsonBodyLoggingHandler` sits in an `HttpClient` pipeline and turns a call into one `JsonBodyLogEntry` — method, path, status, outcome, elapsed time, and the request and response bodies **masked** by a `JsonObserver` you choose per body model type. The caller is never affected: the response streams as usual, its body stays readable in full, and a failure inside logging never reaches the caller. + +Package: `DragoAnt.System.Text.Json.Observer.Http` (net8.0, net9.0, net10.0), namespace `DragoAnt.System.Text.Json.Observer.Http`. Masking rules come from `DragoAnt.System.Text.Json.Observer` — see the `json-observer-masking` skill. + +## Quick start + +```csharp +using System.Net; +using System.Net.Http.Json; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using DragoAnt.System.Text.Json.Observer.Strategies; +using Microsoft.Extensions.DependencyInjection; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var sink = new ConsoleSink(); +var services = new ServiceCollection(); +services.AddSingleton(); +services.AddSingleton(sink); +services.AddHttpClient("payments", client => client.BaseAddress = new Uri("https://api.example.com/")) + .AddJsonBodyLogging(options => options.When = JsonBodyLogWhen.OnFailure) + .ConfigurePrimaryHttpMessageHandler(() => new DeclineEverything()); + +await using var provider = services.BuildServiceProvider(); +var client = provider.GetRequiredService().CreateClient("payments"); + +using var request = new HttpRequestMessage(HttpMethod.Post, "charges?apiKey=k") +{ + Content = JsonContent.Create(new ChargeRequest("4111111111111111", 10.5m)), +}.WithBodyLogging("Charge"); +using var response = await client.SendAsync(request); + +await sink.Written.Task.WaitAsync(TimeSpan.FromSeconds(5)); +// Output: +// Charge POST /charges 402 Failure +// request (Masked): {"cardNumber":"***1111","amount":10.5} +// response (Masked): {"id":"ch_1","error":"card_declined","cardNumber":"***1111"} + +public sealed record ChargeRequest(string CardNumber, decimal Amount); + +public sealed record ChargeResponse(string Id, string? Error); + +public sealed class PaymentMaskers : IJsonBodyMaskerProvider +{ + private static readonly JsonObserver Charge = JsonObserver.Obj(Relative(rules => rules + .Match("cardNumber").MaskAny(MaskTag.Last4), + BlockList)); + + public JsonObserver? GetMasker(Type? modelType, string clientName) => + modelType == typeof(ChargeRequest) || modelType == typeof(ChargeResponse) ? Charge : null; +} + +public sealed class ConsoleSink : IJsonBodyLogSink +{ + public TaskCompletionSource Written { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Write(in JsonBodyLogEntry entry) + { + Console.WriteLine($"{entry.Operation} {entry.Method} {entry.Path} {entry.StatusCode} {entry.Outcome}"); + Console.WriteLine($"request ({entry.RequestBodyStatus}): {entry.RequestBody}"); + Console.WriteLine($"response ({entry.ResponseBodyStatus}): {entry.ResponseBody}"); + Written.TrySetResult(); + } +} + +public sealed class DeclineEverything : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.PaymentRequired) + { + Content = new StringContent("""{"id":"ch_1","error":"card_declined","cardNumber":"4111111111111111"}""", Encoding.UTF8, "application/json"), + }); +} +``` + +In an application you register only the provider (and optionally a sink); the default sink writes to `ILogger`. The fake primary handler and the console sink above only make the example self-contained. + +## Decision path + +1. **Register** with `services.AddHttpClient(name).AddJsonBodyLogging(o => …)` — never `AddHttpMessageHandler()`. Building a pipeline by hand: `new JsonBodyLoggingHandler(options, provider, sink, logger) { InnerHandler = … }` → [examples.md#manual-handler-chain](./examples.md#manual-handler-chain). +2. **Choose what is logged** with `When`: `OnFailure` (default — a non-2xx status, an exception, or a cancellation), `Always`, or `Never`. +3. **Register one `IJsonBodyMaskerProvider`** that maps body model types to observers built once. Without a provider every value is masked (`"***"`); a provider returning `null` withholds the body → [recipes.md#a-masker-provider-built-from-your-models](./recipes.md#a-masker-provider-built-from-your-models). +4. **Tell the handler the model types** per request: `request.WithBodyLogging("Operation")` or `client.SendWithBodyLoggingAsync(request, "Operation")`; or set `DefaultRequestType` / `DefaultResponseType` for a client that always sends one model. +5. **Need entries elsewhere** (metrics, a structured store, a test)? Register an `IJsonBodyLogSink` → [recipes.md#a-custom-sink](./recipes.md#a-custom-sink). + +## Rules that matter + +1. **Options are named per client** and read on every call; `services.ConfigureAll(…)` changes every client. Calling `AddJsonBodyLogging` twice for one client adds the handler once. +2. **Defaults:** `When = OnFailure`, `MaxBodyBytes = 4096` (0 turns bodies off, calls are still logged), `IncludeSensitive = false`. +3. **Cache observers in the provider.** `GetMasker` runs for every logged body, possibly concurrently; return `static readonly` observers, never build one inside it. +4. **The response entry is written later**, on a background read, once `MaxBodyBytes` were captured, the body ended, the read failed, or the caller disposed the response. A sink must be thread-safe and must not assume it runs before `SendAsync` returns. +5. **A body that cannot be logged as JSON becomes a marker** and `RequestBodyStatus`/`ResponseBodyStatus` says why: `[body not logged: text/plain]` (`Skipped`), `[body not buffered]` (`NotBuffered`), `[body withheld]` (`Withheld`), `[body not JSON]` (`NotJson`), `[invalid JSON]` (`Invalid`), `[body incomplete]` (`Incomplete`), `[body not logged: masking failed]` (`Failed`). +6. **Cancellation and `HttpClient.Timeout` are both `Canceled`**; the handler cannot tell them apart. The original exception is rethrown unchanged. +7. **The query string is never logged** — `Path` is the path only. +8. **`IncludeSensitive = true` logs bodies unmasked** (`BodyUnmasked = true`, `Raw` status, a warning in the log). Local debugging only. + +## Pitfalls (details in [pitfalls.md](./pitfalls.md)) + +- Every logged value is `"***"` → no `IJsonBodyMaskerProvider` is registered, or its observer uses the default `AllowList`. +- `[body withheld]` → the provider returned `null`: the request has no `WithBodyLogging<,>` and no `Default*Type`, so `modelType` is `null`. +- Nothing is logged for a successful call → `When` is `OnFailure` (the default). +- A test reads the sink right after `SendAsync` and finds nothing → the response entry is written asynchronously; wait for it. + +## Companions + +- [examples.md](./examples.md) — DI registration, a typed client, a manual handler chain, options, the entry fields. +- [recipes.md](./recipes.md) — a provider from your models, a custom sink, logging only some clients, large and streaming bodies. +- [pitfalls.md](./pitfalls.md) — the markers and surprises, with their causes. + +Every C# block in these files is a complete program (top-level statements, implicit usings) that compiles against the library and prints what its `// Output:` comment shows. diff --git a/skills/json-observer-http-logging/examples.md b/skills/json-observer-http-logging/examples.md new file mode 100644 index 0000000..4a93f76 --- /dev/null +++ b/skills/json-observer-http-logging/examples.md @@ -0,0 +1,250 @@ +# Examples — json-observer-http-logging + +Each block is a complete program: a console project referencing `DragoAnt.System.Text.Json.Observer.Http` (it brings `Microsoft.Extensions.Http` and the core package). A fake primary handler stands in for the network so the programs run offline; the `// Output:` comment is what they print. + +## Typed client with DI + +`AddJsonBodyLogging` works on any `IHttpClientBuilder`, so typed clients get it too. The handler resolves `IJsonBodyMaskerProvider`, `IJsonBodyLogSink` and `ILogger` from DI when they are registered. `SendWithBodyLoggingAsync` attaches the model types and sends in one call. + +```csharp +using System.Net; +using System.Net.Http.Json; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using Microsoft.Extensions.DependencyInjection; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var sink = new CaptureSink(); +var services = new ServiceCollection(); +services.AddSingleton(); +services.AddSingleton(sink); +services.AddHttpClient(client => client.BaseAddress = new Uri("https://users.example.com/")) + .AddJsonBodyLogging(options => options.When = JsonBodyLogWhen.Always) + .ConfigurePrimaryHttpMessageHandler(() => new FakeApi()); + +await using var provider = services.BuildServiceProvider(); +var users = provider.GetRequiredService(); +Console.WriteLine(await users.CreateAsync(new CreateUser("alice", "s3cret"))); + +var entry = await sink.Entry.Task.WaitAsync(TimeSpan.FromSeconds(5)); +Console.WriteLine($"{entry.ClientName} {entry.Operation} {entry.StatusCode} {entry.Outcome} request={entry.RequestBody} response={entry.ResponseBody}"); +// Output: +// created 42 +// UsersClient CreateUser 201 Success request={"login":"alice","password":"***"} response={"id":42,"apiToken":"***"} + +public sealed record CreateUser(string Login, string Password); + +public sealed record CreatedUser(int Id, string ApiToken); + +public sealed class UsersClient(HttpClient http) +{ + public async Task CreateAsync(CreateUser user) + { + using var request = new HttpRequestMessage(HttpMethod.Post, "users") { Content = JsonContent.Create(user) }; + using var response = await http.SendWithBodyLoggingAsync(request, "CreateUser"); + var created = await response.Content.ReadFromJsonAsync(); + return $"created {created!.Id}"; + } +} + +public sealed class UserMaskers : IJsonBodyMaskerProvider +{ + private static readonly JsonObserver Users = JsonObserver.Obj(Relative(rules => rules + .Match("password").MaskAny("***") + .Match("apiToken").MaskAny("***"), + BlockList)); + + public JsonObserver? GetMasker(Type? modelType, string clientName) => + modelType == typeof(CreateUser) || modelType == typeof(CreatedUser) ? Users : null; +} + +public sealed class CaptureSink : IJsonBodyLogSink +{ + public TaskCompletionSource Entry { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Write(in JsonBodyLogEntry entry) => Entry.TrySetResult(entry); +} + +public sealed class FakeApi : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.Created) + { + Content = new StringContent("""{"id":42,"apiToken":"tok_live_123"}""", Encoding.UTF8, "application/json"), + }); +} +``` + +A typed client's name is the type name (`UsersClient`); that is the name its options and `entry.ClientName` use. + +## Manual handler chain + +Without DI, construct the handler with fixed options and set its `InnerHandler`. The client name is empty in this form. + +```csharp +using System.Net; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var sink = new ListSink(); +var options = new JsonBodyLoggingOptions +{ + When = JsonBodyLogWhen.Always, + MaxBodyBytes = 1024, + DefaultRequestType = typeof(object), + DefaultResponseType = typeof(object), +}; +var handler = new JsonBodyLoggingHandler(options, new OneMasker(), sink) { InnerHandler = new Echo() }; +using var client = new HttpClient(handler) { BaseAddress = new Uri("https://echo.example.com/") }; + +using var response = await client.PostAsync("echo", new StringContent("""{"user":"bob","token":"t-1"}""", Encoding.UTF8, "application/json")); +Console.WriteLine(await response.Content.ReadAsStringAsync()); + +var entry = await sink.WaitAsync(); +Console.WriteLine($"[{entry.ClientName}] {entry.Method} {entry.Path} {entry.RequestBody} -> {entry.ResponseBody}"); +// Output: +// {"user":"bob","token":"t-1"} +// [] POST /echo {"user":"bob","token":"***"} -> {"user":"bob","token":"***"} + +public sealed class OneMasker : IJsonBodyMaskerProvider +{ + private static readonly JsonObserver Masker = JsonObserver.Obj(Relative(rules => rules.Match("token").MaskAny("***"), BlockList)); + + public JsonObserver? GetMasker(Type? modelType, string clientName) => Masker; +} + +public sealed class ListSink : IJsonBodyLogSink +{ + private readonly TaskCompletionSource _first = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Write(in JsonBodyLogEntry entry) => _first.TrySetResult(entry); + + public Task WaitAsync() => _first.Task.WaitAsync(TimeSpan.FromSeconds(5)); +} + +public sealed class Echo : HttpMessageHandler +{ + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + new(HttpStatusCode.OK) + { + Content = new StringContent(await request.Content!.ReadAsStringAsync(cancellationToken), Encoding.UTF8, "application/json"), + }; +} +``` + +The caller still reads the full, unmasked response: only the logged copy is masked. + +## Options + +| Option | Default | Effect | +| --- | --- | --- | +| `When` | `OnFailure` | `Never`, `OnFailure` (non-2xx, exception, cancellation) or `Always` | +| `MaxBodyBytes` | 4096 | bytes of each body read and masked; a longer body is logged truncated (`Truncated = true`); 0 or less logs calls without bodies | +| `IncludeSensitive` | `false` | log bodies unmasked; marks the entry `BodyUnmasked` and logs a warning once | +| `DefaultRequestType` / `DefaultResponseType` | `null` | model types passed to the provider when the request has no `WithBodyLogging` | + +With DI they are named options per client name, read on every call, so an options reload applies to the next call: + +```csharp +using DragoAnt.System.Text.Json.Observer.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +var services = new ServiceCollection(); +services.ConfigureAll(options => options.MaxBodyBytes = 2048); +services.AddHttpClient("orders").AddJsonBodyLogging(options => options.When = JsonBodyLogWhen.Always); +services.AddHttpClient("search").AddJsonBodyLogging(); + +await using var provider = services.BuildServiceProvider(); +var monitor = provider.GetRequiredService>(); +foreach (var name in new[] { "orders", "search" }) +{ + var options = monitor.Get(name); + Console.WriteLine($"{name}: {options.When} {options.MaxBodyBytes}"); +} +// Output: +// orders: Always 2048 +// search: OnFailure 2048 +``` + +## The entry + +| Field | Content | +| --- | --- | +| `ClientName` | client name; empty for a handler built without one | +| `Operation` | the name given to `WithBodyLogging` / `SendWithBodyLoggingAsync`, or `null` | +| `Method`, `Path` | HTTP method; request path without query string and fragment | +| `StatusCode` | response status, or `null` when no response arrived | +| `ElapsedMs` | time until the response headers arrived or the call failed | +| `Outcome` | `Success` (2xx), `Failure` (other status), `Exception`, `Canceled` | +| `RequestBody`, `ResponseBody` | masked JSON, a bracketed marker, or `null` when there is no body | +| `RequestBodyStatus`, `ResponseBodyStatus` | `None`, `Masked`, `Truncated`, `Invalid`, `NotJson`, `Raw`, `Withheld`, `Skipped`, `NotBuffered`, `Incomplete`, `Failed` | +| `Truncated` | a body was longer than `MaxBodyBytes` | +| `BodyUnmasked` | a body was logged with `IncludeSensitive` | +| `Exception` | the exception the call failed with; rethrown to the caller unchanged | + +## Exceptions and cancellations + +Under `OnFailure`, a call that throws is logged without a response, then the exception reaches the caller unchanged. A canceled token and `HttpClient.Timeout` both log `Canceled`. + +```csharp +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var sink = new PrintSink(); +var masker = new FixedMasker(JsonObserver.Obj(Relative(rules => rules.Match("pin").MaskAny("***"), BlockList))); +var handler = new JsonBodyLoggingHandler(new JsonBodyLoggingOptions(), masker, sink) { InnerHandler = new Unreachable() }; +using var client = new HttpClient(handler) { BaseAddress = new Uri("https://bank.example.com/") }; + +try +{ + await client.PostAsync("pin", new StringContent("""{"pin":"1234"}""", Encoding.UTF8, "application/json")); +} +catch (HttpRequestException ex) +{ + Console.WriteLine($"caller sees: {ex.Message}"); +} + +using var canceled = new CancellationTokenSource(); +canceled.Cancel(); +try +{ + await client.GetAsync("balance", canceled.Token); +} +catch (OperationCanceledException) +{ + Console.WriteLine("caller sees: canceled"); +} +// Output: +// Exception status= request={"pin":"***"} error=connection refused +// caller sees: connection refused +// Canceled status= request= error=The operation was canceled. +// caller sees: canceled + +public sealed class FixedMasker(JsonObserver masker) : IJsonBodyMaskerProvider +{ + public JsonObserver? GetMasker(Type? modelType, string clientName) => masker; +} + +public sealed class PrintSink : IJsonBodyLogSink +{ + public void Write(in JsonBodyLogEntry entry) => + Console.WriteLine($"{entry.Outcome} status={entry.StatusCode} request={entry.RequestBody} error={entry.Exception?.Message}"); +} + +public sealed class Unreachable : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + throw new HttpRequestException("connection refused"); + } +} +``` + +An entry without a response is written synchronously, before the exception reaches the caller; an entry with a response is written when the background body read ends. diff --git a/skills/json-observer-http-logging/pitfalls.md b/skills/json-observer-http-logging/pitfalls.md new file mode 100644 index 0000000..69c8084 --- /dev/null +++ b/skills/json-observer-http-logging/pitfalls.md @@ -0,0 +1,160 @@ +# Pitfalls — json-observer-http-logging + +Symptom first, then the cause and the fix. The programs share one shape: a handler with fixed options, a capturing sink, and a fake inner handler. + +## Every logged value is `"***"` + +**Cause:** no `IJsonBodyMaskerProvider` is registered — the handler then masks every value of an object or array body — or the provider's observer uses the default `AllowList` (`JsonObserver.Obj(rules)` without a second argument). + +**Fix:** register a provider; build its observers with `BlockList` (mask what you name) or from a shape (allow-list with known fields visible). + +## `[body withheld]` + +**Cause:** the provider returned `null`. Usually `modelType` is `null` because the request carries no `WithBodyLogging()` and the options set no `DefaultRequestType` / `DefaultResponseType`. + +**Fix:** attach the model types per request, set the defaults for a single-model client, or return a fallback observer for `null`. + +```csharp +using System.Net; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +foreach (var attachTypes in new[] { false, true }) +{ + var sink = new CaptureSink(); + var handler = new JsonBodyLoggingHandler(new JsonBodyLoggingOptions { When = JsonBodyLogWhen.Always }, new LoginMaskers(), sink) + { + InnerHandler = new Ok(), + }; + using var client = new HttpClient(handler); + using var request = new HttpRequestMessage(HttpMethod.Post, "https://auth.example.com/login") + { + Content = new StringContent("""{"user":"bob","password":"p"}""", Encoding.UTF8, "application/json"), + }; + if (attachTypes) + { + request.WithBodyLogging("Login"); + } + + using var response = await client.SendAsync(request); + var entry = await sink.Entry.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Console.WriteLine($"{entry.RequestBodyStatus} {entry.RequestBody}"); +} +// Output: +// Withheld [body withheld] +// Masked {"user":"bob","password":"***"} + +public sealed record Login(string User, string Password); + +public sealed class LoginMaskers : IJsonBodyMaskerProvider +{ + private static readonly JsonObserver Login = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + + public JsonObserver? GetMasker(Type? modelType, string clientName) => modelType == typeof(Login) ? Login : null; +} + +public sealed class CaptureSink : IJsonBodyLogSink +{ + public TaskCompletionSource Entry { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Write(in JsonBodyLogEntry entry) => Entry.TrySetResult(entry); +} + +public sealed class Ok : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.NoContent)); +} +``` + +## `[body not logged: …]`, `[body not buffered]`, `[body not JSON]` + +| Marker | Status | Cause | +| --- | --- | --- | +| `[body not logged: text/plain]` | `Skipped` | the content type is not `application/json`, `text/json` or `*+json` (a missing content type is treated as JSON) | +| `[body not logged: charset utf-16]` | `Skipped` | a charset other than UTF-8 / ASCII | +| `[body not logged: encoding gzip]` | `Skipped` | a `Content-Encoding` other than `identity` | +| `[body not logged: MaxBodyBytes is 0]` | `Skipped` | bodies are switched off | +| `[body not buffered]` | `NotBuffered` | a request `StreamContent` whose stream cannot seek — reading it for the log would consume it | +| `[body not JSON]` | `NotJson` | a JSON body whose root is not an object or array, or an empty one | +| `[invalid JSON]` | `Invalid` | the body is not JSON and nothing could be read before the error | +| `[body incomplete]` | `Incomplete` | the response read failed, or the caller disposed the response before any body arrived | +| `[body not logged: masking failed]` | `Failed` | the provider threw | + +```csharp +using System.Net; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; + +var sink = new CaptureSink(); +var handler = new JsonBodyLoggingHandler(new JsonBodyLoggingOptions { When = JsonBodyLogWhen.Always }, null, sink) +{ + InnerHandler = new PlainText(), +}; +using var client = new HttpClient(handler); + +using var upload = new StreamContent(new ForwardOnlyStream(Encoding.UTF8.GetBytes("""{"file":"data"}"""))); +upload.Headers.ContentType = new("application/json"); +using var response = await client.PostAsync("https://files.example.com/upload", upload); + +var entry = await sink.Entry.Task.WaitAsync(TimeSpan.FromSeconds(5)); +Console.WriteLine($"{entry.RequestBodyStatus} {entry.RequestBody}"); +Console.WriteLine($"{entry.ResponseBodyStatus} {entry.ResponseBody}"); +// Output: +// NotBuffered [body not buffered] +// Skipped [body not logged: text/plain] + +public sealed class CaptureSink : IJsonBodyLogSink +{ + public TaskCompletionSource Entry { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Write(in JsonBodyLogEntry entry) => Entry.TrySetResult(entry); +} + +public sealed class PlainText : HttpMessageHandler +{ + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + await request.Content!.CopyToAsync(Stream.Null, cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("stored", Encoding.UTF8, "text/plain") }; + } +} + +public sealed class ForwardOnlyStream(byte[] data) : MemoryStream(data) +{ + public override bool CanSeek => false; +} +``` + +## Nothing is logged for successful calls + +**Cause:** `When` defaults to `OnFailure`. **Fix:** set `When = JsonBodyLogWhen.Always` for the clients you want fully logged. + +## The log entry appears after `SendAsync` returned, or a test finds no entry + +**Cause:** the response body is read for logging in the background; its entry is written once `MaxBodyBytes` were captured, the body ended, the read failed, or the caller disposed the response. An entry for a call **without** a response (exception, cancellation) is written before the exception reaches the caller. + +**Fix:** in tests, wait on the sink with a timeout (a `TaskCompletionSource` or `SemaphoreSlim` released in `Write`) instead of reading it right after `SendAsync`. In production, make the sink thread-safe and do not depend on ordering with the caller. + +## The response body looks consumed after adding logging + +It is not: the handler replaces `response.Content` with a stream that replays the captured prefix and then continues from the network. Read the content you get **from the response after `SendAsync`**, not a reference to the original content taken inside a lower handler. + +## A timeout is logged as `Canceled` + +`HttpClient.Timeout` and a canceled caller token reach the handler as the same token, so both are `JsonBodyOutcome.Canceled`. Tell them apart at the call site (the caller knows whether it canceled). + +## `IncludeSensitive` in production + +`IncludeSensitive = true` writes bodies **unmasked** (`Raw`, `BodyUnmasked = true`) and logs a warning the first time. Keep it to local debugging; gate it on the environment if it is configurable at all. + +## Registering the handler with `AddHttpMessageHandler()` + +The handler needs the client name and named options; `AddJsonBodyLogging()` supplies them. Registering the type directly loses both. Call `AddJsonBodyLogging` once per client (a second call only applies its `configure` action). + +## Building observers inside `GetMasker` + +`GetMasker` runs for every logged body, possibly concurrently. Building an observer there costs far more than masking with one; build them once (`static readonly` or in the provider's constructor) and return the cached instance. diff --git a/skills/json-observer-http-logging/recipes.md b/skills/json-observer-http-logging/recipes.md new file mode 100644 index 0000000..5266661 --- /dev/null +++ b/skills/json-observer-http-logging/recipes.md @@ -0,0 +1,281 @@ +# Recipes — json-observer-http-logging + +Task-shaped solutions. Each block is a complete program with a fake primary handler; the `// Output:` comment is what it prints. + +## A masker provider built from your models + +Build one observer per body model **once**, from the model's System.Text.Json metadata, and look it up by type. `JsonShape.FromTypeInfo` + `JsonObserver.FromShape` is an allow-list: known fields are logged, the fields you classify are masked, and a field the remote side adds later is masked too. Unknown model types return `null` → `[body withheld]`. + +```csharp +using System.Collections.Frozen; +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using DragoAnt.System.Text.Json.Observer.Strategies; + +var sink = new CaptureSink(); +var handler = new JsonBodyLoggingHandler(new JsonBodyLoggingOptions { When = JsonBodyLogWhen.Always }, new ModelMaskers(), sink) +{ + InnerHandler = new FakeApi(), +}; +using var client = new HttpClient(handler) { BaseAddress = new Uri("https://shop.example.com/") }; + +using var request = new HttpRequestMessage(HttpMethod.Post, "orders") +{ + Content = JsonContent.Create(new PlaceOrder("A1", 2, "4111111111111111")), +}.WithBodyLogging("PlaceOrder"); +using var response = await client.SendAsync(request); + +var entry = await sink.Entry.Task.WaitAsync(TimeSpan.FromSeconds(5)); +Console.WriteLine(entry.RequestBody); +Console.WriteLine(entry.ResponseBody); +// Output: +// {"sku":"A1","quantity":2,"cardNumber":"***1111"} +// {"orderId":"o-7","status":"accepted","fraudScore":"***"} + +public sealed record PlaceOrder(string Sku, int Quantity, string CardNumber); + +public sealed record OrderPlaced(string OrderId, string Status); + +public sealed class ModelMaskers : IJsonBodyMaskerProvider +{ + private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web) { TypeInfoResolver = new DefaultJsonTypeInfoResolver() }; + + private static readonly FrozenDictionary Maskers = new Dictionary + { + [typeof(PlaceOrder)] = For(), + [typeof(OrderPlaced)] = For(), + }.ToFrozenDictionary(); + + public JsonObserver? GetMasker(Type? modelType, string clientName) => + modelType is not null && Maskers.TryGetValue(modelType, out var masker) ? masker : null; + + private static JsonObserver For() => + JsonObserver.FromShape(JsonShape.FromTypeInfo(Json.GetTypeInfo(typeof(T)), Classify)); + + private static MaskTag? Classify(JsonPropertyInfo property) => property.Name switch + { + "cardNumber" => MaskTag.Last4, + "password" or "token" => MaskTag.Full, + _ => null, + }; +} + +public sealed class CaptureSink : IJsonBodyLogSink +{ + public TaskCompletionSource Entry { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Write(in JsonBodyLogEntry entry) => Entry.TrySetResult(entry); +} + +public sealed class FakeApi : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("""{"orderId":"o-7","status":"accepted","fraudScore":0.93}""", Encoding.UTF8, "application/json"), + }); +} +``` + +`fraudScore` is not in `OrderPlaced`, so the shape masks it. Use the same `JsonSerializerOptions` the client serializes with, so property names match what goes over the wire. + +## A custom sink + +Register an `IJsonBodyLogSink` to send entries somewhere other than `ILogger` — a metrics counter, an audit store, a test. `Write` may run concurrently and, for a response, on a background thread after `SendAsync` returned; keep it fast and thread-safe, and never throw for control flow (an exception is reported through the handler's logger, never to the caller). + +```csharp +using System.Collections.Concurrent; +using System.Net; +using System.Text; +using System.Text.Json; +using DragoAnt.System.Text.Json.Observer.Http; +using Microsoft.Extensions.DependencyInjection; + +var sink = new JsonLinesSink(); +var services = new ServiceCollection(); +services.AddSingleton(sink); +services.AddHttpClient("inventory").AddJsonBodyLogging().ConfigurePrimaryHttpMessageHandler(() => new NotFound()); + +await using var provider = services.BuildServiceProvider(); +var client = provider.GetRequiredService().CreateClient("inventory"); +using var response = await client.GetAsync("https://inventory.example.com/items/9?sig=secret"); + +Console.WriteLine(await sink.NextLineAsync()); +// Output: +// {"client":"inventory","method":"GET","path":"/items/9","status":404,"outcome":"Failure","response":"{\u0022error\u0022:\u0022***\u0022}"} + +public sealed class JsonLinesSink : IJsonBodyLogSink +{ + private readonly BlockingCollection _lines = new(); + + public void Write(in JsonBodyLogEntry entry) => _lines.Add(JsonSerializer.Serialize(new + { + client = entry.ClientName, + method = entry.Method.Method, + path = entry.Path, + status = entry.StatusCode, + outcome = entry.Outcome.ToString(), + response = entry.ResponseBody, + })); + + public Task NextLineAsync() => Task.Run(() => _lines.Take()).WaitAsync(TimeSpan.FromSeconds(5)); +} + +public sealed class NotFound : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent("""{"error":"item 9 not found"}""", Encoding.UTF8, "application/json"), + }); +} +``` + +No provider is registered here, so every value is masked; the query string (`sig=secret`) never reaches the entry. + +## The default sink and its log message + +Without a registered sink, entries go to `ILogger` as one structured message, `Information` for `Success` and `Warning` otherwise. Every entry field except `Exception` is a template property (`ClientName`, `Operation`, `Method`, `Path`, `StatusCode`, `ElapsedMs`, `Outcome`, `Truncated`, `BodyUnmasked`, `RequestBodyStatus`, `RequestBody`, `ResponseBodyStatus`, `ResponseBody`), so a structured logger indexes them; the exception is attached to the message. + +```csharp +using System.Net; +using System.Text; +using DragoAnt.System.Text.Json.Observer.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +var logs = new CapturingProvider(); +var services = new ServiceCollection(); +services.AddLogging(builder => builder.AddProvider(logs)); +services.AddHttpClient("weather").AddJsonBodyLogging().ConfigurePrimaryHttpMessageHandler(() => new Unavailable()); + +await using var provider = services.BuildServiceProvider(); +var client = provider.GetRequiredService().CreateClient("weather"); +using var response = await client.GetAsync("https://weather.example.com/today"); + +var (level, properties) = await logs.First.Task.WaitAsync(TimeSpan.FromSeconds(5)); +Console.WriteLine(level); +Console.WriteLine($"{properties["ClientName"]} {properties["StatusCode"]} {properties["Outcome"]} {properties["ResponseBody"]}"); +// Output: +// Warning +// weather 503 Failure {"retryAfter":"***"} + +public sealed class CapturingProvider : ILoggerProvider +{ + public TaskCompletionSource<(LogLevel, Dictionary)> First { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ILogger CreateLogger(string categoryName) => new Logger(this); + + public void Dispose() + { + } + + private sealed class Logger(CapturingProvider owner) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (state is IEnumerable> pairs && pairs.Any(p => p.Key == "ResponseBody")) + { + owner.First.TrySetResult((logLevel, pairs.ToDictionary(p => p.Key, p => p.Value))); + } + } + } +} + +public sealed class Unavailable : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable) + { + Content = new StringContent("""{"retryAfter":30}""", Encoding.UTF8, "application/json"), + }); +} +``` + +## Log every call for one client, failures for the rest + +Options are named after the client. Configure the exception client in its own `AddJsonBodyLogging`, and shared settings with `ConfigureAll` (applied first, so the per-client action wins). + +```csharp +using DragoAnt.System.Text.Json.Observer.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +var services = new ServiceCollection(); +services.ConfigureAll(options => +{ + options.When = JsonBodyLogWhen.OnFailure; + options.MaxBodyBytes = 8 * 1024; +}); +services.AddHttpClient("payments").AddJsonBodyLogging(options => options.When = JsonBodyLogWhen.Always); +services.AddHttpClient("catalog").AddJsonBodyLogging(); +services.AddHttpClient("health").AddJsonBodyLogging(options => options.When = JsonBodyLogWhen.Never); + +await using var provider = services.BuildServiceProvider(); +var monitor = provider.GetRequiredService>(); +Console.WriteLine(string.Join(", ", new[] { "payments", "catalog", "health" }.Select(n => $"{n}={monitor.Get(n).When}"))); +// Output: +// payments=Always, catalog=OnFailure, health=Never +``` + +## Large and streaming responses + +Only the first `MaxBodyBytes` of a body are read for the log; the caller still receives the whole body, and `HttpCompletionOption.ResponseHeadersRead` still returns as soon as the headers arrive. A longer body is logged as its masked prefix, closed into valid JSON, with `ResponseBodyStatus = Truncated` and `Truncated = true`. + +```csharp +using System.Net; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var sink = new CaptureSink(); +var options = new JsonBodyLoggingOptions { When = JsonBodyLogWhen.Always, MaxBodyBytes = 40 }; +var handler = new JsonBodyLoggingHandler(options, new KeepAll(), sink) { InnerHandler = new BigList() }; +using var client = new HttpClient(handler); + +using var response = await client.GetAsync("https://api.example.com/items", HttpCompletionOption.ResponseHeadersRead); +var body = await response.Content.ReadAsStringAsync(); +var entry = await sink.Entry.Task.WaitAsync(TimeSpan.FromSeconds(5)); + +Console.WriteLine($"caller read {body.Length} chars"); +Console.WriteLine($"{entry.ResponseBodyStatus} {entry.Truncated} {entry.ResponseBody}"); +// Output: +// caller read 1001 chars +// Truncated True {"items":[{"id":0},{"id":1},{"id":2}]} + +public sealed class KeepAll : IJsonBodyMaskerProvider +{ + private static readonly JsonObserver Masker = JsonObserver.Obj(BlockList); + + public JsonObserver? GetMasker(Type? modelType, string clientName) => Masker; +} + +public sealed class CaptureSink : IJsonBodyLogSink +{ + public TaskCompletionSource Entry { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Write(in JsonBodyLogEntry entry) => Entry.TrySetResult(entry); +} + +public sealed class BigList : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var items = string.Join(",", Enumerable.Range(0, 100).Select(i => $$"""{"id":{{i}}}""")); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent($$"""{"items":[{{items}}]}""", Encoding.UTF8, "application/json"), + }); + } +} +``` From c3f694f38426322e0311105ceba380beca484ba0 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:06:38 +0200 Subject: [PATCH 3/4] Add the json-observer-testing skill Golden, secret-absent, load-bearing, truncation and corruption fuzz, allocation, concurrency and HTTP logging tests, each a passing xUnit class run by the skill snippet tests. --- skills/json-observer-testing/SKILL.md | 49 +++ skills/json-observer-testing/examples.md | 417 +++++++++++++++++++++++ skills/json-observer-testing/pitfalls.md | 121 +++++++ skills/json-observer-testing/recipes.md | 235 +++++++++++++ 4 files changed, 822 insertions(+) create mode 100644 skills/json-observer-testing/SKILL.md create mode 100644 skills/json-observer-testing/examples.md create mode 100644 skills/json-observer-testing/pitfalls.md create mode 100644 skills/json-observer-testing/recipes.md diff --git a/skills/json-observer-testing/SKILL.md b/skills/json-observer-testing/SKILL.md new file mode 100644 index 0000000..59815f1 --- /dev/null +++ b/skills/json-observer-testing/SKILL.md @@ -0,0 +1,49 @@ +--- +name: json-observer-testing +description: Write xUnit tests that prove JSON masking built with DragoAnt.System.Text.Json.Observer never leaks a secret — golden input-to-output cases, "the secret never appears" assertions across payload shapes, truncation and byte-corruption fuzzing of the never-throw APIs (output must stay valid JSON with no secret), MaskResult/MaskStatus checks, a load-bearing check that fails when a rule is removed, allocation budgets with GC.GetAllocatedBytesForCurrentThread, thread-safety checks on a shared JsonObserver, and HttpClient body-logging tests for JsonBodyLoggingHandler with a fake inner handler and a capturing IJsonBodyLogSink. Use when adding unit or regression tests for masking rules, a JsonObserver configuration, an IJsonBodyMaskerProvider or a body-logging setup, reproducing a secret that leaked into logs, or when asked "how do I prove nothing sensitive gets logged". +--- + +# Testing masking with DragoAnt.System.Text.Json.Observer + +A masking rule is a security control, so its tests must show that **the secret is absent**, not only that the output looks right. Seven kinds of test cover it; each has a complete xUnit v3 example in [examples.md](./examples.md), using plain `Assert`. + +Packages: `DragoAnt.System.Text.Json.Observer` (and `.Http` for handler tests), `xunit.v3`. Keep the observer under test in the production code (a `static readonly` field or a factory method) and call **that** from the test — a test that rebuilds the rules by hand tests its own copy. + +## Which test, when + +| Test | Proves | Use it for | +| --- | --- | --- | +| [Golden](./examples.md#golden-cases) | exact masked output for given input | every rule set; documents the intent | +| [Secret absent](./examples.md#the-secret-never-appears) | no secret value appears, across payload shapes (nested, arrays, numbers, casing) | every sensitive field; regression tests for a leak | +| [Load-bearing rule](./examples.md#the-rule-is-load-bearing) | the test fails when the rule is removed | proving a secret-absent test can fail at all | +| [Truncation fuzz](./examples.md#truncation-fuzz) | every prefix of a payload stays valid JSON with no secret | bodies logged with a size cap or cut streams | +| [Corruption fuzz](./examples.md#corruption-fuzz) | random byte damage never throws or leaks | untrusted or binary-unsafe input | +| [Allocation budget](./examples.md#allocation-budget) | per-call allocations stay under a measured budget | hot-path maskers | +| [Concurrency](./examples.md#concurrency) | a shared observer gives the same output on many threads | observers in `static` fields or singletons | +| [HTTP logging](./examples.md#http-body-logging) | `JsonBodyLoggingHandler` logs masked bodies and the caller still gets the full response | `IJsonBodyMaskerProvider` and handler setups | + +## Rules that matter + +1. **Assert absence, not just shape.** `Assert.DoesNotContain(secret, output)` for every secret value, plus a golden check of the exact output. A golden string alone hides a leak inside a long literal nobody reads. +2. **Use distinctive secret values** (`S3cr3t-7f2a`), never `"x"` or `"1"` — a short value appears by accident elsewhere and makes absence assertions meaningless. +3. **Cover the shapes that defeat rules:** the field nested deeper, inside an array, as a number or boolean, as an object, with different casing, cut off mid-value. Each is a separate case. +4. **Prove the test can fail.** Remove the rule (or build the observer without it) and the test must go red; a check that passes either way protects nothing. Run this once by hand when writing the test, or keep it as a test ([examples.md#the-rule-is-load-bearing](./examples.md#the-rule-is-load-bearing)). +5. **Fuzz with an allow-list observer.** Random corruption can rename a property (`password` → `pas#word`); under `BlockList` that value is then legitimately unmasked, so a block-list fuzz fails for the wrong reason. Under an allow-list, unknown names are masked, so "no secret" must always hold. Truncation fuzz (cutting, not changing bytes) is safe with any observer. +6. **When output is non-empty it must parse.** For every status, `BytesWritten > 0` means the output is valid JSON; `NotJson` means empty output. +7. **Set allocation budgets from measurements**, with warm-up calls first and a reused output buffer, on the UTF-8 API. Never assert zero. +8. **HTTP response entries are written asynchronously.** Wait on the sink with a timeout; never read it straight after `SendAsync`. + +## Pitfalls (details in [pitfalls.md](./pitfalls.md)) + +- A test that passes before the fix — the regression test does not reproduce the leak; write it red first. +- Asserting `Masked` on a payload you cut on purpose (it is `Truncated`), or on plain text (it is `Invalid`). +- Comparing output that contains `MaskTag.Hash` without a fixed `HashKey` — the default key is random per process. +- Culture-dependent assertions on numbers read with `ReadDecimal` — format with the invariant culture. + +## Companions + +- [examples.md](./examples.md) — one complete xUnit test class per test kind. +- [recipes.md](./recipes.md) — a reusable fuzz helper, a regression test for a reported leak, testing a masker provider, payload builders. +- [pitfalls.md](./pitfalls.md) — false greens and false reds, and why. + +Every C# block in these files is a complete xUnit v3 test class (implicit usings) that compiles against the library and passes. diff --git a/skills/json-observer-testing/examples.md b/skills/json-observer-testing/examples.md new file mode 100644 index 0000000..bee5675 --- /dev/null +++ b/skills/json-observer-testing/examples.md @@ -0,0 +1,417 @@ +# Examples — json-observer-testing + +Each block is a complete xUnit v3 test file: a test project referencing `xunit.v3` and `DragoAnt.System.Text.Json.Observer` (plus `.Http` for the last one). The `LogMaskers` class stands in for **your production code** — in a real project the tests reference it instead of declaring it. + +## Golden cases + +Exact input → output pairs. Raw string literals keep the JSON readable; one `[InlineData]` per behaviour you rely on. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public static class LogMaskers +{ + public static readonly JsonObserver Body = JsonObserver.Obj(Relative(rules => rules + .Match(PropMatches.OneOf("password", "pin")).MaskAny(MaskTag.Full) + .Match("card", "number").MaskAny(MaskTag.Last4), + BlockList)); +} + +public sealed class LogMaskersGoldenTests +{ + [Theory] + [InlineData("""{"user":"bob","password":"p"}""", """{"user":"bob","password":"***"}""")] + [InlineData("""{"PIN":1234}""", """{"PIN":"***"}""")] + [InlineData("""{"card":{"number":"4111111111111111","brand":"visa"}}""", """{"card":{"number":"***1111","brand":"visa"}}""")] + [InlineData("""{"a":{"b":{"password":{"old":"p1","new":"p2"}}}}""", """{"a":{"b":{"password":"***"}}}""")] + [InlineData("""{"password":null}""", """{"password":null}""")] + public void Body_MasksSensitiveFields(string input, string expected) => + Assert.Equal(expected, LogMaskers.Body.Mask(input)); +} +``` + +## The secret never appears + +One secret value, many shapes that could defeat a rule. Assert absence first, then that the non-sensitive data survived — so a masker that masks everything fails too. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public static class LogMaskers +{ + public static readonly JsonObserver Body = JsonObserver.Any(_ => { }, _ => { }, Relative(rules => rules + .Match(PropMatches.Contains("password")).MaskAny("***") + .Match(PropMatches.EndsWith("token")).MaskAny("***"), + BlockList)); +} + +public sealed class LogMaskersSecretTests +{ + private const string Secret = "S3cr3t-7f2a"; + + [Theory] + [InlineData("""{"password":"S3cr3t-7f2a","keep":"visible"}""")] + [InlineData("""{"login":{"newPassword":"S3cr3t-7f2a"},"keep":"visible"}""")] + [InlineData("""{"users":[{"PASSWORD":"S3cr3t-7f2a"}],"keep":"visible"}""")] + [InlineData("""[{"refresh_token":"S3cr3t-7f2a"},{"keep":"visible"}]""")] + [InlineData("""{"password":{"value":"S3cr3t-7f2a","hint":"S3cr3t-7f2a"},"keep":"visible"}""")] + [InlineData("""{"password":["S3cr3t-7f2a"],"keep":"visible"}""")] + public void Body_NeverContainsTheSecret(string input) + { + var output = LogMaskers.Body.Mask(input, out var result); + + Assert.Equal(MaskStatus.Masked, result.Status); + Assert.DoesNotContain(Secret, output); + Assert.Contains("visible", output); + } + + [Fact] + public void Body_NumericSecret_IsMasked() + { + var output = LogMaskers.Body.Mask("""{"accessToken":987654321}"""); + + Assert.DoesNotContain("987654321", output); + } +} +``` + +## The rule is load-bearing + +A secret-absent test is only worth something if it fails without the rule. Keep that proof as a test: build the same masker without the rule and show the secret comes through, using the **same** payload. When the production masker is built by a method with a switch for the rule, this costs two lines. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public static class LogMaskers +{ + public static readonly JsonObserver Body = Build(maskCvv: true); + + internal static JsonObserver Build(bool maskCvv) => JsonObserver.Obj(Relative(rules => + { + rules.Match("password").MaskAny("***"); + if (maskCvv) + { + rules.Match("cvv").MaskAny("***"); + } + }, BlockList)); +} + +public sealed class LogMaskersLoadBearingTests +{ + private const string Payload = """{"card":{"cvv":731,"brand":"visa"}}"""; + + [Fact] + public void Cvv_IsMasked() => Assert.DoesNotContain("731", LogMaskers.Body.Mask(Payload)); + + [Fact] + public void Cvv_WithoutTheRule_Leaks_SoTheTestAboveCanFail() => + Assert.Contains("731", LogMaskers.Build(maskCvv: false).Mask(Payload)); +} +``` + +## Truncation fuzz + +Logged bodies are often cut at a size limit. Feed every prefix of a realistic payload through the UTF-8 API: it must never throw, never write a secret, and produce valid JSON whenever it writes anything. + +```csharp +using System.Buffers; +using System.Text; +using System.Text.Json; +using DragoAnt.System.Text.Json.Observer; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public static class LogMaskers +{ + public static readonly JsonObserver Body = JsonObserver.Obj(Relative(rules => rules + .Match("password").MaskAny("***") + .Match("pin").MaskAny("***"), + BlockList)); +} + +public sealed class LogMaskersTruncationTests +{ + private const string Secret = "S3cr3t-7f2a"; + + [Fact] + public void EveryPrefix_IsValidJson_WithoutTheSecret() + { + var payload = Encoding.UTF8.GetBytes( + $$"""{"user":"bob","password":"{{Secret}}","items":[{"pin":"{{Secret}}","qty":1.5},{"pin":["{{Secret}}"]}],"ok":true}"""); + var output = new ArrayBufferWriter(); + + for (var cut = 0; cut <= payload.Length; cut++) + { + output.ResetWrittenCount(); + var result = LogMaskers.Body.Mask(payload.AsSpan(0, cut), output); + var text = Encoding.UTF8.GetString(output.WrittenSpan); + + Assert.DoesNotContain(Secret, text); + Assert.Equal(result.BytesWritten, output.WrittenCount); + if (result.BytesWritten > 0) + { + using var parsed = JsonDocument.Parse(text); + } + + var expected = cut == payload.Length ? MaskStatus.Masked : cut == 0 ? MaskStatus.NotJson : MaskStatus.Truncated; + Assert.Equal(expected, result.Status); + } + } +} +``` + +## Corruption fuzz + +Random byte damage with a **fixed seed** (reproducible failures), on an **allow-list** observer: a corrupted property name is then still masked, so "no secret" must hold for every case. Any status is acceptable; throwing, leaking, or unparseable output is not. + +```csharp +using System.Buffers; +using System.Text; +using System.Text.Json; +using DragoAnt.System.Text.Json.Observer; +using Xunit; + +public static class LogMaskers +{ + public static readonly JsonObserver AllowListed = JsonObserver.Any( + root => root.Match("user").Unmasked().Match("ok").Unmasked(), + _ => { }); +} + +public sealed class LogMaskersCorruptionTests +{ + private const string Secret = "S3cr3t-7f2a"; + + [Fact] + public void CorruptedPayloads_NeverThrowNeverLeak() + { + var payload = Encoding.UTF8.GetBytes($$"""{"user":"bob","password":"{{Secret}}","items":[{"pin":"{{Secret}}"}],"ok":true}"""); + var random = new Random(20261003); + var output = new ArrayBufferWriter(); + + for (var i = 0; i < 2_000; i++) + { + var corrupted = payload.ToArray(); + for (var flips = random.Next(1, 4); flips > 0; flips--) + { + corrupted[random.Next(corrupted.Length)] = (byte)random.Next(256); + } + + output.ResetWrittenCount(); + var result = LogMaskers.AllowListed.Mask(corrupted.AsSpan(0, random.Next(corrupted.Length + 1)), output); + var text = Encoding.UTF8.GetString(output.WrittenSpan); + + Assert.DoesNotContain(Secret, text); + if (result.BytesWritten > 0) + { + using var parsed = JsonDocument.Parse(text); + } + } + } +} +``` + +## Allocation budget + +Warm up first (path buffers grow on the first calls), reuse the output buffer, measure many calls on the current thread, and compare the average with a budget taken from a measurement plus headroom. Keep the payload and rule kinds representative: a masking function allocates the `string` it receives, so it needs a larger budget than constant or tag rules. + +```csharp +using System.Buffers; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public static class LogMaskers +{ + public static readonly JsonObserver Body = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); +} + +public sealed class LogMaskersAllocationTests +{ + [Theory] + [InlineData(1_024, 1_024)] + [InlineData(64 * 1_024, 1_024)] + public void Utf8Api_StaysWithinBudget(int size, long budgetPerCall) + { + var payload = Payload(size); + var output = new ArrayBufferWriter(payload.Length * 2); + for (var i = 0; i < 20; i++) + { + output.ResetWrittenCount(); + LogMaskers.Body.Mask(payload, output); + } + + const int calls = 50; + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < calls; i++) + { + output.ResetWrittenCount(); + LogMaskers.Body.Mask(payload, output); + } + + var perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / calls; + Assert.True(perCall <= budgetPerCall, $"{size} B payload allocates {perCall} B per call"); + } + + private static byte[] Payload(int size) + { + var json = new StringBuilder("""{"password":"secret" """); + for (var i = 0; json.Length < size; i++) + { + json.Append(",\"field").Append(i).Append("\":\"value ").Append(i).Append('"'); + } + + return Encoding.UTF8.GetBytes(json.Append('}').ToString()); + } +} +``` + +## Concurrency + +An observer is meant to be shared. Run the same inputs on many threads at once — including the very first calls, while internal buffers are still sized — and compare with the single-threaded result. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public sealed class SharedObserverTests +{ + [Fact] + public void ConcurrentCalls_GiveTheSameOutput() + { + var observer = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + var deep = """{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"password":"p","keep":1}}}}}}}}"""; + var flat = """{"password":"p","keep":1}"""; + var expectedDeep = """{"a":{"b":{"c":{"d":{"e":{"f":{"g":{"password":"***","keep":1}}}}}}}}"""; + var expectedFlat = """{"password":"***","keep":1}"""; + + var failures = 0; + Parallel.For(0, 2_000, new ParallelOptions { MaxDegreeOfParallelism = 8 }, i => + { + var (input, expected) = i % 2 == 0 ? (deep, expectedDeep) : (flat, expectedFlat); + if (observer.Mask(input) != expected) + { + Interlocked.Increment(ref failures); + } + }); + + Assert.Equal(0, failures); + } +} +``` + +## HTTP body logging + +Test `JsonBodyLoggingHandler` without a network: a fake inner handler returns a canned response, a capturing sink records entries, and a provider returns your production observers. The response entry is written on a background read — wait for it with a timeout. Assert three things: the secrets are absent from the entry, the non-sensitive data is present, and the caller still received the full, unmasked response. + +```csharp +using System.Collections.Concurrent; +using System.Net; +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public sealed record Login(string User, string Password); + +public sealed record Session(string Token, string User); + +public sealed class AuthMaskers : IJsonBodyMaskerProvider +{ + private static readonly JsonObserver Auth = JsonObserver.Obj(Relative(rules => rules + .Match("password").MaskAny("***") + .Match("token").MaskAny("***"), + BlockList)); + + public JsonObserver? GetMasker(Type? modelType, string clientName) => + modelType == typeof(Login) || modelType == typeof(Session) ? Auth : null; +} + +public sealed class AuthClientLoggingTests +{ + [Fact] + public async Task FailedLogin_LogsMaskedBodies_AndCallerGetsFullResponse() + { + var sink = new CapturingSink(); + var inner = new StubHandler(HttpStatusCode.Unauthorized, """{"token":"tok-9f1c","user":"bob","error":"locked"}"""); + var handler = new JsonBodyLoggingHandler(new JsonBodyLoggingOptions(), new AuthMaskers(), sink) { InnerHandler = inner }; + using var client = new HttpClient(handler); + + using var request = new HttpRequestMessage(HttpMethod.Post, "https://auth.example.com/login?otp=123456") + { + Content = new StringContent("""{"user":"bob","password":"pw-5e8d"}""", Encoding.UTF8, "application/json"), + }.WithBodyLogging("Login"); + using var response = await client.SendAsync(request); + var received = await response.Content.ReadAsStringAsync(); + + var entry = await sink.WaitAsync(); + Assert.Equal(JsonBodyOutcome.Failure, entry.Outcome); + Assert.Equal("/login", entry.Path); + Assert.Equal(JsonBodyStatus.Masked, entry.RequestBodyStatus); + Assert.Equal(JsonBodyStatus.Masked, entry.ResponseBodyStatus); + Assert.DoesNotContain("pw-5e8d", entry.RequestBody); + Assert.DoesNotContain("tok-9f1c", entry.ResponseBody); + Assert.Contains("locked", entry.ResponseBody); + Assert.Contains("tok-9f1c", received); + } + + [Fact] + public async Task SuccessfulCall_IsNotLogged_ByDefault() + { + var sink = new CapturingSink(); + var handler = new JsonBodyLoggingHandler(new JsonBodyLoggingOptions(), new AuthMaskers(), sink) + { + InnerHandler = new StubHandler(HttpStatusCode.OK, """{"token":"t","user":"bob"}"""), + }; + using var client = new HttpClient(handler); + + using var response = await client.GetAsync("https://auth.example.com/me"); + await response.Content.ReadAsStringAsync(); + + Assert.Empty(sink.Entries); + } +} + +public sealed class StubHandler(HttpStatusCode status, string json) : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(status) { Content = new StringContent(json, Encoding.UTF8, "application/json") }); +} + +public sealed class CapturingSink : IJsonBodyLogSink +{ + private readonly ConcurrentQueue _entries = new(); + private readonly SemaphoreSlim _written = new(0); + + public IReadOnlyCollection Entries => _entries.ToArray(); + + public void Write(in JsonBodyLogEntry entry) + { + _entries.Enqueue(entry); + _written.Release(); + } + + public async Task WaitAsync(TimeSpan? timeout = null) + { + if (!await _written.WaitAsync(timeout ?? TimeSpan.FromSeconds(5))) + { + throw new TimeoutException("no log entry was written"); + } + + return Assert.Single(_entries); + } +} +``` + +A call the options do not log (a success under `OnFailure`) never starts a capture, so "nothing was logged" can be asserted right after the call. A logged call's response entry, by contrast, always needs the wait. diff --git a/skills/json-observer-testing/pitfalls.md b/skills/json-observer-testing/pitfalls.md new file mode 100644 index 0000000..fb328ef --- /dev/null +++ b/skills/json-observer-testing/pitfalls.md @@ -0,0 +1,121 @@ +# Pitfalls — json-observer-testing + +False greens (a test that cannot catch the leak) and false reds (a test that fails for the wrong reason). Each block is a complete xUnit v3 test file that passes and shows the correct form. + +## False green: the test rebuilds the rules + +A test that declares its own `JsonObserver.Obj(...)` with "the same" rules tests that copy, not the production masker. Reference the production field or factory; if it is `internal`, use `InternalsVisibleTo` for the test project. + +## False green: the secret value is too short or too common + +`Assert.DoesNotContain("1", output)` fails on any other `1`; `Assert.DoesNotContain("x", output)` may pass while the real value leaks under another name. Use distinctive values (`S3cr3t-7f2a`, `4111111111111111`) that cannot occur by accident. + +## False green: only the golden string is checked + +A long golden literal hides a leak nobody reads. Add an explicit `DoesNotContain` per secret, and a `Contains` for a safe value so a mask-everything regression also fails. + +## False green: the regression test passes before the fix + +It does not reproduce the leak — usually the payload was simplified and lost the shape that defeated the rule (an array level, a number instead of a string, different casing). Rebuild it from the reported structure and see it fail first ([recipes.md](./recipes.md#a-regression-test-for-a-reported-leak)). + +## False red: corruption fuzz on a block-list observer + +Random corruption can change a property name; under `BlockList` the renamed property is not sensitive any more and its value is written — correctly. Fuzz corruption with an allow-list observer; fuzz block-list observers with prefixes only. + +```csharp +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public sealed class WhyNotBlockListCorruptionTests +{ + [Fact] + public void RenamedProperty_EscapesABlockListRule_ButNotAnAllowList() + { + var corrupted = """{"pas#word":"S3cr3t-7f2a"}"""; + + var blockList = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + var allowList = JsonObserver.Obj(root => root.Match("user").Unmasked()); + + Assert.Contains("S3cr3t-7f2a", blockList.Mask(corrupted)); + Assert.DoesNotContain("S3cr3t-7f2a", allowList.Mask(corrupted)); + } +} +``` + +## False red: expecting `Masked` for a cut or non-JSON input + +A cut payload is `Truncated`; plain text is `Invalid`; a JSON scalar root (`42`, `"text"`) or empty input is `NotJson`; a root array given to `JsonObserver.Obj(...)` is `Invalid`. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public sealed class StatusExpectationTests +{ + private static readonly JsonObserver Observer = JsonObserver.Obj(BlockList); + + [Theory] + [InlineData("""{"a":1}""", MaskStatus.Masked)] + [InlineData("""{"a":1""", MaskStatus.Truncated)] + [InlineData("hello", MaskStatus.Invalid)] + [InlineData("42", MaskStatus.NotJson)] + [InlineData("", MaskStatus.NotJson)] + [InlineData("[1]", MaskStatus.Invalid)] + public void Status_MatchesTheInput(string input, MaskStatus expected) + { + Observer.Mask(input, out var result); + Assert.Equal(expected, result.Status); + } +} +``` + +## False red: `MaskTag.Hash` output in a golden string + +Without `JsonObserverOptions.HashKey`, the key is random per process, so the hash changes every run. Pass a fixed key in the test, or assert the shape (`hash:` + 16 hex characters) and equality between two calls. + +```csharp +using System.Text; +using System.Text.RegularExpressions; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public sealed class HashTests +{ + private static readonly JsonObserver Observer = JsonObserver.Obj(Relative(rules => rules.Match("email").MaskAny(MaskTag.Hash), BlockList)); + + [Fact] + public void Hash_HasAStableShape_AndCorrelates() + { + var options = new JsonObserverOptions(HashKey: Encoding.UTF8.GetBytes("test-key")); + + var first = Observer.Mask("""{"email":"a@b.c"}""", options); + var second = Observer.Mask("""{"email":"a@b.c"}""", options); + var other = Observer.Mask("""{"email":"z@b.c"}""", options); + + Assert.Matches("""^\{"email":"hash:[0-9a-f]{16}"\}$""", first); + Assert.Equal(first, second); + Assert.NotEqual(first, other); + } +} +``` + +## False red: an HTTP response entry read too early + +The response entry is written by a background read. A test that inspects the sink right after `SendAsync` races it. Wait on the sink with a timeout ([examples.md](./examples.md#http-body-logging)). + +## False red: culture-dependent numbers + +A context value read with `ReadDecimal` is a `decimal`; formatting it with the current culture prints `19,90` on some machines. Compare the `decimal` itself, or format with `CultureInfo.InvariantCulture`. + +## Flaky: allocation budgets measured cold or across threads + +Measure on one thread with `GC.GetAllocatedBytesForCurrentThread()`, after warm-up calls, with a reused output buffer and an average over many calls. `GC.GetTotalAllocatedBytes` counts other threads (the test runner included). + +## Slow: fuzz loops that are too large + +Every prefix of a payload of a few hundred bytes and a few thousand seeded corruptions take well under a second. Grow the payload, not the iteration count, when you need more coverage — and keep the seed fixed so a failure is reproducible. diff --git a/skills/json-observer-testing/recipes.md b/skills/json-observer-testing/recipes.md new file mode 100644 index 0000000..2cd4da7 --- /dev/null +++ b/skills/json-observer-testing/recipes.md @@ -0,0 +1,235 @@ +# Recipes — json-observer-testing + +Each block is a complete xUnit v3 test file that compiles and passes; `LogMaskers` and the record types stand in for your production code. + +## A reusable leak fuzz helper + +Put the fuzz loop in one helper and call it for every masker and payload that matters. It cuts the payload at every byte and, for an allow-list observer, also corrupts random bytes with a fixed seed. It collects every failure before asserting, so one run shows them all. + +```csharp +using System.Buffers; +using System.Text; +using System.Text.Json; +using DragoAnt.System.Text.Json.Observer; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public static class LogMaskers +{ + public static readonly JsonObserver BlockListed = JsonObserver.Obj(Relative(rules => rules.Match("password").MaskAny("***"), BlockList)); + + public static readonly JsonObserver AllowListed = JsonObserver.Obj(root => root.Match("user").Unmasked()); +} + +public static class MaskingFuzz +{ + /// + /// Masks every prefix of (and, when is positive, that many randomly + /// corrupted copies) and fails when a call throws, writes a secret, or writes output that is not valid JSON. + /// Corrupt only with an allow-list observer: a corrupted property name legitimately escapes a block-list rule. + /// + public static void AssertNoLeak(JsonObserver observer, string payload, IReadOnlyList secrets, int corruptions = 0, int seed = 1) + { + var utf8 = Encoding.UTF8.GetBytes(payload); + var failures = new List(); + var output = new ArrayBufferWriter(); + + for (var cut = 0; cut <= utf8.Length; cut++) + { + Check(utf8.AsSpan(0, cut), $"prefix {cut}"); + } + + var random = new Random(seed); + for (var i = 0; i < corruptions; i++) + { + var corrupted = utf8.ToArray(); + for (var flips = random.Next(1, 4); flips > 0; flips--) + { + corrupted[random.Next(corrupted.Length)] = (byte)random.Next(256); + } + + Check(corrupted.AsSpan(0, random.Next(corrupted.Length + 1)), $"corruption {i}"); + } + + Assert.True(failures.Count == 0, string.Join(Environment.NewLine, failures.Take(20))); + + void Check(ReadOnlySpan input, string label) + { + output.ResetWrittenCount(); + MaskResult result; + try + { + result = observer.Mask(input, output); + } + catch (Exception ex) + { + failures.Add($"{label}: threw {ex.GetType().Name}"); + return; + } + + var text = Encoding.UTF8.GetString(output.WrittenSpan); + foreach (var secret in secrets.Where(s => text.Contains(s, StringComparison.Ordinal))) + { + failures.Add($"{label}: {result.Status} leaked '{secret}' in {text}"); + } + + if (result.BytesWritten > 0 && !IsJson(text)) + { + failures.Add($"{label}: {result.Status} wrote invalid JSON {text}"); + } + } + } + + private static bool IsJson(string text) + { + try + { + using var _ = JsonDocument.Parse(text); + return true; + } + catch (JsonException) + { + return false; + } + } +} + +public sealed class LogMaskersFuzzTests +{ + private const string Payload = """{"user":"bob","password":"S3cr3t-7f2a","profile":{"password":"S3cr3t-7f2a","age":41},"list":[{"password":"S3cr3t-7f2a"}]}"""; + + [Fact] + public void BlockListed_Prefixes_NeverLeak() => + MaskingFuzz.AssertNoLeak(LogMaskers.BlockListed, Payload, ["S3cr3t-7f2a"]); + + [Fact] + public void AllowListed_PrefixesAndCorruptions_NeverLeak() => + MaskingFuzz.AssertNoLeak(LogMaskers.AllowListed, Payload, ["S3cr3t-7f2a", "41"], corruptions: 2_000, seed: 20261003); +} +``` + +## A regression test for a reported leak + +1. **Take the payload from the report** and replace real data with distinctive fakes, keeping the structure exactly — the structure is usually what defeated the rule. +2. **Write the test against the current masker and watch it fail.** A regression test that passes before the fix does not reproduce the leak. +3. Fix the rule; the test goes green. Keep the failing shape in the test name. + +Below, a leak through an array: `Match("credentials", "secret")` does not cross the array, because an array item is a path level of its own. The fixed rule names the item level. + +```csharp +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using Xunit; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +public static class LogMaskers +{ + private static readonly PropMatchingStrategy AnyItem = new(_ => true); + + public static readonly JsonObserver BeforeFix = JsonObserver.Obj(Relative(rules => rules + .Match("credentials", "secret").MaskAny("***"), + BlockList)); + + public static readonly JsonObserver Body = JsonObserver.Obj(Relative(rules => rules + .Match("credentials", "secret").MaskAny("***") + .Match("credentials", AnyItem, "secret").MaskAny("***"), + BlockList)); +} + +public sealed class CredentialsLeakRegressionTests +{ + private const string ReportedShape = """{"account":"a-1","credentials":[{"kind":"api","secret":"fake-secret-91c3"}]}"""; + + [Fact] + public void SecretInsideCredentialsArray_IsMasked() => + Assert.DoesNotContain("fake-secret-91c3", LogMaskers.Body.Mask(ReportedShape)); + + [Fact] + public void SecretInsideCredentialsArray_LeakedBeforeTheFix() => + Assert.Contains("fake-secret-91c3", LogMaskers.BeforeFix.Mask(ReportedShape)); + + [Fact] + public void SecretInsideCredentialsObject_StillMasked() => + Assert.DoesNotContain("fake-secret-91c3", LogMaskers.Body.Mask("""{"credentials":{"secret":"fake-secret-91c3"}}""")); +} +``` + +## Test every model a masker provider serves + +Serialize a sample of each model with secret-looking values in its sensitive properties — the same serializer settings the `HttpClient` uses — and mask it with the observer the provider returns. Adding a model to the provider then needs one line in the test data, and forgetting its masker fails the test. + +```csharp +using System.Text.Json; +using System.Text.Json.Serialization.Metadata; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Http; +using DragoAnt.System.Text.Json.Observer.Strategies; +using Xunit; + +public sealed record SignUp(string Email, string Password, string Country); + +public sealed record Payment(string CardNumber, decimal Amount); + +public sealed class ApiMaskers : IJsonBodyMaskerProvider +{ + public static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web) { TypeInfoResolver = new DefaultJsonTypeInfoResolver() }; + + private static readonly Dictionary Maskers = new() + { + [typeof(SignUp)] = For(), + [typeof(Payment)] = For(), + }; + + public JsonObserver? GetMasker(Type? modelType, string clientName) => + modelType is not null && Maskers.TryGetValue(modelType, out var masker) ? masker : null; + + private static JsonObserver For() => JsonObserver.FromShape(JsonShape.FromTypeInfo( + Json.GetTypeInfo(typeof(T)), + property => property.Name switch + { + "email" => MaskTag.Hash, + "password" => MaskTag.Full, + "cardNumber" => MaskTag.Last4, + _ => null, + })); +} + +public sealed class ApiMaskersTests +{ + private static readonly (object Sample, string[] Secrets)[] Samples = + [ + (new SignUp("alice@example.com", "S3cr3t-7f2a", "NL"), ["alice@example.com", "S3cr3t-7f2a"]), + (new Payment("4111111111111111", 10.5m), ["4111111111111111"]), + ]; + + [Theory] + [InlineData(typeof(SignUp))] + [InlineData(typeof(Payment))] + public void EveryModel_HasAMasker(Type model) => + Assert.NotNull(new ApiMaskers().GetMasker(model, "api")); + + [Fact] + public void EverySample_IsMaskedWithoutLosingSafeFields() + { + foreach (var (sample, secrets) in Samples) + { + var masker = new ApiMaskers().GetMasker(sample.GetType(), "api")!; + var masked = masker.Mask(JsonSerializer.Serialize(sample, sample.GetType(), ApiMaskers.Json))!; + + foreach (var secret in secrets) + { + Assert.DoesNotContain(secret, masked); + } + } + + var signUp = new ApiMaskers().GetMasker(typeof(SignUp), "api")!.Mask("""{"email":"a@b.c","password":"p","country":"NL"}"""); + Assert.Contains("\"country\":\"NL\"", signUp); + } + + [Fact] + public void AFieldTheModelDoesNotKnow_IsMasked() => + Assert.Equal( + """{"cardNumber":"***1111","amount":10.5,"cvv":"***"}""", + new ApiMaskers().GetMasker(typeof(Payment), "api")!.Mask("""{"cardNumber":"4111111111111111","amount":10.5,"cvv":"123"}""")); +} +``` From 78c2e931a5a7657d701e328d6eeab87ea52f55bc Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:09:01 +0200 Subject: [PATCH 4/4] Describe the agent skills and how to install them per agent Install folders checked against the Claude Code, Codex, Copilot, Cursor, Gemini CLI and Antigravity docs on 2026-10-03. --- README.md | 2 +- docs/skills.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1cc8f82..c73293a 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ sealed class Order ### Using an AI coding agent? -Ready-made agent skills for this library are described in [Agent skills](./docs/skills.md). +Ready-made agent skills for masking, HTTP body logging and testing — and how to install them in Claude Code, Codex, GitHub Copilot, Cursor, Gemini CLI or Antigravity — are in [Agent skills](./docs/skills.md). --- diff --git a/docs/skills.md b/docs/skills.md index 42a9a81..3ab262e 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -1,5 +1,68 @@ # Agent skills -Agent skills for DragoAnt.System.Text.Json.Observer and DragoAnt.System.Text.Json.Observer.Http are being written: plain-Markdown instructions an AI coding agent loads before it writes masking rules, body logging or tests with these libraries. +The [`skills/`](../skills) folder holds three [Agent Skills](https://agentskills.io) that teach an AI coding agent to use these packages correctly — the rules, the defaults that surprise people, and runnable examples. Each skill is a folder with a `SKILL.md` (a `name` and a `description` in its frontmatter, then plain Markdown) and companion files the agent opens when it needs them. They use no agent-specific features, so any agent that reads `SKILL.md` files can use them. -Until they land, point your agent at the [README](../README.md) and the [changelog](../CHANGELOG.md). +| Skill | Use it for | +| --- | --- | +| [`json-observer-masking`](../skills/json-observer-masking/SKILL.md) | writing masking or extraction rules with `DragoAnt.System.Text.Json.Observer`, choosing a default policy, shapes from DTOs, cut-off bodies, the UTF-8 hot path, upgrading from 1.x | +| [`json-observer-http-logging`](../skills/json-observer-http-logging/SKILL.md) | logging masked `HttpClient` bodies with `DragoAnt.System.Text.Json.Observer.Http` — registration, masker providers, sinks, body markers | +| [`json-observer-testing`](../skills/json-observer-testing/SKILL.md) | xUnit tests proving the masking never leaks: golden, secret-absent, fuzz, allocation, concurrency and HTTP logging tests | + +Every C# block in the skills compiles and runs in this repository's test suite, against the source of the version they ship with. + +## Install + +Copy (or symlink) the skill folders into the directory your agent reads. `` is a folder from `skills/`; copy all three unless you want fewer. + +**`.agents/skills/` in your repository is read by Codex, GitHub Copilot, Cursor, Gemini CLI and Antigravity**; Claude Code reads `.claude/skills/`. Commit the folder to share the skills with your team, or use the personal folder to have them in every project. + +| Agent | Project folder | Personal folder | +| --- | --- | --- | +| Claude Code | `.claude/skills//` | `~/.claude/skills//` | +| OpenAI Codex | `.agents/skills//` (any folder from the working directory up to the repository root) | `~/.agents/skills//` | +| GitHub Copilot (CLI, VS Code, coding agent) | `.github/skills//`, `.agents/skills//` or `.claude/skills//` | `~/.copilot/skills//` or `~/.agents/skills//` | +| Cursor | `.agents/skills//` or `.cursor/skills//` (also reads `.claude/skills/` and `.codex/skills/`) | `~/.agents/skills//` or `~/.cursor/skills//` | +| Gemini CLI | `.gemini/skills//` or `.agents/skills//` | `~/.gemini/skills//` or `~/.agents/skills//` | +| Antigravity | `.agents/skills//` | `~/.gemini/config/skills//` (Antigravity CLI: `~/.gemini/antigravity-cli/skills//`) | + +Paths were checked against each agent's documentation on 2026-10-03; agents add locations over time, so check yours if a skill does not show up. + +### Copy the skills into a repository + +From the root of your repository (shell): + +```sh +git clone --depth 1 --filter=blob:none --sparse https://github.com/DragoAnt/Extensions.System.Text.Json.git .skills-src +git -C .skills-src sparse-checkout set skills +mkdir -p .agents/skills +cp -R .skills-src/skills/. .agents/skills/ +rm -rf .skills-src +``` + +PowerShell: + +```powershell +git clone --depth 1 --filter=blob:none --sparse https://github.com/DragoAnt/Extensions.System.Text.Json.git .skills-src +git -C .skills-src sparse-checkout set skills +New-Item -ItemType Directory -Force .agents/skills | Out-Null +Copy-Item -Recurse -Force .skills-src/skills/* .agents/skills/ +Remove-Item -Recurse -Force .skills-src +``` + +For Claude Code, use `.claude/skills` instead of `.agents/skills`. Gemini CLI can also install from a Git repository with `gemini skills install`, and link a local folder with `/skills link `. + +### An agent without skill support + +Paste the `SKILL.md` into the conversation, or add one line to the instructions file the agent reads (`AGENTS.md`, `CLAUDE.md`, `GEMINI.md`, `.github/copilot-instructions.md`): + +```text +Before writing code with DragoAnt.System.Text.Json.Observer, read .agents/skills/json-observer-masking/SKILL.md and follow its links when needed. +``` + +## Update + +The skills change with the library. Re-run the copy above after upgrading the package, and pin the source to the version you use by cloning its release tag: add `--branch v` to the `git clone` line. If you symlinked a clone instead of copying, `git pull` (or `git checkout v`) in that clone updates every project that links to it. + +## Contributing + +Skills live in `skills//`. Keep `SKILL.md` short (when to use it, the decision path, the rules that matter, pitfalls) and put detail in companion files linked with `./`. Every `csharp` block must be a complete program whose `// Output:` comment is what it prints, or a complete xUnit test class that passes; mark a deliberate fragment with `` on the line above it. `DragoAnt.System.Text.Json.Observer.Skills.Tests` compiles and runs them all, and checks the frontmatter and the relative links.