From 063cf95b7f96003b097d64d471892f4f5c56e6e9 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:31:25 +0200 Subject: [PATCH 01/13] Apply Obj rules nested in a property's Array An array rule returned depth + 1 and the matcher added the depth again, so an Obj under a property's Array looked for its names one or more levels too deep: BlockList left those values in clear and AllowList masked the whole item. An Obj directly under a root Array worked only because the root depth is 0. --- .../NestedArrayRuleTests.cs | 83 +++++++++++++++++++ .../RunNestedArrayRuleTests.cs | 3 + .../Builders/JsonArrayBuilder.cs | 2 +- 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/NestedArrayRuleTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunNestedArrayRuleTests.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestedArrayRuleTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestedArrayRuleTests.cs new file mode 100644 index 0000000..cae9888 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/NestedArrayRuleTests.cs @@ -0,0 +1,83 @@ +using DragoAnt.System.Text.Json.Observer.Builders; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class NestedArrayRuleTests +{ + private static readonly PropMatchingStrategy AnyItem = new(_ => true); + + private static void Line(JsonObjBuilder line, bool allowList) + { + if (allowList) + { + line.Match("sku").Unmasked(); + } + else + { + line.Match("qty").MaskAny("***"); + } + } + + private static JsonObserverValueDelegate Policy(bool allowList) => allowList ? AllowList : BlockList; + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ObjInPropertyArray_OneLevel_RulesApply(bool allowList) + { + var observer = JsonObserver.Obj(root => root.Match("lines").Array(l => l.Obj(x => Line(x, allowList))), Policy(allowList)); + + observer.Mask("""{"lines":[{"qty":5,"sku":"A"},{"qty":7,"sku":"B"}]}""") + .Should().Be("""{"lines":[{"qty":"***","sku":"A"},{"qty":"***","sku":"B"}]}"""); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ObjInArrayInPropertyArray_TwoLevels_RulesApply(bool allowList) + { + var observer = JsonObserver.Obj(root => root.Match("m").Array(l => l.Array(a => a.Obj(x => Line(x, allowList)))), Policy(allowList)); + + observer.Mask("""{"m":[[{"qty":5,"sku":"A"}],[{"qty":6,"sku":"B"}]]}""") + .Should().Be("""{"m":[[{"qty":"***","sku":"A"}],[{"qty":"***","sku":"B"}]]}"""); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ObjInArrayInObjInArray_ThreeLevels_RulesApply(bool allowList) + { + var observer = JsonObserver.Obj( + root => root.Match("a").Obj(a => a + .Match("orders").Array(o => o.Obj(order => order + .Match("lines").Array(l => l.Array(q => q.Obj(x => Line(x, allowList))))))), + Policy(allowList)); + + observer.Mask("""{"a":{"orders":[{"lines":[[{"qty":5,"sku":"A"}]]},{"lines":[[{"qty":6,"sku":"B"}]]}]}}""") + .Should().Be("""{"a":{"orders":[{"lines":[[{"qty":"***","sku":"A"}]]},{"lines":[[{"qty":"***","sku":"B"}]]}]}}"""); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ObjInRootArray_Unchanged(bool allowList) + { + var observer = JsonObserver.Array(root => root.Obj(x => x.Match("lines").Array(l => l.Obj(y => Line(y, allowList)))), Policy(allowList)); + + observer.Mask("""[{"lines":[{"qty":5,"sku":"A"}]}]""") + .Should().Be("""[{"lines":[{"qty":"***","sku":"A"}]}]"""); + } + + [Fact] + public void AnyItemWorkaround_StillMatches() + { + var byPath = JsonObserver.Obj(root => root.Match("lines", AnyItem, "qty").MaskAny("***"), BlockList); + var byRelative = JsonObserver.Obj(Relative(rules => rules.Match("lines", AnyItem, "qty").MaskAny("***"), BlockList)); + const string json = """{"lines":[{"qty":5,"sku":"A"}]}"""; + + byPath.Mask(json).Should().Be("""{"lines":[{"qty":"***","sku":"A"}]}"""); + byRelative.Mask(json).Should().Be("""{"lines":[{"qty":"***","sku":"A"}]}"""); + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunNestedArrayRuleTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunNestedArrayRuleTests.cs new file mode 100644 index 0000000..f562ee9 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunNestedArrayRuleTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunNestedArrayRuleTests : Shared.NestedArrayRuleTests; diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs index b2e93d4..2b98368 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs @@ -128,7 +128,7 @@ public JsonArrayBuilder MaskValue(JsonObserverDelegate polic private JsonArrayBuilder Add(Func typeMatch, JsonObserverDelegate policy) { - _policies.Add(new JsonObserverItem((int depth, ref PropertyPath _, JsonTokenType type) => (typeMatch(type), depth + 1), policy)); + _policies.Add(new JsonObserverItem((int _, ref PropertyPath _, JsonTokenType type) => (typeMatch(type), 1), policy)); return this; } } From 2abdf8a27849a7f847453d667e61d5a84530ab88 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:32:32 +0200 Subject: [PATCH 02/13] Call relative rules for null values like absolute ones The default-policy branch wrote a null directly, so a relative MaskStr, MaskInt, MaskBool, MaskRawValue or ReadStr never saw a null while the same absolute rule did. A null now reaches the relative rules; MaskAny and MaskTag keep it null as documented. --- .../RelativeNullTests.cs | 58 +++++++++++++++++++ .../RunRelativeNullTests.cs | 3 + .../JsonObserverItem.cs | 3 + 3 files changed, 64 insertions(+) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/RelativeNullTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunRelativeNullTests.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RelativeNullTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RelativeNullTests.cs new file mode 100644 index 0000000..5034b29 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RelativeNullTests.cs @@ -0,0 +1,58 @@ +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class RelativeNullTests +{ + private static string? Mark(string? value, JsonObserveringEmptyContext _) => value is null ? "was-null" : "x"; + + [Fact] + public void MaskStr_Null_CalledForRelativeLikeAbsolute() + { + var absolute = JsonObserver.Obj(root => root.Match("a").Obj(a => a.Match("p").MaskStr(Mark)), BlockList); + var relative = JsonObserver.Obj(Relative(rules => rules.Match("p").MaskStr(Mark), BlockList)); + const string json = """{"a":{"p":null,"q":null}}"""; + + absolute.Mask(json).Should().Be("""{"a":{"p":"was-null","q":null}}"""); + relative.Mask(json).Should().Be(absolute.Mask(json)); + } + + [Fact] + public void TypedMask_Null_CalledForRelative() => + JsonObserver.Obj(Relative(rules => rules + .Match("i").MaskInt((v, _) => v is null ? "i-null" : "i") + .Match("b").MaskBool((v, _) => v is null ? "b-null" : "b") + .Match("r").MaskRawValue((v, _) => v is null ? "r-null" : "r"), + BlockList)) + .Mask("""{"x":{"i":null,"b":null,"r":null}}""") + .Should().Be("""{"x":{"i":"i-null","b":"b-null","r":"r-null"}}"""); + + [Fact] + public void MaskAny_Null_StaysNullForRelative() => + JsonObserver.Obj(Relative(rules => rules.Match("p").MaskAny((_, _) => "called"), BlockList)) + .Mask("""{"p":null,"o":{"p":null}}""") + .Should().Be("""{"p":null,"o":{"p":null}}"""); + + [Fact] + public void ReadStr_Null_ReadForRelative() + { + var context = new Holder(); + var observer = JsonObserver.Obj(JsonObserverValuePolicies.Relative( + rules => rules.Match("p").ReadStr((v, c) => c.Calls.Add(v ?? "")), + JsonObserverValuePolicies.BlockList)); + + observer.Mask("""{"p":null,"o":{"p":"v"}}""", context).Should().Be("""{"p":null,"o":{"p":"v"}}"""); + context.Calls.Should().Equal("", "v"); + } + + [Fact] + public void Null_NoRelativeRule_KeptByDefaultPolicy() => + JsonObserver.Obj(Relative(rules => rules.Match("p").MaskStr(Mark), AllowList)) + .Mask("""{"q":null,"s":"x"}""") + .Should().Be("""{"q":null,"s":"***"}"""); + + public sealed class Holder + { + public List Calls { get; } = []; + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunRelativeNullTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunRelativeNullTests.cs new file mode 100644 index 0000000..e7ef77a --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunRelativeNullTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunRelativeNullTests : Shared.RelativeNullTests; diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs index d286c85..d4f7469 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs @@ -585,6 +585,9 @@ private static JsonObserverDelegate GetApplyDefaultPolicy(JsonObserver case False: effective(ref reader, writer, context, ref propPath); break; + case Null when effective.Target is RelativeValuePolicy: + effective(ref reader, writer, context, ref propPath); + break; case Null: writer.WriteNullValue(); break; From fad5c248f04dbda40f02fafa12ff8954c8ea2de8 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:34:42 +0200 Subject: [PATCH 03/13] Add span and Base64 overloads to JsonWriter WriteStringValue(ReadOnlySpan), WritePropertyName(ReadOnlySpan), WriteBase64StringValue and WriteNumberValue(double) let a char-based redactor or a hash strategy write without an intermediate string. MaxValueBytes cuts them like any string; a non-finite double is written as a string so a strategy can never fail the call. --- .../JsonWriterSpanTests.cs | 73 +++++++++++++++ .../RunJsonWriterSpanTests.cs | 3 + .../BoundedJsonWriter.cs | 68 +++++++++++++- .../JsonWriter.cs | 91 +++++++++++++++++-- 4 files changed, 225 insertions(+), 10 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonWriterSpanTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunJsonWriterSpanTests.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonWriterSpanTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonWriterSpanTests.cs new file mode 100644 index 0000000..d2e43c8 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonWriterSpanTests.cs @@ -0,0 +1,73 @@ +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class JsonWriterSpanTests +{ + private static JsonObserver Writing(JsonObserverValueDelegate rule) => + JsonObserver.Obj(b => b.Match("a").MaskValue(rule), BlockList); + + private static void EverySpanOverload(ref Utf8JsonReader reader, JsonWriter writer, JsonObserveringEmptyContext context, ref PropertyPath path) + { + writer.WriteStartObject(); + writer.WritePropertyName("chars".AsSpan()); + writer.WriteStringValue("é\"x".AsSpan()); + writer.WritePropertyName("b64".AsSpan()); + writer.WriteBase64StringValue([1, 2, 3, 250]); + writer.WritePropertyName("d".AsSpan()); + writer.WriteNumberValue(1.25d); + writer.WritePropertyName("nan".AsSpan()); + writer.WriteNumberValue(double.NaN); + writer.WritePropertyName("none".AsSpan()); + writer.WriteStringValue(ReadOnlySpan.Empty); + writer.WriteEndObject(); + } + + [Fact] + public void SpanOverloads_StringApi() => + Writing(EverySpanOverload).Mask("""{"a":0}""") + .Should().Be("""{"a":{"chars":"é\"x","b64":"AQID+g==","d":1.25,"nan":"NaN","none":""}}"""); + + [Fact] + public void SpanOverloads_BytesApiMatchesStringApi() + { + var (result, output) = BytesApiTests.Mask(Writing(EverySpanOverload), """{"a":0}"""); + + result.Status.Should().Be(MaskStatus.Masked); + output.Should().Be(Writing(EverySpanOverload).Mask("""{"a":0}""")); + } + + [Fact] + public void SpanOverloads_IgnoreNulls_WritesPendingNames() => + Writing(EverySpanOverload).Mask("""{"a":0}""", new JsonObserverOptions(IgnoreNulls: true)) + .Should().Be("""{"a":{"chars":"é\"x","b64":"AQID+g==","d":1.25,"nan":"NaN","none":""}}"""); + + [Fact] + public void CharSpan_LongerThanMaxValueBytes_Cut() + { + var observer = Writing((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath _) => + writer.WriteStringValue("abcdefgh".AsSpan())); + + var masked = observer.Mask("""{"a":0}""", out var result, new JsonObserverOptions(MaxValueBytes: 4)); + + masked.Should().Be("""{"a":"abcd…"}"""); + result.Status.Should().Be(MaskStatus.Truncated); + } + + [Fact] + public void Base64_LongerThanMaxValueBytes_Cut() + { + var observer = Writing((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath _) => + writer.WriteBase64StringValue([1, 2, 3, 4, 5, 6])); + + observer.Mask("""{"a":0}""", new JsonObserverOptions(MaxValueBytes: 4)).Should().Be("""{"a":"AQID…"}"""); + } + + [Fact] + public void SpanOverloads_ReadOnly_WriteNothing() + { + var observer = JsonObserver.Obj(b => b.Match("a").MaskValue(EverySpanOverload), JsonObserverValuePolicies.BlockList); + + observer.Read("""{"a":0}""", JsonObserveringEmptyContext.Instance).Status.Should().Be(MaskStatus.Masked); + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunJsonWriterSpanTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunJsonWriterSpanTests.cs new file mode 100644 index 0000000..609cf55 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunJsonWriterSpanTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunJsonWriterSpanTests : Shared.JsonWriterSpanTests; diff --git a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs index 2ab63f0..abf4642 100644 --- a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs @@ -1,4 +1,6 @@ using System.Buffers; +using System.Buffers.Text; +using System.Globalization; using System.Text; using System.Text.Encodings.Web; @@ -83,6 +85,16 @@ public override void WriteStringValue(string? value) return; } + WriteStringValue(value.AsSpan()); + } + + public override void WriteStringValue(ReadOnlySpan value) + { + if (Exhausted) + { + return; + } + if ((long)value.Length * 3 <= _maxValueBytes) { _writer.WriteStringValue(value); @@ -90,7 +102,7 @@ public override void WriteStringValue(string? value) return; } - var chars = value.AsSpan(0, (int)Math.Min(value.Length, (long)_maxValueBytes + 1)); + var chars = value[..(int)Math.Min(value.Length, (long)_maxValueBytes + 1)]; if (chars.Length < value.Length && char.IsHighSurrogate(chars[^1])) { chars = chars[..^1]; @@ -189,6 +201,52 @@ public override void WriteNumberValue(decimal value) Completed(); } + public override void WriteNumberValue(double value) + { + if (Exhausted) + { + return; + } + + if (!double.IsFinite(value)) + { + Span text = stackalloc char[16]; + value.TryFormat(text, out var length, provider: CultureInfo.InvariantCulture); + WriteStringValue(text[..length]); + return; + } + + _writer.WriteNumberValue(value); + Completed(); + } + + public override void WriteBase64StringValue(ReadOnlySpan bytes) + { + if (Exhausted) + { + return; + } + + var length = Base64.GetMaxEncodedToUtf8Length(bytes.Length); + if (length <= _maxValueBytes) + { + _writer.WriteBase64StringValue(bytes); + Completed(); + return; + } + + var encoded = ArrayPool.Shared.Rent(length); + try + { + Base64.EncodeToUtf8(bytes, encoded, out _, out var written); + WriteStringValue(encoded.AsSpan(0, written)); + } + finally + { + ArrayPool.Shared.Return(encoded, clearArray: true); + } + } + public override void WritePropertyName(string propertyName) { if (!Exhausted) @@ -197,6 +255,14 @@ public override void WritePropertyName(string propertyName) } } + public override void WritePropertyName(ReadOnlySpan propertyName) + { + if (!Exhausted) + { + _writer.WritePropertyName(propertyName); + } + } + public override void WritePropertyName(ReadOnlySpan utf8PropertyName) { if (!Exhausted) diff --git a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs index 879740d..9aae3bd 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs @@ -89,6 +89,31 @@ private protected JsonWriter() /// UTF-8 JSON value. public abstract void WriteRawValue(ReadOnlySpan utf8Json); + /// + /// Writes a string value given as UTF-16 text, for example the output of a char-based redactor, without a + /// allocation. + /// + /// Unescaped text; an empty span writes "". + public abstract void WriteStringValue(ReadOnlySpan value); + + /// + /// Writes a property name given as UTF-16 text; the value written next belongs to it. + /// + /// Unescaped name. + public abstract void WritePropertyName(ReadOnlySpan propertyName); + + /// + /// Writes bytes as a Base64 string value, for example a hash or an encrypted value. + /// + /// Bytes to encode. + public abstract void WriteBase64StringValue(ReadOnlySpan bytes); + + /// + /// Writes a number; and infinities, which JSON cannot represent, are written as strings. + /// + /// Value to write. + public abstract void WriteNumberValue(double value); + /// /// Options of the current call; rules with a read their strategy and hash key here. /// @@ -185,6 +210,22 @@ public override void WriteRawValue(ReadOnlySpan utf8Json) { } + public override void WriteStringValue(ReadOnlySpan value) + { + } + + public override void WritePropertyName(ReadOnlySpan propertyName) + { + } + + public override void WriteBase64StringValue(ReadOnlySpan bytes) + { + } + + public override void WriteNumberValue(double value) + { + } + public override void WriteStartObject() { } @@ -272,23 +313,55 @@ public override void WriteNumberValue(decimal value) inner.WriteNumberValue(value); } - public override void WritePropertyName(string propertyName) => WritePropertyName(Encoding.UTF8.GetBytes(propertyName)); + public override void WriteStringValue(ReadOnlySpan value) + { + Flush(); + inner.WriteStringValue(value); + } - public override void WritePropertyName(ReadOnlySpan utf8PropertyName) + public override void WriteBase64StringValue(ReadOnlySpan bytes) { - if (_namesUsed + utf8PropertyName.Length > _names.Length) - { - var grown = ArrayPool.Shared.Rent(Math.Max(_names.Length * 2, _namesUsed + utf8PropertyName.Length)); - _names.AsSpan(0, _namesUsed).CopyTo(grown); - ArrayPool.Shared.Return(_names, clearArray: true); - _names = grown; - } + Flush(); + inner.WriteBase64StringValue(bytes); + } + + public override void WriteNumberValue(double value) + { + Flush(); + inner.WriteNumberValue(value); + } + + public override void WritePropertyName(string propertyName) => WritePropertyName(propertyName.AsSpan()); + public override void WritePropertyName(ReadOnlySpan propertyName) + { + EnsureNames(Encoding.UTF8.GetMaxByteCount(propertyName.Length)); + var written = Encoding.UTF8.GetBytes(propertyName, _names.AsSpan(_namesUsed)); + Push(new Pending(Kind.Name, _namesUsed, written, false)); + _namesUsed += written; + } + + public override void WritePropertyName(ReadOnlySpan utf8PropertyName) + { + EnsureNames(utf8PropertyName.Length); utf8PropertyName.CopyTo(_names.AsSpan(_namesUsed)); Push(new Pending(Kind.Name, _namesUsed, utf8PropertyName.Length, false)); _namesUsed += utf8PropertyName.Length; } + private void EnsureNames(int length) + { + if (_namesUsed + length <= _names.Length) + { + return; + } + + var grown = ArrayPool.Shared.Rent(Math.Max(_names.Length * 2, _namesUsed + length)); + _names.AsSpan(0, _namesUsed).CopyTo(grown); + ArrayPool.Shared.Return(_names, clearArray: true); + _names = grown; + } + public override void WriteStartObject() => Push(new Pending(Kind.Open, 0, 0, false)); public override void WriteStartArray() => Push(new Pending(Kind.Open, 0, 0, true)); From d49f1d065617902dbc864b0a9b79a81e155a28d4 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:35:54 +0200 Subject: [PATCH 04/13] Let a MaskTag carry a classification key MaskTag gains an optional object Key and MaskKind.Custom, so an integration maps its own classification (a compliance taxonomy, a redactor name) to masking without casting MaskKind values. A strategy that does not know the key falls back to the kind; the built-in strategy writes "***" for Custom. --- .../MaskTagTests.cs | 70 +++++++++++++++++++ .../Strategies/MaskTag.cs | 47 ++++++++++++- 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs index a80d591..93242f0 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs @@ -83,6 +83,76 @@ public void Tag_PassedToCustomStrategy() "Omit True true"); } + private sealed record Classification(string Taxonomy, string Name); + + private static readonly Classification Pii = new("Demo", "Pii"); + + private static readonly JsonObserver KeyedObserver = JsonObserver.Obj(Relative(b => b + .Match("email").MaskAny(MaskTag.Custom(Pii)) + .Match("token").MaskAny(new MaskTag(MaskKind.Hash, "secret")) + .Match("plain").MaskAny(MaskTag.Last4), + BlockList)); + + [Fact] + public void CustomKey_ReachesStrategyWithoutCasts() + { + var strategy = new KeyedStrategy(); + + var (_, output) = BytesApiTests.Mask( + KeyedObserver, + """{"email":"a@b.c","token":"t0k3n","plain":"12345678"}""", + new JsonObserverOptions(MaskStrategy: strategy)); + + output.Should().Be("""{"email":"Pii","token":"secret","plain":"none"}"""); + } + + [Fact] + public void CustomKey_DefaultStrategy_FallsBackToKind() + { + var root = Mask("""{"full":"x"}"""); + root.GetProperty("full").GetString().Should().Be("***"); + + var masked = KeyedObserver.Mask("""{"email":"a@b.c","token":"t0k3n","plain":"12345678"}"""); + + masked.Should().StartWith("{\"email\":\"***\",\"token\":\"hash:").And.EndWith("\",\"plain\":\"***5678\"}"); + } + + [Fact] + public void MaskTag_KeyEqualityAndAccessors() + { + var custom = MaskTag.Custom(Pii); + + custom.Kind.Should().Be(MaskKind.Custom); + custom.Should().Be(MaskTag.Custom(new Classification("Demo", "Pii"))); + custom.Should().NotBe(MaskTag.Custom(new Classification("Demo", "Secret"))); + custom.TryGetKey(out var key).Should().BeTrue(); + key.Should().Be(Pii); + custom.TryGetKey(out _).Should().BeFalse(); + ((MaskTag)MaskKind.Last4).Should().Be(MaskTag.Last4).And.Be(new MaskTag(MaskKind.Last4, null)); + MaskTag.Last4.Key.Should().BeNull(); + var build = () => MaskTag.Custom(null!); + build.Should().Throw(); + } + + private sealed class KeyedStrategy : Utf8MaskStrategy + { + public override void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) + { + switch (tag.Key) + { + case Classification classification: + writer.WriteStringValue(classification.Name); + break; + case string name: + writer.WriteStringValue(name); + break; + default: + writer.WriteStringValue("none"); + break; + } + } + } + private sealed class RecordingStrategy : Utf8MaskStrategy { public List Calls { get; } = []; diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/MaskTag.cs b/DragoAnt.System.Text.Json.Observer/Strategies/MaskTag.cs index 56e2960..e5337d3 100644 --- a/DragoAnt.System.Text.Json.Observer/Strategies/MaskTag.cs +++ b/DragoAnt.System.Text.Json.Observer/Strategies/MaskTag.cs @@ -24,14 +24,57 @@ public enum MaskKind /// Replaced by null. /// Omit, + + /// + /// Masked the way a custom decides from ; + /// the built-in strategy replaces it by "***". + /// + Custom, } /// /// Tag a rule passes to the , so that one strategy serves every kind of masking. /// -/// How the value is masked. -public readonly record struct MaskTag(MaskKind Kind) +/// +/// How the value is masked. A strategy that does not know falls back to it, so a tag such as +/// new MaskTag(MaskKind.Hash, classification) is still hashed by the built-in strategy. +/// +/// +/// Optional classification of the value, for example a data classification of a compliance taxonomy or a redactor +/// name, that a custom strategy maps to its own masking. It is compared with , +/// so prefer immutable keys with value equality. +/// +public readonly record struct MaskTag(MaskKind Kind, object? Key = null) { + /// + /// A tag only a custom strategy interprets, by ; the built-in strategy writes "***". + /// + /// Classification the strategy maps to its masking. + /// is null. + public static MaskTag Custom(object key) + { + ArgumentNullException.ThrowIfNull(key); + return new MaskTag(MaskKind.Custom, key); + } + + /// + /// Gets when it is a . + /// + /// The key; default when it is absent or of another type. + /// Expected key type. + /// true when the key is a . + public bool TryGetKey(out T? key) + { + if (Key is T typed) + { + key = typed; + return true; + } + + key = default; + return false; + } + /// /// . /// From 9a2b0b3bbaa7e3e4b50bd9c1a57ed1b6b6cc0654 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:38:31 +0200 Subject: [PATCH 05/13] Keep array indices in PropertyPath An array item level now stores its position: ToString renders items[2].sku (special names as ['a.b']), and TryGetArrayIndex, IsArrayItem and TryGetPropertyNameUtf8 give zero-allocation access for error paths and strategies. Shape observers now push array items too, so both walkers report the same paths. Name matching is unchanged: an array item is still one level that names never match. --- .../PropertyPathTests.cs | 75 +++++++++++++ .../RuleCoverageTests.cs | 2 +- .../RunPropertyPathTests.cs | 3 + .../JsonObserverItem.cs | 3 +- .../PropertyPath.cs | 101 ++++++++++++++---- .../ShapeWalker.cs | 3 + .../Strategies/NameMatcher.cs | 4 +- 7 files changed, 165 insertions(+), 26 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/PropertyPathTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunPropertyPathTests.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/PropertyPathTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/PropertyPathTests.cs new file mode 100644 index 0000000..ab79e2e --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/PropertyPathTests.cs @@ -0,0 +1,75 @@ +using System.Text; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class PropertyPathTests +{ + private static List Collect(string json, Func, JsonObserver> build) + { + var seen = new List(); + var observer = build((ref Utf8JsonReader reader, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath path) => + { + seen.Add(path.ToString()); + writer.WriteStringValue("x"); + }); + observer.Mask(json); + return seen; + } + + [Fact] + public void ToString_KeepsArrayIndices() + { + var seen = Collect( + """{"items":[{"sku":"a"},{"sku":"b"},{"sku":"c","tags":["x","y"]}],"m":[[1,2],[3]]}""", + rule => JsonObserver.Obj(Relative(b => b.Match(new PropMatchingStrategy(_ => true)).MaskValue(rule), BlockList))); + + seen.Should().Equal( + "items[0].sku", "items[1].sku", "items[2].sku", "items[2].tags[0]", "items[2].tags[1]", + "m[0][0]", "m[0][1]", "m[1][0]"); + } + + [Fact] + public void ToString_RootArrayAndSpecialNames() + { + var seen = Collect( + """[{"a.b":1,"it's":2,"":3,"x[1]":4,"é":5}]""", + rule => JsonObserver.Array(root => root.Obj(o => o.Match(new PropMatchingStrategy(_ => true)).MaskValue(rule)), BlockList)); + + seen.Should().Equal("[0]['a.b']", "[0]['it\\'s']", "[0]['']", "[0]['x[1]']", "[0].é"); + } + + [Fact] + public void Accessors_IndexAndUtf8Name() + { + var seen = new List(); + var observer = JsonObserver.Obj(Relative(b => b.Match("sku").MaskValue((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath path) => + { + path.TryGetArrayIndex(1, out var index).Should().BeTrue(); + path.TryGetArrayIndex(0, out _).Should().BeFalse(); + path.TryGetArrayIndex(7, out _).Should().BeFalse(); + path.IsArrayItem(1).Should().BeTrue(); + path.IsArrayItem(2).Should().BeFalse(); + path.TryGetPropertyNameUtf8(2, out var name).Should().BeTrue(); + path.TryGetPropertyNameUtf8(1, out _).Should().BeFalse(); + path.TryGetPropertyNameUtf8(-1, out _).Should().BeFalse(); + path.GetPropertyName(1).Should().BeNull(); + seen.Add($"{index}:{Encoding.UTF8.GetString(name)}"); + writer.WriteStringValue("x"); + }), BlockList)); + + observer.Mask("""{"items":[{"sku":"a"},{"sku":"b"}]}""").Should().Be("""{"items":[{"sku":"x"},{"sku":"x"}]}"""); + seen.Should().Equal("0:sku", "1:sku"); + } + + [Fact] + public void Matching_ArrayItemSegment_StillOneLevel() + { + var anyItem = new PropMatchingStrategy(n => n is null); + + JsonObserver.Obj(root => root.Match("lines", anyItem, "qty").MaskAny("***"), BlockList) + .Mask("""{"lines":[{"qty":1},{"qty":2}],"qty":3}""") + .Should().Be("""{"lines":[{"qty":"***"},{"qty":"***"}],"qty":3}"""); + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RuleCoverageTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RuleCoverageTests.cs index 5b7b5aa..af3ea31 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/RuleCoverageTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/RuleCoverageTests.cs @@ -79,7 +79,7 @@ public void CustomDelegate_PropertyPathApi() }), BlockList)); observer.Mask("""{"a":{"b":[{"c":1}]}}""").Should().Be("""{"a":{"b":[{"c":"x"}]}}"""); - seen.Should().Equal("4|a|c||a.b..c|"); + seen.Should().Equal("4|a|c||a.b[0].c|"); } [Fact] diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunPropertyPathTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunPropertyPathTests.cs new file mode 100644 index 0000000..43d7416 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunPropertyPathTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunPropertyPathTests : Shared.PropertyPathTests; diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs index d4f7469..a56e897 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs @@ -505,6 +505,7 @@ private static JsonObserverDelegate ApplyArrayPolicy( RuntimeHelpers.EnsureSufficientExecutionStack(); writer.WriteStartArray(); + var index = 0; while (true) { if (propPath.Stopped || writer.Stopped || !reader.Read()) @@ -524,7 +525,7 @@ private static JsonObserverDelegate ApplyArrayPolicy( case Null: var tokenType = reader.TokenType; - propPath.AddPropertyName(null); + propPath.AddArrayItem(index++); var (matchPolicy, nextDepth) = MatchPolicy(policies, depth, ref propPath, tokenType); if (matchPolicy is not null) diff --git a/DragoAnt.System.Text.Json.Observer/PropertyPath.cs b/DragoAnt.System.Text.Json.Observer/PropertyPath.cs index 553668e..5b935fe 100644 --- a/DragoAnt.System.Text.Json.Observer/PropertyPath.cs +++ b/DragoAnt.System.Text.Json.Observer/PropertyPath.cs @@ -7,7 +7,8 @@ namespace DragoAnt.System.Text.Json.Observer; /// Path of the value a rule is called for: one level per enclosing property or array item, from the root down. /// /// -/// Valid only during the call it is passed to. Names are kept as UTF-8 and decoded only when asked for. +/// Valid only during the call it is passed to. Names are kept as UTF-8 and decoded only when asked for; an array item +/// keeps its index, so renders items[2].sku. /// public ref struct PropertyPath { @@ -66,14 +67,24 @@ internal void AddPropertyName(ref Utf8JsonReader reader) } /// - /// Add property name considering depth. + /// Adds an unescaped UTF-8 property name. /// - internal void AddPropertyName(string? name) + internal void AddPropertyName(ReadOnlySpan utf8Name) { ref var segment = ref Push(); - segment = name is null - ? new Segment(SegmentKind.ArrayItem, 0, 0) - : new Segment(SegmentKind.Text, 0, 0) { Decoded = name }; + var start = Reserve(utf8Name.Length); + utf8Name.CopyTo(_scratch.AsSpan(start)); + _scratchUsed = start + utf8Name.Length; + segment = new Segment(SegmentKind.Scratch, start, utf8Name.Length); + } + + /// + /// Adds the array item at . + /// + internal void AddArrayItem(int index) + { + ref var segment = ref Push(); + segment = new Segment(SegmentKind.ArrayItem, index, 0); } internal void RemovePropertyName() @@ -94,13 +105,16 @@ internal void RemovePropertyName() } /// - /// UTF-8 name of the level at ; false for an array item or a level out of range. + /// Gets the unescaped UTF-8 name of a level without decoding it. /// - internal readonly bool TryGetUtf8(int index, out ReadOnlySpan name) + /// Level, from 0 to - 1. + /// The name; valid only during the call. + /// false for an array item or an index out of range. + public readonly bool TryGetPropertyNameUtf8(int index, out ReadOnlySpan utf8Name) { if (index < 0 || index > Depth) { - name = default; + utf8Name = default; return false; } @@ -108,24 +122,45 @@ internal readonly bool TryGetUtf8(int index, out ReadOnlySpan name) switch (segment.Kind) { case SegmentKind.Input: - name = _input.Slice(segment.Start, segment.Length); + utf8Name = _input.Slice(segment.Start, segment.Length); return true; case SegmentKind.Scratch: - name = _scratch.AsSpan(segment.Start, segment.Length); - return true; - case SegmentKind.Text: - name = Encoding.UTF8.GetBytes(segment.Decoded!); + utf8Name = _scratch.AsSpan(segment.Start, segment.Length); return true; default: - name = default; + utf8Name = default; return false; } } + /// + /// Whether the level at is an array item. + /// + /// Level, from 0 to - 1. + public readonly bool IsArrayItem(int index) => index >= 0 && index <= Depth && _segments[index].Kind == SegmentKind.ArrayItem; + + /// + /// Gets the zero-based position of an array item level within its array. + /// + /// Level, from 0 to - 1. + /// Position of the item; -1 when the level is not an array item. + /// true when the level is an array item. + public readonly bool TryGetArrayIndex(int index, out int arrayIndex) + { + if (IsArrayItem(index)) + { + arrayIndex = _segments[index].Start; + return true; + } + + arrayIndex = -1; + return false; + } + /// /// UTF-8 name of the deepest level. /// - internal readonly ReadOnlySpan CurrentUtf8 => TryGetUtf8(Depth, out var name) ? name : default; + internal readonly ReadOnlySpan CurrentUtf8 => TryGetPropertyNameUtf8(Depth, out var name) ? name : default; /// /// Name of the level at , 0 being the root's property. @@ -145,7 +180,7 @@ internal readonly bool TryGetUtf8(int index, out ReadOnlySpan name) return segment.Decoded; } - TryGetUtf8(index, out var utf8); + TryGetPropertyNameUtf8(index, out var utf8); return segment.Decoded = Encoding.UTF8.GetString(utf8); } @@ -157,17 +192,40 @@ internal readonly bool TryGetUtf8(int index, out ReadOnlySpan name) public string? GetPropertyNameReverse(int reversedIndex) => GetPropertyName(Depth - reversedIndex); /// - /// The names from the root down, joined with dots; an array item is an empty segment. + /// The path from the root down: names joined with dots and array items as [index], for example + /// items[2].sku; a name that is empty or holds ., [, ] or ' is written as ['name']. /// public override string ToString() { - var names = new string?[Depth + 1]; + var text = new StringBuilder(); for (var i = 0; i <= Depth; i++) { - names[i] = GetPropertyName(i); + if (TryGetArrayIndex(i, out var arrayIndex)) + { + text.Append('[').Append(arrayIndex).Append(']'); + continue; + } + + AppendName(text, GetPropertyName(i)!, first: i == 0); + } + + return text.ToString(); + } + + internal static void AppendName(StringBuilder text, string name, bool first) + { + if (name.Length == 0 || name.AsSpan().IndexOfAny(".[]'") >= 0) + { + text.Append("['").Append(name.Replace("'", "\\'", StringComparison.Ordinal)).Append("']"); + return; + } + + if (!first) + { + text.Append('.'); } - return string.Join('.', names); + text.Append(name); } /// @@ -234,7 +292,6 @@ internal enum SegmentKind : byte ArrayItem, Input, Scratch, - Text, } internal struct Segment(SegmentKind kind, int start, int length) diff --git a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs index a3525f2..87f48a3 100644 --- a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs +++ b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs @@ -116,6 +116,7 @@ private void WriteArray(ref Utf8JsonReader reader, JsonWriter writer, ref Proper { RuntimeHelpers.EnsureSufficientExecutionStack(); writer.WriteStartArray(); + var index = 0; while (true) { if (propPath.Stopped || writer.Stopped || !reader.Read()) @@ -132,7 +133,9 @@ private void WriteArray(ref Utf8JsonReader reader, JsonWriter writer, ref Proper case Comment: break; default: + propPath.AddArrayItem(index++); Write(ref reader, writer, ref propPath, item); + propPath.RemovePropertyName(); break; } } diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs b/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs index 44f51a5..8b63428 100644 --- a/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs +++ b/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs @@ -57,7 +57,7 @@ private sealed class AsciiNameMatcher(AsciiNameMatcher.Mode mode, string pattern public override bool Match(ref PropertyPath path, int index) { - if (!path.TryGetUtf8(index, out var name)) + if (!path.TryGetPropertyNameUtf8(index, out var name)) { return false; } @@ -114,7 +114,7 @@ public OneOfNameMatcher(string[] names) public override bool Match(ref PropertyPath path, int index) { - if (!path.TryGetUtf8(index, out var name)) + if (!path.TryGetPropertyNameUtf8(index, out var name)) { return false; } From af8f01452054d71f1a613c9fc439e095210a85df Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:40:08 +0200 Subject: [PATCH 06/13] Pass the property name and path to the mask strategy Utf8MaskStrategy.Mask(in Utf8MaskContext, JsonWriter) is now the entry point: the context carries the value, its JSON type, the tag, the options, the property name and the whole path without allocating, so a strategy can apply a value:name discriminator. It forwards to the existing overload by default, which in turn masks like the built-in strategy, so neither has to be overridden. A number split across input segments is copied to a pooled buffer instead of an array. --- .../MaskTagTests.cs | 61 +++++++++++++++++++ .../Strategies/Utf8MaskContext.cs | 54 ++++++++++++++++ .../Strategies/Utf8MaskStrategy.cs | 22 ++++++- .../TagMasking.cs | 32 ++++++++-- 4 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskContext.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs index 93242f0..f627def 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/MaskTagTests.cs @@ -134,6 +134,67 @@ public void MaskTag_KeyEqualityAndAccessors() build.Should().Throw(); } + [Fact] + public void ContextStrategy_ReceivesPropertyNameAndPath() + { + var strategy = new DiscriminatingStrategy(); + var rules = JsonObserver.Obj( + root => root + .Match("user").Obj(u => u.Match("email").MaskAny(MaskTag.Custom(Pii))) + .Match("tags").Array(t => t.MaskAny(MaskTag.Hash)), + Relative(b => b.Match("phone").MaskAny(MaskTag.Last4), BlockList)); + var shape = JsonObserver.FromShape(JsonShape.Object( + ("cards", JsonShape.Array(JsonShape.Object(("number", JsonShape.Masked(MaskTag.Last4))))))); + var options = new JsonObserverOptions(MaskStrategy: strategy); + + rules.Mask("""{"user":{"email":"a@b.c"},"tags":["x"],"o":{"phone":"12"}}""", options) + .Should().Be("""{"user":{"email":"a@b.c:email"},"tags":["x:"],"o":{"phone":"12:phone"}}"""); + shape.Mask("""{"cards":[{"number":"4111"},{"number":"4222","cvv":1}]}""", options) + .Should().Be("""{"cards":[{"number":"4111:number"},{"number":"4222:number","cvv":"***"}]}"""); + strategy.Calls.Should().Equal( + "Custom user.email email", + "Hash tags[0] []", + "Last4 o.phone phone", + "Last4 cards[0].number number", + "Last4 cards[1].number number", + "Full cards[1].cvv cvv"); + } + + [Fact] + public void OldSignatureOnly_StillCalled() + { + var strategy = new RecordingStrategy(); + + Observer.Mask("""{"full":"a"}""", new JsonObserverOptions(MaskStrategy: strategy)).Should().Be("""{"full":"?"}"""); + strategy.Calls.Should().Equal("Full String a"); + } + + [Fact] + public void NoOverride_BehavesLikeDefault() => + Observer.Mask("""{"last4":"4111111111111111","omit":1}""", new JsonObserverOptions(MaskStrategy: new NoOverrideStrategy())) + .Should().Be("""{"last4":"***1111","omit":null}"""); + + private sealed class NoOverrideStrategy : Utf8MaskStrategy; + + private sealed class DiscriminatingStrategy : Utf8MaskStrategy + { + public List Calls { get; } = []; + + public override void Mask(in Utf8MaskContext context, JsonWriter writer) + { + var name = Encoding.UTF8.GetString(context.PropertyName); + Calls.Add($"{context.Tag.Kind} {context.Path.ToString()} {(context.IsArrayItem ? "[]" : name)}"); + if (context.TokenType is JsonTokenType.String) + { + writer.WriteStringValue($"{Encoding.UTF8.GetString(context.Value)}:{name}"); + } + else + { + Utf8MaskStrategy.Default.Mask(context, writer); + } + } + } + private sealed class KeyedStrategy : Utf8MaskStrategy { public override void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskContext.cs b/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskContext.cs new file mode 100644 index 0000000..0c7224e --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskContext.cs @@ -0,0 +1,54 @@ +namespace DragoAnt.System.Text.Json.Observer.Strategies; + +/// +/// What a knows about the value it masks. Valid only during the call it is passed to; +/// reading it allocates nothing. +/// +public readonly ref struct Utf8MaskContext +{ + private readonly PropertyPath _path; + + internal Utf8MaskContext(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonObserverOptions options, PropertyPath path) + { + Value = value; + TokenType = tokenType; + Tag = tag; + Options = options; + _path = path; + } + + /// + /// The unescaped text of a string, the literal of a number or boolean, or empty for null, an object or an array. + /// + public ReadOnlySpan Value { get; } + + /// + /// JSON type of the value; or for a container. + /// + public JsonTokenType TokenType { get; } + + /// + /// How the rule asks for the value to be masked. + /// + public MaskTag Tag { get; } + + /// + /// Options of the current call. + /// + public JsonObserverOptions Options { get; } + + /// + /// Path of the value from the root, array indices included. + /// + public PropertyPath Path => _path; + + /// + /// Unescaped UTF-8 name of the property that holds the value; empty for an array item. + /// + public ReadOnlySpan PropertyName => _path.TryGetPropertyNameUtf8(_path.Length - 1, out var name) ? name : default; + + /// + /// The value is an item of an array rather than the value of a property. + /// + public bool IsArrayItem => _path.IsArrayItem(_path.Length - 1); +} diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskStrategy.cs b/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskStrategy.cs index 794e5d6..5694f19 100644 --- a/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskStrategy.cs +++ b/DragoAnt.System.Text.Json.Observer/Strategies/Utf8MaskStrategy.cs @@ -5,16 +5,31 @@ namespace DragoAnt.System.Text.Json.Observer.Strategies; /// /// Masks a sensitive value given as UTF-8. One instance serves every rule: the rule's says how. /// +/// +/// Override to also see the property name and path of the value, +/// or the shorter overload when the value and the tag are enough. A strategy that overrides neither masks like +/// . +/// public abstract class Utf8MaskStrategy { /// /// Built-in strategy: , , - /// (HMAC-SHA256 with ) and . + /// (HMAC-SHA256 with ), ; anything else, + /// included, becomes "***". /// public static Utf8MaskStrategy Default { get; } = new DefaultUtf8MaskStrategy(); /// - /// Writes the masked replacement of one value. + /// Writes the masked replacement of one value, knowing where it is. This is the method the observer calls; by + /// default it forwards to . + /// + /// The value, its JSON type, the rule's tag, the call's options and the value's path. + /// Receives exactly one value. + public virtual void Mask(in Utf8MaskContext context, JsonWriter writer) => + Mask(context.Value, context.TokenType, context.Tag, writer, context.Options); + + /// + /// Writes the masked replacement of one value. By default it masks like . /// /// /// The unescaped text of a string, the literal of a number or boolean, or empty for an object or array. @@ -23,7 +38,8 @@ public abstract class Utf8MaskStrategy /// How the rule asks for the value to be masked. /// Receives exactly one value. /// Options of the current call. - public abstract void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options); + public virtual void Mask(ReadOnlySpan value, JsonTokenType tokenType, MaskTag tag, JsonWriter writer, JsonObserverOptions options) => + Default.Mask(value, tokenType, tag, writer, options); private sealed class DefaultUtf8MaskStrategy : Utf8MaskStrategy { diff --git a/DragoAnt.System.Text.Json.Observer/TagMasking.cs b/DragoAnt.System.Text.Json.Observer/TagMasking.cs index c78f1f2..b71f918 100644 --- a/DragoAnt.System.Text.Json.Observer/TagMasking.cs +++ b/DragoAnt.System.Text.Json.Observer/TagMasking.cs @@ -5,6 +5,8 @@ namespace DragoAnt.System.Text.Json.Observer; internal static class TagMasking { + private const int StackallocThreshold = 256; + /// /// Writes the strategy's replacement for the current value and moves past it; a container is never read. /// @@ -17,7 +19,7 @@ public static void Mask(ref Utf8JsonReader reader, JsonWriter writer, MaskTag ta { case JsonTokenType.StartObject: case JsonTokenType.StartArray: - strategy.Mask(default, tokenType, tag, writer, options); + strategy.Mask(new Utf8MaskContext(default, tokenType, tag, options, propPath), writer); if (!reader.TrySkip()) { propPath.Stop(); @@ -25,15 +27,16 @@ public static void Mask(ref Utf8JsonReader reader, JsonWriter writer, MaskTag ta return; case JsonTokenType.Null: - strategy.Mask(default, tokenType, tag, writer, options); + strategy.Mask(new Utf8MaskContext(default, tokenType, tag, options, propPath), writer); return; case JsonTokenType.String when reader.HasValueSequence || reader.ValueIsEscaped: + { var length = reader.HasValueSequence ? checked((int)reader.ValueSequence.Length) : reader.ValueSpan.Length; var buffer = ArrayPool.Shared.Rent(length); try { var written = reader.CopyString(buffer); - strategy.Mask(buffer.AsSpan(0, written), tokenType, tag, writer, options); + strategy.Mask(new Utf8MaskContext(buffer.AsSpan(0, written), tokenType, tag, options, propPath), writer); } finally { @@ -41,8 +44,29 @@ public static void Mask(ref Utf8JsonReader reader, JsonWriter writer, MaskTag ta } return; + } + case not JsonTokenType.String when reader.HasValueSequence: + { + var length = checked((int)reader.ValueSequence.Length); + byte[]? rented = null; + var buffer = length <= StackallocThreshold ? stackalloc byte[StackallocThreshold] : rented = ArrayPool.Shared.Rent(length); + try + { + reader.ValueSequence.CopyTo(buffer); + strategy.Mask(new Utf8MaskContext(buffer[..length], tokenType, tag, options, propPath), writer); + } + finally + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented, clearArray: true); + } + } + + return; + } default: - strategy.Mask(reader.HasValueSequence ? reader.ValueSequence.ToArray() : reader.ValueSpan, tokenType, tag, writer, options); + strategy.Mask(new Utf8MaskContext(reader.ValueSpan, tokenType, tag, options, propPath), writer); return; } } From 7fa4dfead8ca5235d540f96618ed8855e4e512a9 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:48:35 +0200 Subject: [PATCH 07/13] Carry System.Text.Json metadata and annotations on JsonShape Members is now a list of JsonShapeProperty (still deconstructs to name and shape) with the JsonPropertyInfo, CLR member, property and declaring type, IsRequired, IsNullable and custom attributes; nodes built from metadata carry their JsonTypeInfo and CLR type. Both nodes and properties have thread-safe annotations for integrations, FromTypeInfo takes an annotate callback, and FindMember looks a property up by its UTF-8 name like the observer. .NET 8 source-generated metadata has no attributes or reference-type nullability; .NET 9 uses JsonPropertyInfo.IsSetNullable and JsonTypeInfo.ElementType. --- .../JsonShapeMetadataTests.cs | 162 ++++++++++++++ .../ShapeCoverageTests.cs | 2 +- .../RunJsonShapeMetadataTests.cs | 3 + .../JsonShape.cs | 203 ++++++++++++------ .../JsonShapeAnnotations.cs | 96 +++++++++ .../JsonShapeProperty.cs | 145 +++++++++++++ 6 files changed, 548 insertions(+), 63 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeMetadataTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunJsonShapeMetadataTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer/JsonShapeAnnotations.cs create mode 100644 DragoAnt.System.Text.Json.Observer/JsonShapeProperty.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeMetadataTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeMetadataTests.cs new file mode 100644 index 0000000..73fc063 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonShapeMetadataTests.cs @@ -0,0 +1,162 @@ +using System.Reflection; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using DragoAnt.System.Text.Json.Observer.Strategies; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract partial class JsonShapeMetadataTests +{ + private static readonly JsonSerializerOptions Web = new(JsonSerializerDefaults.Web) { TypeInfoResolver = new DefaultJsonTypeInfoResolver() }; + + private static JsonShape Build(Action? annotate = null) => + JsonShape.FromTypeInfo(Web.GetTypeInfo(typeof(Order)), p => p.Name == "secret" ? MaskTag.Full : null, annotate); + + private static JsonShapeProperty Member(JsonShape shape, string name) => shape.Members.Single(m => m.Name == name); + + [Fact] + public void FromTypeInfo_PropertiesCarryClrMetadata() + { + var shape = Build(); + + var sku = Member(shape, "sku"); + sku.PropertyInfo.Should().NotBeNull(); + sku.PropertyType.Should().Be(typeof(string)); + sku.Member.Should().BeAssignableTo().Which.Name.Should().Be(nameof(Order.Sku)); + sku.AttributeProvider.Should().BeSameAs(sku.Member); + sku.IsRequired.Should().BeTrue(); + sku.IsNullable.Should().BeFalse(); + sku.GetCustomAttributes().Select(a => a.Text).Should().Equal("stock keeping unit"); + + Member(shape, "note").IsNullable.Should().BeTrue(); + Member(shape, "note").IsRequired.Should().BeFalse(); + Member(shape, "quantity").IsNullable.Should().BeFalse(); + Member(shape, "discount").IsNullable.Should().BeTrue(); + Member(shape, "ext_ref").Member!.Name.Should().Be(nameof(Order.ExternalReference)); + Member(shape, "secret").Shape.Kind.Should().Be(JsonShapeKind.Masked); + Member(shape, "secret").GetCustomAttributes().Select(a => a.Text).Should().Equal("hidden"); + } + + [Fact] + public void FromTypeInfo_NodesCarryClrType() + { + var shape = Build(); + + shape.ClrType.Should().Be(typeof(Order)); + shape.TypeInfo!.Type.Should().Be(typeof(Order)); + Member(shape, "lines").Shape.ClrType.Should().Be(typeof(List)); + Member(shape, "lines").Shape.Item!.ClrType.Should().Be(typeof(Line)); + Member(shape, "byCode").Shape.Kind.Should().Be(JsonShapeKind.Map); + Member(shape, "quantity").Shape.Kind.Should().Be(JsonShapeKind.Scalar); + Member(shape, "quantity").Shape.ClrType.Should().Be(typeof(int)); + JsonShape.Scalar.ClrType.Should().BeNull(); + } + + [Fact] + public void FromTypeInfo_AnnotateHook_SeesEveryProperty() + { + var seen = new List(); + var shape = Build(p => + { + seen.Add($"{p.DeclaringType?.Name}.{p.Name}"); + p.Annotations.Set(new Rule(p.IsRequired)); + }); + + seen.Should().Contain(["Order.sku", "Order.lines", "Line.qty"]); + Member(shape, "sku").Annotations.Get().Should().Be(new Rule(true)); + Member(shape, "lines").Shape.Item!.Members.Single(m => m.Name == "qty").Annotations.TryGet(out var rule).Should().BeTrue(); + rule.Should().Be(new Rule(false)); + } + + [Fact] + public void Annotations_SetGetRemove() + { + var annotations = JsonShape.Object().Annotations; + + annotations.TryGet(out _).Should().BeFalse(); + annotations.Get().Should().BeNull(); + annotations.Set(new Rule(true)); + annotations.Set("label"); + annotations.Set(new Rule(false)); + annotations.Count.Should().Be(2); + annotations.Get().Should().Be(new Rule(false)); + annotations.Get().Should().Be("label"); + annotations.Remove().Should().BeTrue(); + annotations.Remove().Should().BeFalse(); + annotations.Count.Should().Be(1); + var setNull = () => annotations.Set(null!); + setNull.Should().Throw(); + } + + [Fact] + public void HandBuiltShape_PropertiesHaveNoClrMetadata() + { + var property = new JsonShapeProperty("id", JsonShape.Scalar); + property.Annotations.Set(new Rule(true)); + var shape = JsonShape.Object(("name", JsonShape.Scalar)).Add(property); + + var (name, node) = shape.Members[0]; + name.Should().Be("name"); + node.Should().BeSameAs(JsonShape.Scalar); + shape.Members[1].Should().BeSameAs(property); + property.PropertyInfo.Should().BeNull(); + property.Member.Should().BeNull(); + property.IsNullable.Should().BeNull(); + property.IsRequired.Should().BeFalse(); + property.GetCustomAttributes().Should().BeEmpty(); + JsonObserver.FromShape(shape).Mask("""{"id":1,"name":"a","x":2}""").Should().Be("""{"id":1,"name":"a","x":"***"}"""); + } + + [Fact] + public void SourceGenerated_MetadataAvailability() + { + var shape = JsonShape.FromTypeInfo(SourceGenContext.Default.Line, _ => null); + + var qty = Member(shape, "Qty"); + qty.PropertyType.Should().Be(typeof(int)); + qty.IsNullable.Should().BeFalse(); + var label = Member(shape, "Label"); +#if NET9_0_OR_GREATER + label.IsNullable.Should().BeTrue(); +#else + label.IsNullable.Should().BeNull(); + label.Member.Should().BeNull(); +#endif + } + + [AttributeUsage(AttributeTargets.Property)] + public sealed class NoteAttribute(string text) : Attribute + { + public string Text { get; } = text; + } + + public sealed record Rule(bool Required); + + public sealed class Order + { + [Note("stock keeping unit")] + public required string Sku { get; set; } + + public string? Note { get; set; } + public int Quantity { get; set; } + public decimal? Discount { get; set; } + + [JsonPropertyName("ext_ref")] + public string? ExternalReference { get; set; } + + [Note("hidden")] + public string? Secret { get; set; } + + public List? Lines { get; set; } + public Dictionary? ByCode { get; set; } + } + + public sealed class Line + { + public int Qty { get; set; } + public string? Label { get; set; } + } + + [JsonSerializable(typeof(Line))] + internal sealed partial class SourceGenContext : JsonSerializerContext; +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ShapeCoverageTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ShapeCoverageTests.cs index 8747c98..4f4a5d3 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ShapeCoverageTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ShapeCoverageTests.cs @@ -106,7 +106,7 @@ public void Members_IsReadOnly() { var shape = JsonShape.Object(("id", JsonShape.Scalar)); - shape.Members.Should().NotBeAssignableTo>(); + shape.Members.Should().NotBeAssignableTo>(); shape.Members.Should().ContainSingle(); } diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunJsonShapeMetadataTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunJsonShapeMetadataTests.cs new file mode 100644 index 0000000..d89ee3d --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunJsonShapeMetadataTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunJsonShapeMetadataTests : Shared.JsonShapeMetadataTests; diff --git a/DragoAnt.System.Text.Json.Observer/JsonShape.cs b/DragoAnt.System.Text.Json.Observer/JsonShape.cs index a967e12..e877947 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonShape.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonShape.cs @@ -1,3 +1,4 @@ +using System.Reflection; using System.Text; using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; @@ -49,15 +50,16 @@ public sealed class JsonShape { private static readonly Type[] OpaqueTypes = [typeof(object), typeof(JsonElement), typeof(JsonDocument), typeof(JsonNode)]; - private readonly List<(string Name, JsonShape Shape)>? _members; - private (byte[]? Ascii, string Name, JsonShape Shape)[] _lookup = []; + private readonly List? _members; + private (byte[]? Ascii, JsonShapeProperty Property)[] _lookup = []; private bool _sealed; - private JsonShape(JsonShapeKind kind, MaskTag tag = default, JsonShape? item = null) + private JsonShape(JsonShapeKind kind, MaskTag tag = default, JsonShape? item = null, JsonTypeInfo? typeInfo = null) { Kind = kind; Tag = tag; Item = item; + TypeInfo = typeInfo; _members = kind == JsonShapeKind.Object ? [] : null; } @@ -77,9 +79,25 @@ private JsonShape(JsonShapeKind kind, MaskTag tag = default, JsonShape? item = n public JsonShape? Item { get; private set; } /// - /// Known properties of an . + /// Known properties of an , in the order they were added. /// - public IReadOnlyList<(string Name, JsonShape Shape)> Members => (IReadOnlyList<(string Name, JsonShape Shape)>?)_members?.AsReadOnly() ?? []; + public IReadOnlyList Members => (IReadOnlyList?)_members?.AsReadOnly() ?? []; + + /// + /// The System.Text.Json metadata the node was built from; null for a hand-built node and for the shared + /// and nodes. + /// + public JsonTypeInfo? TypeInfo { get; } + + /// + /// CLR type of the node, from . + /// + public Type? ClrType => TypeInfo?.Type; + + /// + /// Data an integration attaches to the node. + /// + public JsonShapeAnnotations Annotations { get; } = new(); /// /// A value written as is. @@ -111,7 +129,7 @@ private JsonShape(JsonShapeKind kind, MaskTag tag = default, JsonShape? item = n public static JsonShape Map(JsonShape value) => new(JsonShapeKind.Map, item: value); /// - /// An object with the given properties; add more with , which also allows cycles. + /// An object with the given properties; add more with , which also allows cycles. /// public static JsonShape Object(params (string Name, JsonShape Shape)[] members) { @@ -128,8 +146,15 @@ public static JsonShape Object(params (string Name, JsonShape Shape)[] members) /// Adds a known property to an object shape. Names match case-insensitively; when two names collide the masked one wins. /// /// The shape is not an object, or an observer was already built from it. - public JsonShape Add(string name, JsonShape shape) + public JsonShape Add(string name, JsonShape shape) => Add(new JsonShapeProperty(name, shape)); + + /// + /// Adds a known property, with its annotations, to an object shape. When two names collide the masked one wins. + /// + /// The shape is not an object, or an observer was already built from it. + public JsonShape Add(JsonShapeProperty property) { + ArgumentNullException.ThrowIfNull(property); if (_members is null || _sealed) { throw new InvalidOperationException(_members is null @@ -137,7 +162,7 @@ public JsonShape Add(string name, JsonShape shape) : "The shape is in use by an observer and can no longer change."); } - _members.Add((name, shape)); + _members.Add(property); return this; } @@ -147,27 +172,51 @@ public JsonShape Add(string name, JsonShape shape) /// /// Metadata of the root type. /// Mask for a sensitive property, or null for one shown as is. + /// + /// Called once per property with its , for example to attach annotations; a type + /// reached twice, or recursively, is built and annotated once. + /// /// - /// On .NET 8, metadata from a source-generated JsonSerializerContext has no , - /// so a that reads attributes finds none and shows every property; classify by name there, - /// or use reflection-based metadata. + /// Every node and property carries its metadata: , , + /// the CLR member, nullability and attributes. On .NET 8, metadata from a source-generated JsonSerializerContext + /// has no , so a that reads attributes finds + /// none and shows every property; classify by name there, or use reflection-based metadata. /// - public static JsonShape FromTypeInfo(JsonTypeInfo typeInfo, Func classify) + public static JsonShape FromTypeInfo(JsonTypeInfo typeInfo, Func classify, Action? annotate = null) { ArgumentNullException.ThrowIfNull(typeInfo); ArgumentNullException.ThrowIfNull(classify); - return Build(typeInfo, classify, []); + return new Builder(classify, annotate).Build(typeInfo); + } + + /// + /// Finds a known property of an object shape by its unescaped UTF-8 JSON name, as the observer does. The first lookup + /// freezes the shape like building an observer from it. + /// + /// Unescaped UTF-8 name. + /// Compare names ignoring case, as by default. + /// The property, or null when the shape does not know it or is not an object. + public JsonShapeProperty? FindMember(ReadOnlySpan utf8Name, bool propertyNameCaseInsensitive = true) + { + if (!_sealed) + { + Seal([]); + } + + return FindProperty(utf8Name); } - internal JsonShape? Find(ReadOnlySpan utf8Name) + internal JsonShape? Find(ReadOnlySpan utf8Name) => FindProperty(utf8Name)?.Shape; + + private JsonShapeProperty? FindProperty(ReadOnlySpan utf8Name) { if (Ascii.IsValid(utf8Name)) { - foreach (var (ascii, _, shape) in _lookup) + foreach (var (ascii, property) in _lookup) { if (ascii is not null && ascii.Length == utf8Name.Length && Ascii.EqualsIgnoreCase(ascii, utf8Name)) { - return shape; + return property; } } @@ -175,11 +224,11 @@ public static JsonShape FromTypeInfo(JsonTypeInfo typeInfo, Func visited) _sealed = true; if (_members is not null) { - var merged = new List<(string Name, JsonShape Shape)>(); - foreach (var (name, shape) in _members) + var merged = new List(); + foreach (var property in _members) { - var index = merged.FindIndex(m => string.Equals(m.Name, name, StringComparison.OrdinalIgnoreCase)); + var index = merged.FindIndex(m => string.Equals(m.Name, property.Name, StringComparison.OrdinalIgnoreCase)); if (index < 0) { - merged.Add((name, shape)); + merged.Add(property); } - else if (shape.Kind is JsonShapeKind.Masked or JsonShapeKind.Opaque) + else if (property.Shape.Kind is JsonShapeKind.Masked or JsonShapeKind.Opaque) { - merged[index] = (name, shape); + merged[index] = property; } } _lookup = merged - .Select(m => (Ascii.IsValid(m.Name) ? Encoding.ASCII.GetBytes(m.Name) : null, m.Name, m.Shape)) + .Select(m => (Ascii.IsValid(m.Name) ? Encoding.ASCII.GetBytes(m.Name) : null, m)) .ToArray(); - foreach (var (_, shape) in merged) + foreach (var property in merged) { - shape.Seal(visited); + property.Shape.Seal(visited); } } Item?.Seal(visited); } - private static JsonShape Build(JsonTypeInfo typeInfo, Func classify, Dictionary built) + private sealed class Builder(Func classify, Action? annotate) { - var type = typeInfo.Type; - if (built.TryGetValue(type, out var existing)) - { - return existing; - } - - if (OpaqueTypes.Any(t => t.IsAssignableFrom(type) && (t != typeof(object) || type == typeof(object)))) - { - return Opaque; - } + private readonly Dictionary _built = []; +#if !NET9_0_OR_GREATER + private readonly NullabilityInfoContext _nullability = new(); +#endif - switch (typeInfo.Kind) + public JsonShape Build(JsonTypeInfo typeInfo) { - case JsonTypeInfoKind.Object: + var type = typeInfo.Type; + if (_built.TryGetValue(type, out var existing)) { - var shape = new JsonShape(JsonShapeKind.Object); - built[type] = shape; - foreach (var property in typeInfo.Properties.Where(p => !p.IsExtensionData)) - { - var tag = classify(property); - shape.Add(property.Name, tag is { } mask - ? Masked(mask) - : Build(typeInfo.Options.GetTypeInfo(property.PropertyType), classify, built)); - } + return existing; + } - return shape; + if (OpaqueTypes.Any(t => t.IsAssignableFrom(type) && (t != typeof(object) || type == typeof(object)))) + { + return Opaque; } - case JsonTypeInfoKind.Enumerable: - case JsonTypeInfoKind.Dictionary: + + switch (typeInfo.Kind) { - var shape = new JsonShape(typeInfo.Kind == JsonTypeInfoKind.Enumerable ? JsonShapeKind.Array : JsonShapeKind.Map); - built[type] = shape; - shape.Item = ElementTypeOf(type, typeInfo.Kind == JsonTypeInfoKind.Dictionary) is { } elementType - ? Build(typeInfo.Options.GetTypeInfo(elementType), classify, built) - : Opaque; - return shape; + case JsonTypeInfoKind.Object: + { + var shape = new JsonShape(JsonShapeKind.Object, typeInfo: typeInfo); + _built[type] = shape; + foreach (var property in typeInfo.Properties.Where(p => !p.IsExtensionData)) + { + var tag = classify(property); + var member = new JsonShapeProperty( + property.Name, + tag is { } mask ? Masked(mask) : Build(typeInfo.Options.GetTypeInfo(property.PropertyType)), + property, + Nullability(property)); + shape.Add(member); + annotate?.Invoke(member); + } + + return shape; + } + case JsonTypeInfoKind.Enumerable: + case JsonTypeInfoKind.Dictionary: + { + var shape = new JsonShape(typeInfo.Kind == JsonTypeInfoKind.Enumerable ? JsonShapeKind.Array : JsonShapeKind.Map, typeInfo: typeInfo); + _built[type] = shape; + shape.Item = ElementTypeOf(typeInfo) is { } elementType + ? Build(typeInfo.Options.GetTypeInfo(elementType)) + : Opaque; + return shape; + } + default: + { + var shape = new JsonShape(JsonShapeKind.Scalar, typeInfo: typeInfo); + _built[type] = shape; + return shape; + } } - default: - return Scalar; } + +#if NET9_0_OR_GREATER + private static bool? Nullability(JsonPropertyInfo property) => JsonShapeProperty.NullabilityOf(property, null); +#else + private bool? Nullability(JsonPropertyInfo property) => JsonShapeProperty.NullabilityOf(property, _nullability); +#endif } - private static Type? ElementTypeOf(Type type, bool dictionary) + private static Type? ElementTypeOf(JsonTypeInfo typeInfo) { +#if NET9_0_OR_GREATER + if (typeInfo.ElementType is { } elementType) + { + return elementType; + } +#endif + var type = typeInfo.Type; + var dictionary = typeInfo.Kind == JsonTypeInfoKind.Dictionary; if (type.IsArray) { return type.GetElementType(); diff --git a/DragoAnt.System.Text.Json.Observer/JsonShapeAnnotations.cs b/DragoAnt.System.Text.Json.Observer/JsonShapeAnnotations.cs new file mode 100644 index 0000000..3ba87b7 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/JsonShapeAnnotations.cs @@ -0,0 +1,96 @@ +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// Data an integration attaches to a node or a , one value per +/// type, for example validation rules or a data classification. Annotations never change masking; they can be set at +/// any time and are safe to read and write from several threads. +/// +public sealed class JsonShapeAnnotations +{ + private readonly object _sync = new(); + private (Type Type, object Value)[] _values = []; + + internal JsonShapeAnnotations() + { + } + + /// + /// Number of annotations. + /// + public int Count => Volatile.Read(ref _values).Length; + + /// + /// Sets the annotation of type , replacing any previous one. + /// + /// The annotation. + /// Type the annotation is stored under. + /// is null. + public void Set(T value) + where T : notnull + { + ArgumentNullException.ThrowIfNull(value); + lock (_sync) + { + var values = _values; + var index = IndexOf(values, typeof(T)); + var updated = new (Type Type, object Value)[index >= 0 ? values.Length : values.Length + 1]; + values.CopyTo(updated, 0); + updated[index >= 0 ? index : values.Length] = (typeof(T), value); + Volatile.Write(ref _values, updated); + } + } + + /// + /// Gets the annotation of type . + /// + /// The annotation; default when there is none. + /// Type the annotation is stored under. + /// true when there is one. + public bool TryGet(out T? value) + { + var values = Volatile.Read(ref _values); + var index = IndexOf(values, typeof(T)); + value = index >= 0 ? (T)values[index].Value : default; + return index >= 0; + } + + /// + /// Gets the annotation of type , or default when there is none. + /// + /// Type the annotation is stored under. + public T? Get() => TryGet(out var value) ? value : default; + + /// + /// Removes the annotation of type . + /// + /// Type the annotation is stored under. + /// true when there was one. + public bool Remove() + { + lock (_sync) + { + var values = _values; + var index = IndexOf(values, typeof(T)); + if (index < 0) + { + return false; + } + + Volatile.Write(ref _values, [.. values[..index], .. values[(index + 1)..]]); + return true; + } + } + + private static int IndexOf((Type Type, object Value)[] values, Type type) + { + for (var i = 0; i < values.Length; i++) + { + if (values[i].Type == type) + { + return i; + } + } + + return -1; + } +} diff --git a/DragoAnt.System.Text.Json.Observer/JsonShapeProperty.cs b/DragoAnt.System.Text.Json.Observer/JsonShapeProperty.cs new file mode 100644 index 0000000..00c7d5b --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/JsonShapeProperty.cs @@ -0,0 +1,145 @@ +using System.Reflection; +using System.Text.Json.Serialization.Metadata; + +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// A known property of an object : its JSON name, the shape of its value and, when the shape was +/// built from System.Text.Json metadata, what that metadata says about the CLR member. +/// +/// +/// On .NET 8, metadata from a source-generated JsonSerializerContext has no attribute provider, so +/// , and are empty and +/// is null for reference types; reflection-based metadata and .NET 9 or later carry them. +/// +public sealed class JsonShapeProperty +{ + /// + /// Creates a property for a hand-built shape; it carries no CLR metadata. + /// + /// JSON name of the property. + /// Shape of its value. + public JsonShapeProperty(string name, JsonShape shape) + : this(name, shape, null, null) + { + } + + internal JsonShapeProperty(string name, JsonShape shape, JsonPropertyInfo? propertyInfo, bool? isNullable) + { + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(shape); + Name = name; + Shape = shape; + PropertyInfo = propertyInfo; + IsNullable = isNullable; + } + + /// + /// JSON name of the property, after the naming policy and JsonPropertyNameAttribute. + /// + public string Name { get; } + + /// + /// Shape of the property's value. + /// + public JsonShape Shape { get; } + + /// + /// The System.Text.Json metadata the property was built from; null for a hand-built shape. + /// + public JsonPropertyInfo? PropertyInfo { get; } + + /// + /// CLR type of the property. + /// + public Type? PropertyType => PropertyInfo?.PropertyType; + + /// + /// The CLR type that declares the property. + /// + public Type? DeclaringType => +#if NET9_0_OR_GREATER + PropertyInfo?.DeclaringType; +#else + Member?.DeclaringType; +#endif + + /// + /// Attributes of the CLR member, see the remarks for .NET 8. + /// + public ICustomAttributeProvider? AttributeProvider => PropertyInfo?.AttributeProvider; + + /// + /// The CLR property or field, see the remarks for .NET 8. + /// + public MemberInfo? Member => AttributeProvider as MemberInfo; + + /// + /// The property must be present when deserializing (required or JsonRequiredAttribute). + /// + public bool IsRequired => PropertyInfo?.IsRequired ?? false; + + /// + /// Whether the value may be null when deserializing: from for value types and from the + /// nullable annotations for reference types; null when unknown, see the remarks for .NET 8. + /// + public bool? IsNullable { get; } + + /// + /// Data an integration attaches to the property. + /// + public JsonShapeAnnotations Annotations { get; } = new(); + + /// + /// Custom attributes of type on the CLR member; empty when unknown. + /// + /// Search the member's inheritance chain. + /// Attribute type. + public IEnumerable GetCustomAttributes(bool inherit = true) + where T : Attribute => + AttributeProvider?.GetCustomAttributes(typeof(T), inherit).OfType() ?? []; + + /// + /// Lets a property be deconstructed like a (Name, Shape) tuple. + /// + /// JSON name. + /// Shape of the value. + public void Deconstruct(out string name, out JsonShape shape) + { + name = Name; + shape = Shape; + } + + internal static bool? NullabilityOf(JsonPropertyInfo property, NullabilityInfoContext? context) + { + var type = property.PropertyType; + if (type.IsValueType) + { + return Nullable.GetUnderlyingType(type) is not null; + } + +#if NET9_0_OR_GREATER + return property.IsSetNullable; +#else + try + { + var info = property.AttributeProvider switch + { + PropertyInfo p when context is not null => context.Create(p), + FieldInfo f when context is not null => context.Create(f), + _ => null, + }; + return info?.WriteState switch + { + NullabilityState.Nullable => true, + NullabilityState.NotNull => false, + _ => null, + }; + } + catch (Exception) + { + return null; + } +#endif + } +} From c0acb368f9831cd398a6bc0cd6efdf93f4aefedd Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:53:17 +0200 Subject: [PATCH 08/13] Let name matching follow the serializer's case sensitivity JsonObserverOptions.PropertyNameCaseInsensitive (default true, as before) switches rule names, PropMatches tests and shape lookups to ordinal matching per call; PropertyPath exposes the mode to custom rules. JsonShapeOptions.PropertyNameCaseInsensitive overrides it for one shape observer, and JsonShapeOptions.FromSerializerOptions takes it from the serializer's options. In exact mode a shape keeps names that differ only in case apart. Name matchers can describe themselves, which Explain will use. --- .../CaseSensitivityTests.cs | 92 ++++++++++++++++ .../RunCaseSensitivityTests.cs | 3 + .../JsonObserver.cs | 2 +- .../JsonObserverOptions.cs | 11 +- .../JsonShape.cs | 76 ++++++++----- .../JsonShapeOptions.cs | 28 ++++- .../PropertyPath.cs | 5 + .../PropertyPathMatch.cs | 2 - .../ShapeWalker.cs | 6 +- .../Strategies/NameMatcher.cs | 102 +++++++++++------- .../Strategies/PropMatches.cs | 2 +- 11 files changed, 249 insertions(+), 80 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/CaseSensitivityTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunCaseSensitivityTests.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/CaseSensitivityTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/CaseSensitivityTests.cs new file mode 100644 index 0000000..3f0f490 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/CaseSensitivityTests.cs @@ -0,0 +1,92 @@ +using System.Text; +using System.Text.Json.Serialization.Metadata; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class CaseSensitivityTests +{ + private static readonly JsonObserverOptions Exact = new(PropertyNameCaseInsensitive: false); + + private static readonly JsonObserver Rules = JsonObserver.Obj(Relative(b => b + .Match("password").MaskAny("1") + .Match(PropMatches.StartsWith("tok")).MaskAny("2") + .Match(PropMatches.EndsWith("Card")).MaskAny("3") + .Match(PropMatches.Contains("mail")).MaskAny("4") + .Match(PropMatches.OneOf("pin", "cvv")).MaskAny("5") + .Match("ключ").MaskAny("6") + .Match(PropMatches.StartsWith("пар")).MaskAny("7"), + BlockList)); + + private const string Payload = """{"password":"a","Password":"b","token":"c","Token":"d","myCard":"e","mycard":"f","email":"g","eMail":"h","pin":"i","PIN":"j","ключ":"k","КЛЮЧ":"l","пароль":"m","Пароль":"n"}"""; + + [Fact] + public void Rules_Default_IgnoreCase() => + Rules.Mask(Payload).Should().Be( + """{"password":"1","Password":"1","token":"2","Token":"2","myCard":"3","mycard":"3","email":"4","eMail":"4","pin":"5","PIN":"5","ключ":"6","КЛЮЧ":"6","пароль":"7","Пароль":"7"}"""); + + [Fact] + public void Rules_CaseSensitive_MatchExactNamesOnly() => + Rules.Mask(Payload, Exact).Should().Be( + """{"password":"1","Password":"b","token":"2","Token":"d","myCard":"3","mycard":"f","email":"4","eMail":"h","pin":"5","PIN":"j","ключ":"6","КЛЮЧ":"l","пароль":"7","Пароль":"n"}"""); + + [Fact] + public void AbsoluteRules_CaseSensitive_UnderAllowList_MaskUnmatchedCase() => + JsonObserver.Obj(root => root.Match("order").Obj(o => o.Match("id").Unmasked())) + .Mask("""{"order":{"id":1,"ID":2},"Order":{"id":3}}""", Exact) + .Should().Be("""{"order":{"id":1,"ID":"***"},"Order":{"id":"***"}}"""); + + [Fact] + public void CustomRule_SeesTheCallsMatchingMode() + { + var modes = new List(); + var observer = JsonObserver.Obj(b => b.Match("a").MaskValue((ref Utf8JsonReader _, JsonWriter writer, JsonObserveringEmptyContext _, ref PropertyPath path) => + { + modes.Add(path.PropertyNameCaseInsensitive); + writer.WriteNullValue(); + }), BlockList); + + observer.Mask("""{"a":1}"""); + observer.Mask("""{"a":1}""", Exact); + + modes.Should().Equal(true, false); + } + + [Fact] + public void Shape_FollowsSerializerOptions() + { + var general = new JsonSerializerOptions { TypeInfoResolver = new DefaultJsonTypeInfoResolver() }; + var shape = JsonShape.FromTypeInfo(general.GetTypeInfo(typeof(Person)), p => p.Name == "Secret" ? MaskTag.Full : null); + const string json = """{"Name":"a","name":"b","Secret":"c","secret":"d"}"""; + + JsonObserver.FromShape(shape).Mask(json).Should().Be("""{"Name":"a","name":"b","Secret":"***","secret":"***"}"""); + JsonObserver.FromShape(shape, JsonShapeOptions.FromSerializerOptions(general)).Mask(json) + .Should().Be("""{"Name":"a","name":"***","Secret":"***","secret":"***"}"""); + JsonObserver.FromShape(shape).Mask(json, Exact).Should().Be("""{"Name":"a","name":"***","Secret":"***","secret":"***"}"""); + JsonObserver.FromShape(shape, new JsonShapeOptions(PropertyNameCaseInsensitive: true)).Mask(json, Exact) + .Should().Be("""{"Name":"a","name":"b","Secret":"***","secret":"***"}"""); + JsonShapeOptions.FromSerializerOptions(new JsonSerializerOptions(JsonSerializerDefaults.Web)).PropertyNameCaseInsensitive.Should().BeTrue(); + } + + [Fact] + public void Shape_CaseSensitive_KeepsNamesDifferingInCase() + { + var shape = JsonShape.Object(("id", JsonShape.Scalar), ("ID", JsonShape.Masked(MaskTag.Full)), ("ключ", JsonShape.Scalar)); + + JsonObserver.FromShape(shape).Mask("""{"id":1,"ID":2}""").Should().Be("""{"id":"***","ID":"***"}"""); + JsonObserver.FromShape(shape).Mask("""{"id":1,"ID":2,"ключ":3,"КЛЮЧ":4}""", Exact) + .Should().Be("""{"id":1,"ID":"***","ключ":3,"КЛЮЧ":"***"}"""); + shape.FindMember("Id"u8)!.Name.Should().Be("ID"); + shape.FindMember("Id"u8, propertyNameCaseInsensitive: false).Should().BeNull(); + shape.FindMember("id"u8, propertyNameCaseInsensitive: false)!.Name.Should().Be("id"); + shape.FindMember(Encoding.UTF8.GetBytes("КЛЮЧ"), propertyNameCaseInsensitive: false).Should().BeNull(); + shape.FindMember(Encoding.UTF8.GetBytes("КЛЮЧ"))!.Name.Should().Be("ключ"); + } + + public sealed class Person + { + public string? Name { get; set; } + public string? Secret { get; set; } + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunCaseSensitivityTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunCaseSensitivityTests.cs new file mode 100644 index 0000000..e850ac4 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunCaseSensitivityTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunCaseSensitivityTests : Shared.CaseSensitivityTests; diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs index 911922d..21d4389 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs @@ -331,7 +331,7 @@ public MaskResult Read(ReadOnlySpan utf8, TContext context, JsonObserverOp AllowTrailingCommas = true, MaxDepth = Math.Max(options.MaxDepth, 1), })); - var propPath = new PropertyPath(_maxDepth, utf8); + var propPath = new PropertyPath(_maxDepth, utf8) { PropertyNameCaseInsensitive = options.PropertyNameCaseInsensitive }; try { if (!reader.Read() || reader.TokenType is not (JsonTokenType.StartObject or JsonTokenType.StartArray)) diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverOptions.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverOptions.cs index df291fe..c1a2664 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverOptions.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverOptions.cs @@ -19,6 +19,12 @@ namespace DragoAnt.System.Text.Json.Observer; /// Strategy for rules added with a ; when null. /// Drop properties and array items whose value is null, and objects and arrays left empty by that. /// Write the output indented. +/// +/// Match rule names, tests and shape properties ignoring case, as by default. Pass the +/// PropertyNameCaseInsensitive of the serializer's options to match names the way deserialization does. +/// With false a rule no longer catches a differently cased name: under a block list such a value is written +/// unchanged, under an allow list it is masked. +/// public sealed record JsonObserverOptions( int MaxOutputBytes = int.MaxValue, int MaxValueBytes = int.MaxValue, @@ -27,11 +33,12 @@ public sealed record JsonObserverOptions( ReadOnlyMemory HashKey = default, Utf8MaskStrategy? MaskStrategy = null, bool IgnoreNulls = false, - bool Indented = false) + bool Indented = false, + bool PropertyNameCaseInsensitive = true) { /// /// Defaults: no size limits, depth 64, relaxed escaping, a per-process hash key, the built-in strategy, - /// null values kept and compact output. + /// null values kept, compact output and names matched ignoring case. /// public static JsonObserverOptions Default { get; } = new(); } diff --git a/DragoAnt.System.Text.Json.Observer/JsonShape.cs b/DragoAnt.System.Text.Json.Observer/JsonShape.cs index e877947..522883f 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonShape.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonShape.cs @@ -51,8 +51,10 @@ public sealed class JsonShape private static readonly Type[] OpaqueTypes = [typeof(object), typeof(JsonElement), typeof(JsonDocument), typeof(JsonNode)]; private readonly List? _members; + private static readonly object SealSync = new(); private (byte[]? Ascii, JsonShapeProperty Property)[] _lookup = []; - private bool _sealed; + private (byte[]? Ascii, JsonShapeProperty Property)[] _exactLookup = []; + private volatile bool _sealed; private JsonShape(JsonShapeKind kind, MaskTag tag = default, JsonShape? item = null, JsonTypeInfo? typeInfo = null) { @@ -200,21 +202,23 @@ public static JsonShape FromTypeInfo(JsonTypeInfo typeInfo, Func utf8Name) => FindProperty(utf8Name)?.Shape; + internal JsonShape? Find(ReadOnlySpan utf8Name, bool ignoreCase) => FindProperty(utf8Name, ignoreCase)?.Shape; - private JsonShapeProperty? FindProperty(ReadOnlySpan utf8Name) + private JsonShapeProperty? FindProperty(ReadOnlySpan utf8Name, bool ignoreCase) { + var lookup = ignoreCase ? _lookup : _exactLookup; if (Ascii.IsValid(utf8Name)) { - foreach (var (ascii, property) in _lookup) + foreach (var (ascii, property) in lookup) { - if (ascii is not null && ascii.Length == utf8Name.Length && Ascii.EqualsIgnoreCase(ascii, utf8Name)) + if (ascii is not null && ascii.Length == utf8Name.Length && + (ignoreCase ? Ascii.EqualsIgnoreCase(ascii, utf8Name) : utf8Name.SequenceEqual(ascii))) { return property; } @@ -224,9 +228,10 @@ public static JsonShape FromTypeInfo(JsonTypeInfo typeInfo, Func visited) + internal void Freeze() + { + lock (SealSync) + { + Seal([]); + } + } + + private void Seal(HashSet visited) { if (!visited.Add(this)) { return; } - _sealed = true; if (_members is not null) { - var merged = new List(); - foreach (var property in _members) + _lookup = Merge(_members, StringComparison.OrdinalIgnoreCase); + _exactLookup = Merge(_members, StringComparison.Ordinal); + } + + _sealed = true; + foreach (var property in _members ?? []) + { + property.Shape.Seal(visited); + } + + Item?.Seal(visited); + } + + private static (byte[]? Ascii, JsonShapeProperty Property)[] Merge(List members, StringComparison comparison) + { + var merged = new List(); + foreach (var property in members) + { + var index = merged.FindIndex(m => string.Equals(m.Name, property.Name, comparison)); + if (index < 0) { - var index = merged.FindIndex(m => string.Equals(m.Name, property.Name, StringComparison.OrdinalIgnoreCase)); - if (index < 0) - { - merged.Add(property); - } - else if (property.Shape.Kind is JsonShapeKind.Masked or JsonShapeKind.Opaque) - { - merged[index] = property; - } + merged.Add(property); } - - _lookup = merged - .Select(m => (Ascii.IsValid(m.Name) ? Encoding.ASCII.GetBytes(m.Name) : null, m)) - .ToArray(); - foreach (var property in merged) + else if (property.Shape.Kind is JsonShapeKind.Masked or JsonShapeKind.Opaque) { - property.Shape.Seal(visited); + merged[index] = property; } } - Item?.Seal(visited); + return merged.Select(m => (Ascii.IsValid(m.Name) ? Encoding.ASCII.GetBytes(m.Name) : null, m)).ToArray(); } private sealed class Builder(Func classify, Action? annotate) diff --git a/DragoAnt.System.Text.Json.Observer/JsonShapeOptions.cs b/DragoAnt.System.Text.Json.Observer/JsonShapeOptions.cs index 8bf2434..73ace67 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonShapeOptions.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonShapeOptions.cs @@ -26,10 +26,34 @@ public enum UnknownMemberPolicy /// /// What happens to a property the shape does not know. /// Write null as null where a value would be masked; otherwise it is masked too. -public sealed record JsonShapeOptions(UnknownMemberPolicy Unknown = UnknownMemberPolicy.MaskWhole, bool KeepNulls = true) +/// +/// Match property names ignoring case; null follows +/// of the call, which ignores case by default. A property whose name does not match is unknown, so with +/// a differently cased sensitive property is written unchanged. +/// +public sealed record JsonShapeOptions( + UnknownMemberPolicy Unknown = UnknownMemberPolicy.MaskWhole, + bool KeepNulls = true, + bool? PropertyNameCaseInsensitive = null) { /// - /// Defaults: unknown properties masked whole, null kept. + /// Defaults: unknown properties masked whole, null kept, names matched as the call says. /// public static JsonShapeOptions Default { get; } = new(); + + /// + /// Options that match property names the way the serializer does, typically those the shape was built from + /// (JsonTypeInfo.Options). + /// + /// Options whose is used. + /// What happens to a property the shape does not know. + /// Write null as null where a value would be masked. + public static JsonShapeOptions FromSerializerOptions( + JsonSerializerOptions serializerOptions, + UnknownMemberPolicy unknown = UnknownMemberPolicy.MaskWhole, + bool keepNulls = true) + { + ArgumentNullException.ThrowIfNull(serializerOptions); + return new JsonShapeOptions(unknown, keepNulls, serializerOptions.PropertyNameCaseInsensitive); + } } diff --git a/DragoAnt.System.Text.Json.Observer/PropertyPath.cs b/DragoAnt.System.Text.Json.Observer/PropertyPath.cs index 5b935fe..f07fbeb 100644 --- a/DragoAnt.System.Text.Json.Observer/PropertyPath.cs +++ b/DragoAnt.System.Text.Json.Observer/PropertyPath.cs @@ -40,6 +40,11 @@ internal PropertyPath(int capacity, ReadOnlySpan input) /// public readonly int Length => Depth + 1; + /// + /// Whether names are matched ignoring case in this call, see . + /// + public bool PropertyNameCaseInsensitive { readonly get; internal set; } = true; + /// /// The input ended inside a value: every rule must stop reading. /// diff --git a/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs b/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs index d02f22e..93263fc 100644 --- a/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs +++ b/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs @@ -53,6 +53,4 @@ private PropertyPathMatch(NameMatcher[] matches) return (true, _matches.Length); } - - internal static bool DefaultPropertyNameEquals(string value, string? other) => string.Equals(value, other, DefaultComparison); } diff --git a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs index 87f48a3..433231b 100644 --- a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs +++ b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs @@ -12,12 +12,14 @@ internal sealed class ShapeWalker private readonly JsonShape _root; private readonly JsonShape _unknown; private readonly bool _keepNulls; + private readonly bool? _ignoreCase; public ShapeWalker(JsonShape root, JsonShapeOptions options) { - root.Seal([]); + root.Freeze(); _root = root; _keepNulls = options.KeepNulls; + _ignoreCase = options.PropertyNameCaseInsensitive; _unknown = options.Unknown switch { UnknownMemberPolicy.Descend => JsonShape.UnknownDescend, @@ -96,7 +98,7 @@ private void WriteObject(ref Utf8JsonReader reader, JsonWriter writer, ref Prope case PropertyName: propPath.AddPropertyName(ref reader); var name = propPath.CurrentUtf8; - var child = values ?? shape!.Find(name) ?? _unknown; + var child = values ?? shape!.Find(name, _ignoreCase ?? propPath.PropertyNameCaseInsensitive) ?? _unknown; if (!reader.Read()) { propPath.RemovePropertyName(); diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs b/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs index 8b63428..cd7f396 100644 --- a/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs +++ b/DragoAnt.System.Text.Json.Observer/Strategies/NameMatcher.cs @@ -9,50 +9,53 @@ internal abstract class NameMatcher { public static readonly NameMatcher Never = new FuncNameMatcher(_ => false); - public abstract bool MatchString(string? name); + public abstract string Describe(); - public virtual bool Match(ref PropertyPath path, int index) => MatchString(path.GetPropertyName(index)); + public bool MatchString(string? name) => MatchString(name, PropertyPathMatch.DefaultComparison); - public static NameMatcher Exact(string pattern) => - IsAscii(pattern) ? new AsciiNameMatcher(AsciiNameMatcher.Mode.Equals, pattern) : new FuncNameMatcher(v => PropertyPathMatch.DefaultPropertyNameEquals(pattern, v)); + public abstract bool MatchString(string? name, StringComparison comparison); - public static NameMatcher StartsWith(string pattern) => - IsAscii(pattern) - ? new AsciiNameMatcher(AsciiNameMatcher.Mode.StartsWith, pattern) - : new FuncNameMatcher(v => v?.StartsWith(pattern, PropertyPathMatch.DefaultComparison) == true); + public virtual bool Match(ref PropertyPath path, int index) => MatchString(path.GetPropertyName(index), ComparisonOf(ref path)); - public static NameMatcher EndsWith(string pattern) => - IsAscii(pattern) - ? new AsciiNameMatcher(AsciiNameMatcher.Mode.EndsWith, pattern) - : new FuncNameMatcher(v => v?.EndsWith(pattern, PropertyPathMatch.DefaultComparison) == true); + private protected static StringComparison ComparisonOf(ref PropertyPath path) => + path.PropertyNameCaseInsensitive ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - public static NameMatcher Contains(string pattern) => - IsAscii(pattern) - ? new AsciiNameMatcher(AsciiNameMatcher.Mode.Contains, pattern) - : new FuncNameMatcher(v => v?.Contains(pattern, PropertyPathMatch.DefaultComparison) == true); + public static NameMatcher Exact(string pattern) => new TextNameMatcher(TextNameMatcher.Mode.Equals, pattern); - public static NameMatcher OneOf(string[] names) => new OneOfNameMatcher(names); + public static NameMatcher StartsWith(string pattern) => new TextNameMatcher(TextNameMatcher.Mode.StartsWith, pattern); + + public static NameMatcher EndsWith(string pattern) => new TextNameMatcher(TextNameMatcher.Mode.EndsWith, pattern); - private static bool IsAscii(string value) => Ascii.IsValid(value); + public static NameMatcher Contains(string pattern) => new TextNameMatcher(TextNameMatcher.Mode.Contains, pattern); + + public static NameMatcher OneOf(string[] names) => new OneOfNameMatcher(names); - internal sealed class FuncNameMatcher(Func match) : NameMatcher + internal sealed class FuncNameMatcher(Func match, string? description = null) : NameMatcher { - public override bool MatchString(string? name) => match(name); + public override string Describe() => description ?? "custom name test"; + + public override bool MatchString(string? name, StringComparison comparison) => match(name); } /// - /// ASCII pattern compared to ASCII names byte by byte; any other name falls back to the string comparison. + /// Pattern compared byte by byte when both it and the name are ASCII; any other name falls back to the string comparison. /// - private sealed class AsciiNameMatcher(AsciiNameMatcher.Mode mode, string pattern) : NameMatcher + private sealed class TextNameMatcher(TextNameMatcher.Mode mode, string pattern) : NameMatcher { - private readonly byte[] _utf8 = Encoding.ASCII.GetBytes(pattern); + private readonly byte[]? _ascii = Ascii.IsValid(pattern) ? Encoding.ASCII.GetBytes(pattern) : null; - public override bool MatchString(string? name) => name is not null && mode switch + public override string Describe() => mode switch { - Mode.Equals => string.Equals(name, pattern, PropertyPathMatch.DefaultComparison), - Mode.StartsWith => name.StartsWith(pattern, PropertyPathMatch.DefaultComparison), - Mode.EndsWith => name.EndsWith(pattern, PropertyPathMatch.DefaultComparison), - _ => name.Contains(pattern, PropertyPathMatch.DefaultComparison), + Mode.Equals => $"\"{pattern}\"", + _ => $"{mode}(\"{pattern}\")", + }; + + public override bool MatchString(string? name, StringComparison comparison) => name is not null && mode switch + { + Mode.Equals => string.Equals(name, pattern, comparison), + Mode.StartsWith => name.StartsWith(pattern, comparison), + Mode.EndsWith => name.EndsWith(pattern, comparison), + _ => name.Contains(pattern, comparison), }; public override bool Match(ref PropertyPath path, int index) @@ -62,23 +65,29 @@ public override bool Match(ref PropertyPath path, int index) return false; } - if (!Ascii.IsValid(name)) + if (_ascii is null || !Ascii.IsValid(name)) { - return MatchString(path.GetPropertyName(index)); + return MatchString(path.GetPropertyName(index), ComparisonOf(ref path)); } - ReadOnlySpan utf8 = _utf8; + ReadOnlySpan utf8 = _ascii; + var ignoreCase = path.PropertyNameCaseInsensitive; return mode switch { - Mode.Equals => name.Length == utf8.Length && Ascii.EqualsIgnoreCase(name, utf8), - Mode.StartsWith => name.Length >= utf8.Length && Ascii.EqualsIgnoreCase(name[..utf8.Length], utf8), - Mode.EndsWith => name.Length >= utf8.Length && Ascii.EqualsIgnoreCase(name[^utf8.Length..], utf8), - _ => ContainsIgnoreCase(name, utf8), + Mode.Equals => name.Length == utf8.Length && SameText(name, utf8, ignoreCase), + Mode.StartsWith => name.Length >= utf8.Length && SameText(name[..utf8.Length], utf8, ignoreCase), + Mode.EndsWith => name.Length >= utf8.Length && SameText(name[^utf8.Length..], utf8, ignoreCase), + _ => ContainsText(name, utf8, ignoreCase), }; } - private static bool ContainsIgnoreCase(ReadOnlySpan name, ReadOnlySpan value) + private static bool ContainsText(ReadOnlySpan name, ReadOnlySpan value, bool ignoreCase) { + if (!ignoreCase) + { + return name.IndexOf(value) >= 0; + } + for (var i = 0; i + value.Length <= name.Length; i++) { if (Ascii.EqualsIgnoreCase(name.Slice(i, value.Length), value)) @@ -99,18 +108,28 @@ internal enum Mode : byte } } + private static bool SameText(ReadOnlySpan left, ReadOnlySpan right, bool ignoreCase) => + ignoreCase ? Ascii.EqualsIgnoreCase(left, right) : left.SequenceEqual(right); + private sealed class OneOfNameMatcher : NameMatcher { - private readonly HashSet _names; + private readonly string[] _names; + private readonly HashSet _ignoreCase; + private readonly HashSet _exact; private readonly byte[][]? _asciiNames; public OneOfNameMatcher(string[] names) { - _names = new HashSet(names, StringComparer.OrdinalIgnoreCase); + _names = names; + _ignoreCase = new HashSet(names, StringComparer.OrdinalIgnoreCase); + _exact = new HashSet(names, StringComparer.Ordinal); _asciiNames = names.All(n => Ascii.IsValid(n)) ? names.Select(n => Encoding.ASCII.GetBytes(n)).ToArray() : null; } - public override bool MatchString(string? name) => name is not null && _names.Contains(name); + public override string Describe() => $"OneOf({string.Join(", ", _names.Select(n => $"\"{n}\""))})"; + + public override bool MatchString(string? name, StringComparison comparison) => + name is not null && (comparison == StringComparison.Ordinal ? _exact : _ignoreCase).Contains(name); public override bool Match(ref PropertyPath path, int index) { @@ -121,12 +140,13 @@ public override bool Match(ref PropertyPath path, int index) if (_asciiNames is null || !Ascii.IsValid(name)) { - return MatchString(path.GetPropertyName(index)); + return MatchString(path.GetPropertyName(index), ComparisonOf(ref path)); } + var ignoreCase = path.PropertyNameCaseInsensitive; foreach (var candidate in _asciiNames) { - if (candidate.Length == name.Length && Ascii.EqualsIgnoreCase(name, candidate)) + if (candidate.Length == name.Length && SameText(name, candidate, ignoreCase)) { return true; } diff --git a/DragoAnt.System.Text.Json.Observer/Strategies/PropMatches.cs b/DragoAnt.System.Text.Json.Observer/Strategies/PropMatches.cs index 73df9ea..862f6c5 100644 --- a/DragoAnt.System.Text.Json.Observer/Strategies/PropMatches.cs +++ b/DragoAnt.System.Text.Json.Observer/Strategies/PropMatches.cs @@ -29,7 +29,7 @@ public static class PropMatches /// Matches property name by regular expression. /// /// Property name regular expression. - public static PropMatchingStrategy Regex(Regex regex) => new(v => v is not null && regex.IsMatch(v)); + public static PropMatchingStrategy Regex(Regex regex) => new(new NameMatcher.FuncNameMatcher(v => v is not null && regex.IsMatch(v), $"Regex(/{regex}/)")); /// /// Matches property by full name equality. From 08aad1899204953977204f49f1954c855d2f0ca7 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:57:43 +0200 Subject: [PATCH 09/13] Accept ReadOnlySequence input for Mask and Read Mask and Read take an in ReadOnlySequence, so a payload in several buffers, for example from a PipeReader, is masked without being copied into one. A single segment takes the span path; a byte order mark split across segments is still skipped. Values split across segments are copied to pooled buffers instead of arrays. Every masking golden case is replayed in 1-, 3- and 7-byte segments and must match the span output byte for byte; planting a segment-boundary bug in property names fails 61 of those cases. --- .../AllocationTests.cs | 2 +- .../BytesApiTests.cs | 4 +- .../JsonMaskingTests.cs | 8 +- .../JsonShapeTests.cs | 2 +- .../SequenceInputTests.cs | 187 ++++++++++++++++++ .../RunSequenceInputTests.cs | 3 + .../JsonObserver.cs | 111 +++++++++-- .../JsonObserverItem.cs | 2 +- .../JsonWriter.cs | 22 ++- 9 files changed, 318 insertions(+), 23 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/SequenceInputTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunSequenceInputTests.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs index 0749306..938e2d8 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs @@ -44,7 +44,7 @@ public void BytesApi_ReusedOutput_StaysWithinBudget(string shape, int size, long perCall.Should().BeLessThanOrEqualTo(budget, $"{shape} {size} B allocates {perCall} B per call"); } - private static string Payload(string shape, int size) + internal static string Payload(string shape, int size) { var json = new StringBuilder(); var i = 0; diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/BytesApiTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/BytesApiTests.cs index 8544b41..19af04c 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/BytesApiTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/BytesApiTests.cs @@ -8,12 +8,12 @@ public abstract class BytesApiTests { private const string Secret = "S3cr3tV4l"; - private static readonly JsonObserver Observer = JsonObserver.Any( + internal static readonly JsonObserver Observer = JsonObserver.Any( _ => { }, _ => { }, Relative(b => b.Match("password").MaskAny("***").Match("pin").MaskAny("***"), BlockList)); - private static readonly string[] Payloads = + internal static readonly string[] Payloads = [ $$$"""{"user":"bob","password":"{{{Secret}}}","card":{"pin":"{{{Secret}}}","exp":"12/30"},"items":[{"id":1,"password":["{{{Secret}}}",{"x":"{{{Secret}}}"}]},{"id":2,"note":"ok"}],"active":true,"amount":1.5e3}""", $$$$$"""[{"password":{"value":"{{{{{Secret}}}}}","deep":[1,2,{"s":"{{{{{Secret}}}}}"}]}},null,"text",[{"pin":12345678}],{"a":{"b":{"c":{"password":"{{{{{Secret}}}}}"}}}}]""", diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonMaskingTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonMaskingTests.cs index 1ccd0b6..2954a83 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonMaskingTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/JsonMaskingTests.cs @@ -23,7 +23,7 @@ public JsonMaskingTests(ITestOutputHelper outputHelper) private readonly JsonObserver _requestMasking = GetRequestMasking(BlockList); - private static JsonObserver GetRequestMasking(JsonObserverValueDelegate defaultValuePolicy) + internal static JsonObserver GetRequestMasking(JsonObserverValueDelegate defaultValuePolicy) { return JsonObserver.Obj(Relative(policyBuilder => policyBuilder .Match(PropMatches.EndsWith("card"), "saved", "id").MaskStr(MaskingRules.CustomerId) @@ -48,7 +48,7 @@ private static JsonObserver GetRequestMasking(JsonObserverValueDelegate defaultValuePolicy) + internal static JsonObserver GetRequestUnmasking(JsonObserverValueDelegate defaultValuePolicy) { return JsonObserver.Obj(b => b .Match("routing").Obj(sb => sb.Match("method").Unmasked()), @@ -63,7 +63,7 @@ private static JsonObserver GetRequestUnmasking(JsonObserverValueDelegate SensitiveValues = new() + internal static readonly Dictionary SensitiveValues = new() { { "cardId", "0c7ed9e5-1c7f-42bd-9efd-e267edd17e57" }, { "userEntered", "Excepturi quia voluptatem." }, @@ -84,7 +84,7 @@ private static JsonObserver GetRequestUnmasking(JsonObserverValueDelegate + internal static JsonObserver Observer(JsonSerializerOptions? options = null, JsonShapeOptions? shapeOptions = null) => JsonObserver.FromShape( JsonShape.FromTypeInfo((options ?? Web).GetTypeInfo(typeof(Customer)), Classify), shapeOptions); diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/SequenceInputTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/SequenceInputTests.cs new file mode 100644 index 0000000..0a5fcec --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/SequenceInputTests.cs @@ -0,0 +1,187 @@ +using System.Buffers; +using System.Text; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +/// +/// Every masking case replayed as a multi-segment sequence: a name, string or number split across segments must be +/// handled exactly like the same bytes in one span. +/// +public abstract class SequenceInputTests +{ + private static readonly PropMatchingStrategy AnyItem = new(_ => true); + + private static readonly JsonObserver Tags = JsonObserver.Obj(Relative(b => b + .Match("full").MaskAny(MaskTag.Full) + .Match("last4").MaskAny(MaskTag.Last4) + .Match("hash").MaskAny(MaskTag.Hash) + .Match("omit").MaskAny(MaskKind.Omit), + BlockList)); + + private static readonly JsonObserver Nested = JsonObserver.Obj( + root => root.Match("lines").Array(l => l.Obj(x => x.Match("qty").MaskAny("***").Match("sku").MaskStr((v, _) => v + "!"))), + Relative(b => b.Match("lines", AnyItem, "note").MaskRawValue((v, _) => "<" + v + ">"), BlockList)); + + private static readonly JsonObserver NonAscii = JsonObserver.Obj(Relative(b => b + .Match("пароль").MaskAny("1") + .Match(PropMatches.Contains("ключ")).MaskInt((v, _) => $"{v}") + .Match("password").MaskAny("2"), + AllowList)); + + private static readonly Dictionary Cases = BuildCases(); + + private static Dictionary BuildCases() + { + var cases = new Dictionary + { + ["golden-request"] = (JsonMaskingTests.GetRequestMasking(BlockList), JsonMaskingTests.TestJson, null), + ["golden-ignore-nulls"] = (JsonMaskingTests.GetRequestUnmasking(NullList), JsonMaskingTests.TestJson, new JsonObserverOptions(IgnoreNulls: true, Indented: true)), + ["tags"] = (Tags, """{"full":{"a":[1,2]},"last4":"4111111111111111","hash":"S3cr3t","omit":12.5e3,"x":"y"}""", null), + ["shape"] = (JsonShapeTests.Observer(), """{"id":1,"orders":[{"sku":"A1","secretCode":"x","extra":1}],"byCode":{"K1":{"sku":"B"}},"card":"4111111111111111","e_mail":"a@b.c","unknown":{"deep":true}}""", null), + ["nested-array"] = (Nested, """{"lines":[{"qty":5,"sku":"A","note":"n\"1"},{"qty":-7.25,"sku":"Bé"}],"total":12}""", null), + ["non-ascii"] = (NonAscii, """{"пароль":"x","мой ключ":42,"password":"y","Ж":"z"}""", null), + ["escaped-long"] = (BytesApiTests.Observer, "{\"password\":\"" + new string('s', 300) + "\\n\\u00e9\",\"n" + new string('m', 300) + "\":" + new string('9', 40) + "}", null), + ["bom"] = (BytesApiTests.Observer, "{\"password\":\"x\",\"ok\":true}", null), + ["truncated"] = (BytesApiTests.Observer, """{"user":"bob","password":"S3cr3t","card":{"pin":"123""", null), + ["invalid"] = (BytesApiTests.Observer, """{"user":"bob",,"password":"x"}""", null), + ["not-json"] = (BytesApiTests.Observer, " 42", null), + ["max-output"] = (BytesApiTests.Observer, BytesApiTests.Payloads[0], new JsonObserverOptions(MaxOutputBytes: 60)), + ["max-value"] = (BytesApiTests.Observer, """{"text":"éééééééé","password":"x"}""", new JsonObserverOptions(MaxValueBytes: 5)), + ["case-sensitive"] = (NonAscii, """{"PASSWORD":"x","password":"y"}""", new JsonObserverOptions(PropertyNameCaseInsensitive: false)), + ["nested-17"] = (BytesApiTests.Observer, NestingTests.Nested(17).Replace("password", "pin", StringComparison.Ordinal), null), + }; + + for (var i = 0; i < BytesApiTests.Payloads.Length; i++) + { + cases[$"bytes-payload-{i}"] = (BytesApiTests.Observer, BytesApiTests.Payloads[i], null); + } + + foreach (var shape in new[] { "flat", "nested", "array" }) + { + cases[$"allocation-{shape}"] = (BytesApiTests.Observer, AllocationTests.Payload(shape, 2048), null); + } + + return cases; + } + + public static TheoryData Replays() + { + var data = new TheoryData(); + foreach (var name in Cases.Keys) + { + data.Add(name, 1); + data.Add(name, 3); + data.Add(name, 7); + } + + return data; + } + + internal static ReadOnlySequence Split(byte[] utf8, int segmentSize) + { + if (utf8.Length == 0) + { + return ReadOnlySequence.Empty; + } + + Segment? first = null; + Segment? last = null; + for (var offset = 0; offset < utf8.Length; offset += segmentSize) + { + var memory = utf8.AsMemory(offset, Math.Min(segmentSize, utf8.Length - offset)); + last = last is null ? first = new Segment(memory, 0) : last.Append(memory); + } + + return new ReadOnlySequence(first!, 0, last!, last!.Memory.Length); + } + + [Theory] + [MemberData(nameof(Replays))] + public void Mask_SplitIntoSegments_SameAsSpan(string name, int segmentSize) + { + var (observer, json, options) = Cases[name]; + var utf8 = Encoding.UTF8.GetBytes(json); + var spanOutput = new ArrayBufferWriter(); + var sequenceOutput = new ArrayBufferWriter(); + + var spanResult = observer.Mask(utf8, spanOutput, options); + var sequenceResult = observer.Mask(Split(utf8, segmentSize), sequenceOutput, options); + + sequenceResult.Should().Be(spanResult, name); + Encoding.UTF8.GetString(sequenceOutput.WrittenSpan).Should().Be(Encoding.UTF8.GetString(spanOutput.WrittenSpan), name); + } + + [Fact] + public void Mask_SingleSegmentAndEmpty_SameAsSpan() + { + var utf8 = Encoding.UTF8.GetBytes(BytesApiTests.Payloads[0]); + var spanOutput = new ArrayBufferWriter(); + var sequenceOutput = new ArrayBufferWriter(); + + BytesApiTests.Observer.Mask(new ReadOnlySequence(utf8), sequenceOutput) + .Should().Be(BytesApiTests.Observer.Mask(utf8, spanOutput)); + sequenceOutput.WrittenSpan.SequenceEqual(spanOutput.WrittenSpan).Should().BeTrue(); + BytesApiTests.Observer.Mask(ReadOnlySequence.Empty, new ArrayBufferWriter()) + .Should().Be(new MaskResult(MaskStatus.NotJson, 0, 0)); + BytesApiTests.Observer.Mask(Split([0xEF, 0xBB], 1), new ArrayBufferWriter()) + .Should().Be(BytesApiTests.Observer.Mask([0xEF, 0xBB], new ArrayBufferWriter())); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + public void Read_SplitIntoSegments_ReadsSameValues(int segmentSize) + { + var observer = JsonObserver.Obj(ReadRules(b => b + .Match("id").ReadStr((v, c) => c.Values.Add($"id={v}")) + .Match("n").ReadDecimal((v, c) => c.Values.Add($"n={v}")) + .Match("raw").ReadRaw((v, c) => c.Values.Add($"raw={v}")) + .Match("ok").ReadBool((v, c) => c.Values.Add($"ok={v}")))); + var utf8 = Encoding.UTF8.GetBytes("""{"a":[{"id":"xé-1","n":12345.678},{"raw":"r\"q","ok":true}],"id":""" + "\"" + new string('z', 50) + "\"}"); + var fromSpan = new Extracted(); + var fromSequence = new Extracted(); + + var spanResult = observer.Read(utf8, fromSpan); + var sequenceResult = observer.Read(Split(utf8, segmentSize), fromSequence); + + sequenceResult.Should().Be(spanResult); + fromSequence.Values.Should().Equal(fromSpan.Values); + fromSpan.Values.Should().HaveCount(5); + } + + [Fact] + public void Read_SingleSegment_UsesSpanPath() + { + var context = new Extracted(); + var observer = JsonObserver.Obj(ReadRules(b => b.Match("id").ReadStr((v, c) => c.Values.Add(v!)))); + + observer.Read(new ReadOnlySequence("""{"id":"a"}"""u8.ToArray()), context).Status.Should().Be(MaskStatus.Masked); + context.Values.Should().Equal("a"); + } + + private static JsonObserverValueDelegate ReadRules(Action> init) => + JsonObserverValuePolicies.Relative(init, JsonObserverValuePolicies.BlockList); + + public sealed class Extracted + { + public List Values { get; } = []; + } + + private sealed class Segment : ReadOnlySequenceSegment + { + public Segment(ReadOnlyMemory memory, long runningIndex) + { + Memory = memory; + RunningIndex = runningIndex; + } + + public Segment Append(ReadOnlyMemory memory) + { + var next = new Segment(memory, RunningIndex + Memory.Length); + Next = next; + return next; + } + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunSequenceInputTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunSequenceInputTests.cs new file mode 100644 index 0000000..bf95a73 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunSequenceInputTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunSequenceInputTests : Shared.SequenceInputTests; diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs index 21d4389..68b8aba 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs @@ -164,6 +164,19 @@ private JsonObserver(JsonObserver masking) /// Status, bytes written and the input offset where reading stopped. public MaskResult Mask(ReadOnlySpan utf8, IBufferWriter output, JsonObserverOptions? options = null) => _masking.Mask(utf8, output, JsonObserveringEmptyContext.Instance, options); + + /// + /// Masks a UTF-8 JSON payload held in several buffers, for example read from a PipeReader, into + /// without copying it into one buffer first. Never throws, and writes exactly what + /// writes for the same bytes, + /// however they are split. + /// + /// UTF-8 JSON payload; it may be cut short, for example by a size limit. A leading byte order mark is skipped. + /// Receives the masked JSON. + /// Limits and output settings; when omitted. + /// Status, bytes written and the input offset where reading stopped. + public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter output, JsonObserverOptions? options = null) + => _masking.Mask(utf8, output, JsonObserveringEmptyContext.Instance, options); } /// @@ -246,9 +259,39 @@ internal JsonObserver(JsonObserverDelegate maskDelegate) public MaskResult Mask(ReadOnlySpan utf8, IBufferWriter output, TContext context, JsonObserverOptions? options = null) { options ??= JsonObserverOptions.Default; + utf8 = SkipBom(utf8); + var reader = CreateReader(utf8, options); + return Mask(ref reader, utf8, output, context, options); + } + + /// + /// Masks a UTF-8 JSON payload held in several buffers, for example read from a PipeReader, into + /// and hands values to , without copying it into one buffer first. + /// Never throws, and behaves exactly like + /// for the same bytes, however they are split. + /// + /// UTF-8 JSON payload; it may be cut short, for example by a size limit. A leading byte order mark is skipped. + /// Receives the masked JSON. + /// Receives the values read rules extract. + /// Limits and output settings; when omitted. + /// Status, bytes written and the input offset where reading stopped. + public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter output, TContext context, JsonObserverOptions? options = null) + { + if (utf8.IsSingleSegment) + { + return Mask(utf8.FirstSpan, output, context, options); + } + + options ??= JsonObserverOptions.Default; + var reader = CreateReader(SkipBom(utf8), options); + return Mask(ref reader, default, output, context, options); + } + + private MaskResult Mask(ref Utf8JsonReader reader, ReadOnlySpan input, IBufferWriter output, TContext context, JsonObserverOptions options) + { using var bounded = new BoundedJsonWriter(options); using var ignoreNulls = options.IgnoreNulls ? new IgnoreNullsJsonWriter(bounded) : null; - var (status, failedAt) = Observe(utf8, (JsonWriter?)ignoreNulls ?? bounded, context, options); + var (status, failedAt) = Observe(ref reader, input, (JsonWriter?)ignoreNulls ?? bounded, context, options); if (status == MaskStatus.NotJson) { return new MaskResult(MaskStatus.NotJson, 0, 0); @@ -314,24 +357,70 @@ public MaskResult Read(string? json, TContext context, JsonObserverOptions? opti /// Status and the input offset where reading stopped. public MaskResult Read(ReadOnlySpan utf8, TContext context, JsonObserverOptions? options = null) { - var (status, failedAt) = Observe(utf8, JsonWriter.Empty, context, options ?? JsonObserverOptions.Default); + options ??= JsonObserverOptions.Default; + utf8 = SkipBom(utf8); + var reader = CreateReader(utf8, options); + var (status, failedAt) = Observe(ref reader, utf8, JsonWriter.Empty, context, options); return new MaskResult(status, 0, failedAt); } - private (MaskStatus Status, long FailedAt) Observe(ReadOnlySpan utf8, JsonWriter writer, TContext context, JsonObserverOptions options) + /// + /// Hands values of a UTF-8 JSON payload held in several buffers to without writing + /// anything or copying the payload into one buffer. Never throws, and behaves exactly like + /// for the same bytes, however they are split. + /// + /// UTF-8 JSON payload; it may be cut short. A leading byte order mark is skipped. + /// Receives the values read rules extract. + /// Limits; when omitted. + /// Status and the input offset where reading stopped. + public MaskResult Read(in ReadOnlySequence utf8, TContext context, JsonObserverOptions? options = null) { - if (utf8.StartsWith(Utf8Bom)) + if (utf8.IsSingleSegment) { - utf8 = utf8[Utf8Bom.Length..]; + return Read(utf8.FirstSpan, context, options); } - var reader = new Utf8JsonReader(utf8, isFinalBlock: false, new JsonReaderState(new JsonReaderOptions + options ??= JsonObserverOptions.Default; + var reader = CreateReader(SkipBom(utf8), options); + var (status, failedAt) = Observe(ref reader, default, JsonWriter.Empty, context, options); + return new MaskResult(status, 0, failedAt); + } + + private static ReadOnlySpan SkipBom(ReadOnlySpan utf8) => utf8.StartsWith(Utf8Bom) ? utf8[Utf8Bom.Length..] : utf8; + + private static ReadOnlySequence SkipBom(in ReadOnlySequence utf8) + { + if (utf8.Length < Utf8Bom.Length) { - CommentHandling = JsonCommentHandling.Skip, - AllowTrailingCommas = true, - MaxDepth = Math.Max(options.MaxDepth, 1), - })); - var propPath = new PropertyPath(_maxDepth, utf8) { PropertyNameCaseInsensitive = options.PropertyNameCaseInsensitive }; + return utf8; + } + + Span head = stackalloc byte[3]; + utf8.Slice(0, Utf8Bom.Length).CopyTo(head); + return head.SequenceEqual(Utf8Bom) ? utf8.Slice(Utf8Bom.Length) : utf8; + } + + private static JsonReaderState ReaderState(JsonObserverOptions options) => new(new JsonReaderOptions + { + CommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + MaxDepth = Math.Max(options.MaxDepth, 1), + }); + + private static Utf8JsonReader CreateReader(ReadOnlySpan utf8, JsonObserverOptions options) => + new(utf8, isFinalBlock: false, ReaderState(options)); + + private static Utf8JsonReader CreateReader(in ReadOnlySequence utf8, JsonObserverOptions options) => + new(utf8, isFinalBlock: false, ReaderState(options)); + + private (MaskStatus Status, long FailedAt) Observe( + ref Utf8JsonReader reader, + ReadOnlySpan input, + JsonWriter writer, + TContext context, + JsonObserverOptions options) + { + var propPath = new PropertyPath(_maxDepth, input) { PropertyNameCaseInsensitive = options.PropertyNameCaseInsensitive }; try { if (!reader.Read() || reader.TokenType is not (JsonTokenType.StartObject or JsonTokenType.StartArray)) diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs index a56e897..2d32968 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs @@ -118,7 +118,7 @@ public static JsonObserverDelegate ReadBool(Action re public static JsonObserverDelegate ReadRaw(Action read, JsonObserverValueDelegate? valuePolicy) => ApplyReadPolicy( (ref Utf8JsonReader reader, TContext context) => - read(Encoding.UTF8.GetString(reader.HasValueSequence ? reader.ValueSequence.ToArray() : reader.ValueSpan), context), + read(reader.HasValueSequence ? Encoding.UTF8.GetString(reader.ValueSequence) : Encoding.UTF8.GetString(reader.ValueSpan), context), static type => type is JsonTokenType.String or Number or True or False or Null, valuePolicy); diff --git a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs index 9aae3bd..1808da9 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs @@ -163,13 +163,29 @@ internal void CopyRawValue(ref Utf8JsonReader reader) return; } - if (reader.HasValueSequence) + if (!reader.HasValueSequence) { - WriteRawValue(reader.ValueSequence.ToArray()); + WriteRawValue(reader.ValueSpan); return; } - WriteRawValue(reader.ValueSpan); + var length = checked((int)reader.ValueSequence.Length); + byte[]? rented = null; + var buffer = length <= StackallocThreshold + ? stackalloc byte[StackallocThreshold] + : rented = ArrayPool.Shared.Rent(length); + try + { + reader.ValueSequence.CopyTo(buffer); + WriteRawValue(buffer[..length]); + } + finally + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented, clearArray: true); + } + } } private sealed class EmptyJsonWriter : JsonWriter From 7ff6a6673c4665b895368cf095fe141adcff99c7 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:03:38 +0200 Subject: [PATCH 10/13] Reuse the per-call writers so the bytes API allocates nothing The bounded writer (with its Utf8JsonWriter), the ignore-nulls writer and the string API's output buffer are kept per thread and reset per call; their arrays still come from the pool and are returned after every call, and a nested call on the same thread gets writers of its own. A writer is reused only while the escaping, indentation and depth settings match. The allocation test now pins 0 bytes per warm call for the span, sequence, ignore-nulls and read paths (240 to 288 bytes before). --- .../AllocationTests.cs | 75 ++++++++++++++++--- .../BoundedJsonWriter.cs | 70 ++++++++++++++--- .../JsonObserver.cs | 8 +- .../JsonWriter.cs | 72 ++++++++++++------ .../PooledBufferWriter.cs | 57 +++++++++++++- 5 files changed, 230 insertions(+), 52 deletions(-) diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs index 938e2d8..ee5ceb2 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs @@ -11,37 +11,88 @@ public abstract class AllocationTests _ => { }, Relative(b => b.Match("password").MaskAny("***").Match("card", "number").MaskStr("***"), BlockList)); - public static TheoryData Budgets => new() + private static readonly JsonObserver Reader = JsonObserver.Any( + _ => { }, + _ => { }, + JsonObserverValuePolicies.Relative( + b => b.Match("password").MaskAny("***"), + JsonObserverValuePolicies.BlockList)); + + private static readonly JsonObserverOptions IgnoreNulls = new(IgnoreNulls: true); + + public static TheoryData Budgets() { - { "flat", 1024, 512 }, - { "flat", 64 * 1024, 512 }, - { "nested", 8 * 1024, 512 }, - { "array", 8 * 1024, 512 }, - }; + var data = new TheoryData(); + foreach (var api in new[] { "span", "sequence", "ignore-nulls", "read" }) + { + data.Add("flat", 1024, api); + data.Add("flat", 64 * 1024, api); + data.Add("nested", 8 * 1024, api); + data.Add("array", 8 * 1024, api); + } + return data; + } + + /// + /// The bytes API allocates nothing per call once warm: writers are reused per thread and buffers come from the pool. + /// [Theory] [MemberData(nameof(Budgets))] - public void BytesApi_ReusedOutput_StaysWithinBudget(string shape, int size, long budget) + public void BytesApi_ReusedOutput_AllocatesNothing(string shape, int size, string api) { var utf8 = Encoding.UTF8.GetBytes(Payload(shape, size)); + var sequence = SequenceInputTests.Split(utf8, 4096); var output = new ArrayBufferWriter(utf8.Length * 2); - for (var i = 0; i < 20; i++) + + void Call() { output.ResetWrittenCount(); - Observer.Mask(utf8, output); + _ = api switch + { + "span" => Observer.Mask(utf8, output), + "sequence" => Observer.Mask(sequence, output), + "ignore-nulls" => Observer.Mask(utf8, output, IgnoreNulls), + _ => Reader.Read(utf8, JsonObserveringEmptyContext.Instance), + }; + } + + for (var i = 0; i < 20; i++) + { + Call(); } const int calls = 50; var before = GC.GetAllocatedBytesForCurrentThread(); for (var i = 0; i < calls; i++) { - output.ResetWrittenCount(); - Observer.Mask(utf8, output); + Call(); } var perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / calls; - perCall.Should().BeLessThanOrEqualTo(budget, $"{shape} {size} B allocates {perCall} B per call"); + perCall.Should().Be(0, $"{api} {shape} {size} B allocates {perCall} B per call"); + } + + [Fact] + public void NestedCallOnSameThread_GetsItsOwnWriter() + { + var inner = JsonObserver.Obj(Relative(b => b.Match("pin").MaskAny("#"), BlockList)); + var outer = JsonObserver.Obj(b => b.Match("payload").MaskStr((v, _) => inner.Mask(v)), BlockList); + + outer.Mask("""{"payload":"{\"pin\":1,\"x\":2}","y":3}""") + .Should().Be("""{"payload":"{\"pin\":\"#\",\"x\":2}","y":3}"""); + } + + [Fact] + public void WriterSettingsChange_BetweenCalls_Respected() + { + const string json = """{"a":"é<","password":"x"}"""; + + Observer.Mask(json).Should().Be("""{"a":"é<","password":"***"}"""); + Observer.Mask(json, new JsonObserverOptions(RelaxedEscaping: false)).Should().NotContain("é").And.Contain((char)92 + "u003C").And.EndWith(",\"password\":\"***\"}"); + Observer.Mask(json, new JsonObserverOptions(Indented: true)).Should().Contain(Environment.NewLine); + Observer.Mask(json).Should().Be("""{"a":"é<","password":"***"}"""); } internal static string Payload(string shape, int size) diff --git a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs index abf4642..4e63b3f 100644 --- a/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/BoundedJsonWriter.cs @@ -12,29 +12,36 @@ namespace DragoAnt.System.Text.Json.Observer; /// internal sealed class BoundedJsonWriter : JsonWriter, IDisposable { + [ThreadStatic] + private static BoundedJsonWriter? t_cached; + private static ReadOnlySpan Ellipsis => [0xE2, 0x80, 0xA6]; private readonly PooledBufferWriter _buffer; private readonly Utf8JsonWriter _writer; - private readonly int _maxOutputBytes; - private readonly int _maxValueBytes; - private bool[] _isArray = ArrayPool.Shared.Rent(64); + private readonly bool _relaxedEscaping; + private readonly bool _indented; + private readonly int _writerMaxDepth; + private int _maxOutputBytes; + private int _maxValueBytes; + private bool[] _isArray = []; private int _depth; private int _safeLength; private int _safeDepth; - public BoundedJsonWriter(JsonObserverOptions options) + private BoundedJsonWriter(JsonObserverOptions options) { - Options = options; - _maxOutputBytes = Math.Max(options.MaxOutputBytes, 0); - _maxValueBytes = Math.Max(options.MaxValueBytes, 0); + _relaxedEscaping = options.RelaxedEscaping; + _indented = options.Indented; + _writerMaxDepth = WriterMaxDepth(options); _buffer = new PooledBufferWriter(); _writer = new Utf8JsonWriter(_buffer, new JsonWriterOptions { Encoder = options.RelaxedEscaping ? JavaScriptEncoder.UnsafeRelaxedJsonEscaping : null, - MaxDepth = Math.Min(Math.Max(options.MaxDepth, 1), int.MaxValue - 1) + 1, + MaxDepth = _writerMaxDepth, Indented = options.Indented, }); + Start(options); } public bool Exhausted { get; private set; } @@ -44,7 +51,44 @@ public BoundedJsonWriter(JsonObserverOptions options) /// public bool ValuesTruncated { get; private set; } - internal override JsonObserverOptions Options { get; } + internal override JsonObserverOptions Options => _options; + + private JsonObserverOptions _options = null!; + + /// + /// Takes this thread's spare writer when its fixed settings fit , or creates one; + /// gives it back. A nested call on the same thread gets a writer of its own. + /// + public static BoundedJsonWriter Rent(JsonObserverOptions options) + { + var cached = t_cached; + if (cached is null || cached._relaxedEscaping != options.RelaxedEscaping || cached._indented != options.Indented || + cached._writerMaxDepth != WriterMaxDepth(options)) + { + return new BoundedJsonWriter(options); + } + + t_cached = null; + cached._buffer.Reset(); + cached._writer.Reset(cached._buffer); + cached.Start(options); + return cached; + } + + private static int WriterMaxDepth(JsonObserverOptions options) => Math.Min(Math.Max(options.MaxDepth, 1), int.MaxValue - 1) + 1; + + private void Start(JsonObserverOptions options) + { + _options = options; + _maxOutputBytes = Math.Max(options.MaxOutputBytes, 0); + _maxValueBytes = Math.Max(options.MaxValueBytes, 0); + _isArray = ArrayPool.Shared.Rent(64); + _depth = 0; + _safeLength = 0; + _safeDepth = 0; + Exhausted = false; + ValuesTruncated = false; + } internal override bool Stopped => Exhausted; @@ -303,9 +347,12 @@ public int CopyTo(IBufferWriter output, bool complete) return length; } + /// + /// Returns the pooled buffers and keeps the writer as this thread's spare. + /// public void Dispose() { - _writer.Dispose(); + _writer.Reset(_buffer); _buffer.Dispose(); var isArray = _isArray; _isArray = []; @@ -313,6 +360,9 @@ public void Dispose() { ArrayPool.Shared.Return(isArray); } + + _options = JsonObserverOptions.Default; + t_cached = this; } private void Start(bool isArray) diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs index 68b8aba..c86c3d9 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs @@ -225,11 +225,12 @@ internal JsonObserver(JsonObserverDelegate maskDelegate) } byte[]? input = null; + PooledBufferWriter? output = null; try { input = ArrayPool.Shared.Rent(Encoding.UTF8.GetByteCount(json)); var utf8 = input.AsSpan(0, Encoding.UTF8.GetBytes(json, input)); - using var output = new PooledBufferWriter(Math.Clamp(utf8.Length, 256, 64 * 1024)); + output = PooledBufferWriter.Rent(Math.Clamp(utf8.Length, 256, 64 * 1024)); result = Mask(utf8, output, context, options); return Encoding.UTF8.GetString(output.WrittenSpan); } @@ -240,6 +241,7 @@ internal JsonObserver(JsonObserverDelegate maskDelegate) } finally { + output?.Return(); if (input is not null) { ArrayPool.Shared.Return(input, clearArray: true); @@ -289,8 +291,8 @@ public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter outpu private MaskResult Mask(ref Utf8JsonReader reader, ReadOnlySpan input, IBufferWriter output, TContext context, JsonObserverOptions options) { - using var bounded = new BoundedJsonWriter(options); - using var ignoreNulls = options.IgnoreNulls ? new IgnoreNullsJsonWriter(bounded) : null; + using var bounded = BoundedJsonWriter.Rent(options); + using var ignoreNulls = options.IgnoreNulls ? IgnoreNullsJsonWriter.Rent(bounded) : null; var (status, failedAt) = Observe(ref reader, input, (JsonWriter?)ignoreNulls ?? bounded, context, options); if (status == MaskStatus.NotJson) { diff --git a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs index 1808da9..cfc4084 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonWriter.cs @@ -263,16 +263,20 @@ public override void WriteEndArray() /// /// Drops null values: a property name and an opened container are written only once a non-null value follows. /// -internal sealed class IgnoreNullsJsonWriter(JsonWriter inner) : JsonWriter, IDisposable +internal sealed class IgnoreNullsJsonWriter : JsonWriter, IDisposable { - private Pending[] _pending = ArrayPool.Shared.Rent(16); - private byte[] _names = ArrayPool.Shared.Rent(256); + [ThreadStatic] + private static IgnoreNullsJsonWriter? t_cached; + + private JsonWriter _inner = Empty; + private Pending[] _pending = []; + private byte[] _names = []; private int _count; private int _namesUsed; - internal override JsonObserverOptions Options => inner.Options; + internal override JsonObserverOptions Options => _inner.Options; - internal override bool Stopped => inner.Stopped; + internal override bool Stopped => _inner.Stopped; public override void WriteNullValue() { @@ -283,14 +287,14 @@ public override void WriteNullValue() else if (_count == 0 || !_pending[_count - 1].IsArray) { Flush(); - inner.WriteNullValue(); + _inner.WriteNullValue(); } } public override void WriteBooleanValue(bool value) { Flush(); - inner.WriteBooleanValue(value); + _inner.WriteBooleanValue(value); } public override void WriteStringValue(string? value) @@ -302,49 +306,49 @@ public override void WriteStringValue(string? value) } Flush(); - inner.WriteStringValue(value); + _inner.WriteStringValue(value); } public override void WriteStringValue(ReadOnlySpan utf8Value) { Flush(); - inner.WriteStringValue(utf8Value); + _inner.WriteStringValue(utf8Value); } public override void WriteRawValue(ReadOnlySpan utf8Json) { Flush(); - inner.WriteRawValue(utf8Json); + _inner.WriteRawValue(utf8Json); } public override void WriteNumberValue(long value) { Flush(); - inner.WriteNumberValue(value); + _inner.WriteNumberValue(value); } public override void WriteNumberValue(decimal value) { Flush(); - inner.WriteNumberValue(value); + _inner.WriteNumberValue(value); } public override void WriteStringValue(ReadOnlySpan value) { Flush(); - inner.WriteStringValue(value); + _inner.WriteStringValue(value); } public override void WriteBase64StringValue(ReadOnlySpan bytes) { Flush(); - inner.WriteBase64StringValue(bytes); + _inner.WriteBase64StringValue(bytes); } public override void WriteNumberValue(double value) { Flush(); - inner.WriteNumberValue(value); + _inner.WriteNumberValue(value); } public override void WritePropertyName(string propertyName) => WritePropertyName(propertyName.AsSpan()); @@ -386,12 +390,32 @@ private void EnsureNames(int length) public override void WriteEndArray() => End(isArray: true); + /// + /// Takes this thread's spare writer, or creates one, writing to ; gives it back. + /// + public static IgnoreNullsJsonWriter Rent(JsonWriter inner) + { + var writer = t_cached ?? new IgnoreNullsJsonWriter(); + t_cached = null; + writer._inner = inner; + writer._pending = ArrayPool.Shared.Rent(16); + writer._names = ArrayPool.Shared.Rent(256); + writer._count = 0; + writer._namesUsed = 0; + return writer; + } + + /// + /// Returns the pooled buffers and keeps the writer as this thread's spare. + /// public void Dispose() { ArrayPool.Shared.Return(_pending); ArrayPool.Shared.Return(_names, clearArray: true); _pending = []; _names = []; + _inner = Empty; + t_cached = this; } private void End(bool isArray) @@ -413,11 +437,11 @@ private void End(bool isArray) if (isArray) { - inner.WriteEndArray(); + _inner.WriteEndArray(); } else { - inner.WriteEndObject(); + _inner.WriteEndObject(); } if (_count > 0 && _pending[_count - 1].Kind == Kind.Written) @@ -430,13 +454,13 @@ private void WriteEmptyRoot(bool isArray) { if (isArray) { - inner.WriteStartArray(); - inner.WriteEndArray(); + _inner.WriteStartArray(); + _inner.WriteEndArray(); } else { - inner.WriteStartObject(); - inner.WriteEndObject(); + _inner.WriteStartObject(); + _inner.WriteEndObject(); } } @@ -451,14 +475,14 @@ private void Flush() switch (pending.Kind) { case Kind.Name: - inner.WritePropertyName(_names.AsSpan(pending.Start, pending.Length)); + _inner.WritePropertyName(_names.AsSpan(pending.Start, pending.Length)); break; case Kind.Open when pending.IsArray: - inner.WriteStartArray(); + _inner.WriteStartArray(); pending = pending with { Kind = Kind.Written }; continue; case Kind.Open: - inner.WriteStartObject(); + _inner.WriteStartObject(); pending = pending with { Kind = Kind.Written }; continue; default: diff --git a/DragoAnt.System.Text.Json.Observer/PooledBufferWriter.cs b/DragoAnt.System.Text.Json.Observer/PooledBufferWriter.cs index bcf78ef..2bc2528 100644 --- a/DragoAnt.System.Text.Json.Observer/PooledBufferWriter.cs +++ b/DragoAnt.System.Text.Json.Observer/PooledBufferWriter.cs @@ -2,14 +2,60 @@ namespace DragoAnt.System.Text.Json.Observer; -internal sealed class PooledBufferWriter(int initialCapacity = 1024) : IBufferWriter, IDisposable +/// +/// Buffer writer over pooled arrays. returns the array; makes the instance +/// usable again, so a thread can keep one instance and allocate nothing per call. +/// +internal sealed class PooledBufferWriter : IBufferWriter, IDisposable { - private byte[] _buffer = ArrayPool.Shared.Rent(initialCapacity); + [ThreadStatic] + private static PooledBufferWriter? t_cached; + + private byte[] _buffer; + + public PooledBufferWriter(int initialCapacity = 1024) + { + _buffer = ArrayPool.Shared.Rent(initialCapacity); + } public int WrittenCount { get; private set; } public ReadOnlySpan WrittenSpan => _buffer.AsSpan(0, WrittenCount); + /// + /// Takes this thread's spare instance, or creates one; give it back with . + /// + public static PooledBufferWriter Rent(int initialCapacity) + { + var cached = t_cached; + if (cached is null) + { + return new PooledBufferWriter(initialCapacity); + } + + t_cached = null; + cached.Reset(initialCapacity); + return cached; + } + + /// + /// Returns the array to the pool and keeps the instance as this thread's spare. + /// + public void Return() + { + Dispose(); + t_cached = this; + } + + public void Reset(int initialCapacity = 1024) + { + WrittenCount = 0; + if (_buffer.Length == 0) + { + _buffer = ArrayPool.Shared.Rent(initialCapacity); + } + } + public void Advance(int count) => WrittenCount += count; public Memory GetMemory(int sizeHint = 0) @@ -28,6 +74,7 @@ public void Dispose() { var buffer = _buffer; _buffer = []; + WrittenCount = 0; if (buffer.Length > 0) { ArrayPool.Shared.Return(buffer); @@ -44,7 +91,11 @@ private void Ensure(int sizeHint) var grown = ArrayPool.Shared.Rent(Math.Max(required, _buffer.Length * 2)); WrittenSpan.CopyTo(grown); - ArrayPool.Shared.Return(_buffer); + if (_buffer.Length > 0) + { + ArrayPool.Shared.Return(_buffer); + } + _buffer = grown; } } From 010beef105005380cbda55022c11574bd2a053b4 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:14:00 +0200 Subject: [PATCH 11/13] Add Explain(path) to tell which rule handles a JSON path JsonObserver.Explain(path, valueKind, options) returns a JsonPathExplanation: the outcome (Unchanged, Masked, Read, Custom, Invalid), the deciding rule, its action and one step per level. It walks the observer's rules with the same matcher the masking pass uses, so absolute, nested, relative and default-policy decisions, case sensitivity and descended unknown containers agree with the output; shape observers explain against their JsonShape. Builders now record a description beside every rule. A golden-payload test checks that every leaf Explain calls Masked is exactly a leaf the mask changed. --- .../DefaultPolicyTests.cs | 2 +- .../ExplainTests.cs | 178 +++++++++++++++++ .../RunExplainTests.cs | 3 + .../Builders/JsonArrayBuilder.cs | 74 ++++--- .../Builders/JsonObjBuilder.cs | 105 +++++----- .../Builders/JsonValuePolicyBuilder.cs | 56 +++--- .../Builders/RuleText.cs | 24 +++ .../JsonObserver.cs | 54 +++++- .../JsonObserverItem.cs | 23 ++- .../JsonPathExplanation.cs | 48 +++++ .../PathExplainer.cs | 183 ++++++++++++++++++ .../PropertyPathMatch.cs | 2 + .../RelativeValuePolicy.cs | 4 + .../RuleExplainer.cs | 140 ++++++++++++++ DragoAnt.System.Text.Json.Observer/RuleSet.cs | 18 ++ .../ShapeWalker.cs | 93 ++++++++- 16 files changed, 889 insertions(+), 118 deletions(-) create mode 100644 DragoAnt.System.Text.Json.Observer.Tests.Shared/ExplainTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer.Tests/RunExplainTests.cs create mode 100644 DragoAnt.System.Text.Json.Observer/Builders/RuleText.cs create mode 100644 DragoAnt.System.Text.Json.Observer/JsonPathExplanation.cs create mode 100644 DragoAnt.System.Text.Json.Observer/PathExplainer.cs create mode 100644 DragoAnt.System.Text.Json.Observer/RuleExplainer.cs create mode 100644 DragoAnt.System.Text.Json.Observer/RuleSet.cs diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/DefaultPolicyTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/DefaultPolicyTests.cs index 06ee4e5..c5712ef 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/DefaultPolicyTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/DefaultPolicyTests.cs @@ -9,7 +9,7 @@ public abstract class DefaultPolicyTests [Fact] public void SharedNestedRule_TwoParentsDifferentDefaults_EachUsesOwn() { - var shared = JsonObserverItem.Obj(b => b.Match("pin").MaskStr((_, _) => "***"), null); + var shared = JsonObserverItem.Obj(b => b.Match("pin").MaskStr((_, _) => "***"), null).Delegate; var blockList = JsonObserver.Obj(b => b.Match("a").Obj(shared), BlockList); var nullList = JsonObserver.Obj(b => b.Match("a").Obj(shared), NullList); diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/ExplainTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ExplainTests.cs new file mode 100644 index 0000000..91d3127 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/ExplainTests.cs @@ -0,0 +1,178 @@ +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +namespace DragoAnt.System.Text.Json.Observer.Tests.Shared; + +public abstract class ExplainTests +{ + private static readonly JsonObserver Lines = JsonObserver.Obj( + root => root + .Match("lines").Array(l => l.Obj(x => x.Match("qty").MaskAny("***").Match("note").ReadStr((_, _) => { }))) + .Match("id").Unmasked() + .Match("custom").MaskValue((ref Utf8JsonReader _, JsonWriter w, JsonObserveringEmptyContext _, ref PropertyPath _) => w.WriteNullValue()), + BlockList); + + [Fact] + public void AbsoluteNestedRule_NamesTheChain() + { + var explanation = Lines.Explain("lines[0].qty", JsonTokenType.Number); + + explanation.Should().BeEquivalentTo(new JsonPathExplanation( + "lines[0].qty", + JsonPathOutcome.Masked, + """Match("lines") > object item > Match("qty")""", + """MaskAny("***")""", + [ + """lines: Match("lines") → Array(...)""", + "lines[0]: object item → Obj(...)", + """lines[0].qty: Match("qty") → MaskAny("***")""", + ])); + explanation.ToString().Should().Be("""lines[0].qty: Masked by Match("lines") > object item > Match("qty") → MaskAny("***")"""); + } + + [Fact] + public void RuleKinds_Outcomes() + { + Lines.Explain("lines[3].sku").Should().Match(e => e.Outcome == JsonPathOutcome.Unchanged && e.Rule == "default policy BlockList"); + Lines.Explain("lines[0].note").Outcome.Should().Be(JsonPathOutcome.Read); + Lines.Explain("id", JsonTokenType.Number).Outcome.Should().Be(JsonPathOutcome.Unchanged); + Lines.Explain("custom").Outcome.Should().Be(JsonPathOutcome.Custom); + Lines.Explain("lines[0].qty", JsonTokenType.Null).Should().Match(e => e.Outcome == JsonPathOutcome.Unchanged && e.Action.EndsWith("keeps null")); + Lines.Explain("other.deep", JsonTokenType.StartObject).Should().Match(e => e.Outcome == JsonPathOutcome.Unchanged && e.Action.Contains("descended")); + Lines.Explain("[0].id").Outcome.Should().Be(JsonPathOutcome.Invalid); + Lines.Explain("$").Rule.Should().Be("root"); + } + + [Fact] + public void RelativeRules_AndDefaults() + { + var observer = JsonObserver.Obj(Relative(rules => rules + .Match(PropMatches.EndsWith("card"), "saved", "id").MaskStr("***") + .Match("card").MaskAny(MaskTag.Last4) + .Match(PropMatches.Contains("email")).MaskStr((v, _) => v), + AllowList)); + + observer.Explain("s.MY_card.saved.id").Should().Match(e => + e.Outcome == JsonPathOutcome.Masked && e.Rule == """relative Match(EndsWith("card"), "saved", "id")""" && e.Action == """MaskStr("***")"""); + observer.Explain("a.card.number").Should().Match(e => + e.Rule == """relative Match("card")""" && e.Action == "MaskAny(MaskTag.Last4) on the whole object"); + observer.Explain("c.workEmail").Action.Should().Be("MaskStr(function)"); + observer.Explain("c.tier").Should().Match(e => e.Rule == "default policy AllowList" && e.Action == "writes \"***\""); + observer.Explain("c.tier", JsonTokenType.Null).Action.Should().Be("keeps null"); + observer.Explain("c.Tier", options: new JsonObserverOptions(PropertyNameCaseInsensitive: false)).Rule.Should().Be("default policy AllowList"); + observer.Explain("c.WORKEMAIL", options: new JsonObserverOptions(PropertyNameCaseInsensitive: false)).Rule.Should().Be("default policy AllowList"); + observer.Explain("c.WORKEMAIL").Rule.Should().StartWith("relative"); + } + + [Fact] + public void DefaultPolicies_Named() + { +#pragma warning disable CS0618 + JsonObserver.Obj(LegacyAllowList).Explain("b", JsonTokenType.True).Outcome.Should().Be(JsonPathOutcome.Unchanged); + JsonObserver.Obj(LegacyAllowList).Explain("s").Action.Should().Be("writes \"#str#*****\""); + JsonObserver.Obj(LegacyAllowList).Explain("n", JsonTokenType.Number).Action.Should().Be("writes \"#number#*****\""); +#pragma warning restore CS0618 + JsonObserver.Obj(NullList).Explain("s").Action.Should().Be("writes null"); + JsonObserver.Obj((ref Utf8JsonReader _, JsonWriter w, JsonObserveringEmptyContext _, ref PropertyPath _) => w.WriteNullValue()) + .Explain("s").Should().Match(e => e.Outcome == JsonPathOutcome.Custom && e.Rule == "custom default policy"); + JsonObserver.Array(BlockList).Explain("[2]", JsonTokenType.Number).Outcome.Should().Be(JsonPathOutcome.Unchanged); + JsonObserver.Array(a => a.MaskAny("x")).Explain("[0]").Rule.Should().Be("any item"); + JsonObserver.Array(BlockList).Explain("a").Outcome.Should().Be(JsonPathOutcome.Invalid); + } + + [Fact] + public void Shape_Explained() + { + var observer = JsonShapeTests.Observer(); + + observer.Explain("name").Should().Match(e => e.Outcome == JsonPathOutcome.Unchanged && e.Rule == "shape Scalar"); + observer.Explain("card").Should().Match(e => e.Outcome == JsonPathOutcome.Masked && e.Action == "MaskTag.Last4"); + observer.Explain("password", JsonTokenType.Null).Action.Should().Be("keeps null"); + observer.Explain("orders[1].secretCode").Should().Match(e => e.Outcome == JsonPathOutcome.Masked && e.Steps.Count == 4); + observer.Explain("orders[1].extra").Rule.Should().Be("unknown member (MaskWhole)"); + observer.Explain("byCode.K1.sku").Outcome.Should().Be(JsonPathOutcome.Unchanged); + observer.Explain("unknown.deep").Should().Match(e => e.Outcome == JsonPathOutcome.Masked && e.Rule.Contains("Opaque")); + observer.Explain("name.first").Rule.Should().Contain("where the path has an object"); + observer.Explain("extra").Rule.Should().Be("shape Opaque"); + observer.Explain("orders", JsonTokenType.StartArray).Outcome.Should().Be(JsonPathOutcome.Unchanged); + observer.Explain("NAME").Outcome.Should().Be(JsonPathOutcome.Unchanged); + observer.Explain("NAME", options: new JsonObserverOptions(PropertyNameCaseInsensitive: false)).Outcome.Should().Be(JsonPathOutcome.Masked); + JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions(UnknownMemberPolicy.Descend)).Explain("unknown.deep.x") + .Should().Match(e => e.Outcome == JsonPathOutcome.Masked && e.Rule == "unknown member (Descend)"); + JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions(UnknownMemberPolicy.PassThrough)).Explain("unknown.deep.x") + .Outcome.Should().Be(JsonPathOutcome.Unchanged); + JsonShapeTests.Observer(shapeOptions: new JsonShapeOptions(KeepNulls: false)).Explain("password", JsonTokenType.Null) + .Outcome.Should().Be(JsonPathOutcome.Masked); + } + + [Theory] + [InlineData("$.a.b", "a.b")] + [InlineData("a[2][10]", "a[2][10]")] + [InlineData("$['a.b']['it\\'s'][\"x\"]", "['a.b']['it\\'s'].x")] + [InlineData("[0].id", "[0].id")] + public void Path_Normalized(string path, string normalized) => + JsonObserver.Any(_ => { }, _ => { }, BlockList).Explain(path).Path.Should().Be(normalized); + + [Theory] + [InlineData("a..b")] + [InlineData("a.")] + [InlineData("a[x]")] + [InlineData("a[1")] + [InlineData("a['b]")] + [InlineData("$a")] + [InlineData("a[0]b")] + public void Path_Invalid_Throws(string path) + { + var explain = () => Lines.Explain(path); + + explain.Should().Throw(); + } + + [Fact] + public void ValueKind_NotAValue_Throws() + { + var explain = () => Lines.Explain("a", JsonTokenType.PropertyName); + + explain.Should().Throw(); + } + + [Fact] + public void Explanations_AgreeWithMasking_OnGoldenPayload() + { + var observer = JsonMaskingTests.GetRequestMasking(BlockList); + var options = new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip }; + using var input = JsonDocument.Parse(JsonMaskingTests.TestJson, options); + using var output = JsonDocument.Parse(observer.Mask(JsonMaskingTests.TestJson)!); + var checkedLeaves = 0; + + void Walk(JsonElement before, JsonElement after, string path) + { + switch (before.ValueKind) + { + case JsonValueKind.Object: + foreach (var property in before.EnumerateObject()) + { + Walk(property.Value, after.GetProperty(property.Name), path.Length == 0 ? property.Name : $"{path}.{property.Name}"); + } + + break; + default: + var kind = before.ValueKind switch + { + JsonValueKind.Number => JsonTokenType.Number, + JsonValueKind.Null => JsonTokenType.Null, + _ => JsonTokenType.String, + }; + var explanation = observer.Explain(path, kind); + var changed = before.GetRawText() != after.GetRawText(); + (explanation.Outcome == JsonPathOutcome.Masked).Should().Be(changed, explanation.ToString()); + checkedLeaves++; + break; + } + } + + Walk(input.RootElement, output.RootElement, ""); + + checkedLeaves.Should().BeGreaterThan(20); + } +} diff --git a/DragoAnt.System.Text.Json.Observer.Tests/RunExplainTests.cs b/DragoAnt.System.Text.Json.Observer.Tests/RunExplainTests.cs new file mode 100644 index 0000000..0baa841 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer.Tests/RunExplainTests.cs @@ -0,0 +1,3 @@ +namespace DragoAnt.System.Text.Json.Observer.Tests; + +public sealed class RunExplainTests : Shared.ExplainTests; diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs index 2b98368..e5f5d66 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonArrayBuilder.cs @@ -9,6 +9,9 @@ namespace DragoAnt.System.Text.Json.Observer.Builders; /// Type that read rules write extracted values to. public readonly struct JsonArrayBuilder { + private const string AnyItem = "any item"; + private const string ValueItem = "string, number, boolean or null item"; + private readonly List> _policies = []; private readonly JsonObserverValueDelegate? _builderDefaultValuePolicy; @@ -24,8 +27,11 @@ internal JsonArrayBuilder(JsonObserverValueDelegate? builderDefaultVal /// Policy for the objects' values no rule matches; the enclosing one when null. public JsonArrayBuilder Obj( Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) => - Obj(JsonObserverItem.Obj(init, defaultValuePolicy ?? _builderDefaultValuePolicy)); + JsonObserverValueDelegate? defaultValuePolicy = null) + { + var (policy, set) = JsonObserverItem.Obj(init, defaultValuePolicy ?? _builderDefaultValuePolicy); + return Add(type => type == StartObject, policy, new RuleInfo("object item", "Obj(...)", JsonPathOutcome.Unchanged, set)); + } /// /// Rules for the items that are arrays. @@ -34,8 +40,11 @@ public JsonArrayBuilder Obj( /// Policy for the nested arrays' values no rule matches; the enclosing one when null. public JsonArrayBuilder Array( Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) => - Array(JsonObserverItem.Array(init, defaultValuePolicy ?? _builderDefaultValuePolicy)); + JsonObserverValueDelegate? defaultValuePolicy = null) + { + var (policy, set) = JsonObserverItem.Array(init, defaultValuePolicy ?? _builderDefaultValuePolicy); + return Add(type => type == StartArray, policy, new RuleInfo("array item", "Array(...)", JsonPathOutcome.Unchanged, set)); + } /// public JsonArrayBuilder MaskStr(Func strategy) @@ -43,43 +52,43 @@ public JsonArrayBuilder MaskStr(Func strat /// public JsonArrayBuilder MaskStr(StringMaskingStrategy strategy) => - MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant)); + MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskStr", strategy.Constant)); /// public JsonArrayBuilder ReadStr(Action strategy) - => MaskValue(JsonObserverItem.ReadStr(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadStr(strategy, _builderDefaultValuePolicy), RuleText.ReadStr); /// public JsonArrayBuilder MaskInt(Func strategy) - => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy), "MaskInt(function)"); /// public JsonArrayBuilder ReadInt(Action strategy) - => MaskValue(JsonObserverItem.ReadInt(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadInt(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadInt")); /// public JsonArrayBuilder MaskLong(Func strategy) - => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy), "MaskLong(function)"); /// public JsonArrayBuilder ReadLong(Action strategy) - => MaskValue(JsonObserverItem.ReadLong(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadLong(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadLong")); /// public JsonArrayBuilder MaskDecimal(Func strategy) - => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy), "MaskDecimal(function)"); /// public JsonArrayBuilder ReadDecimal(Action strategy) - => MaskValue(JsonObserverItem.ReadDecimal(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadDecimal(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadDecimal")); /// public JsonArrayBuilder MaskBool(Func strategy) - => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy), "MaskBool(function)"); /// public JsonArrayBuilder ReadBool(Action strategy) - => MaskValue(JsonObserverItem.ReadBool(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadBool(strategy, _builderDefaultValuePolicy), RuleText.ReadBool); /// public JsonArrayBuilder MaskAny(Func strategy) @@ -87,19 +96,19 @@ public JsonArrayBuilder MaskAny(Func strat /// public JsonArrayBuilder MaskAny(StringMaskingStrategy strategy) - => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant)); + => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskAny", strategy.Constant)); /// public JsonArrayBuilder MaskAny(MaskTag tag) - => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag)); + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag)); /// public JsonArrayBuilder MaskRawValue(Func strategy) - => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy), "MaskRawValue(function)"); /// public JsonArrayBuilder ReadRaw(Action strategy) - => MaskValue(JsonObserverItem.ReadRaw(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadRaw(strategy, _builderDefaultValuePolicy), RuleText.ReadRaw); /// public JsonArrayBuilder MaskValue(JsonObserverValueDelegate policy) => @@ -109,26 +118,35 @@ public JsonArrayBuilder MaskValue(JsonObserverValueDelegate /// public JsonArrayBuilder MaskValue(JsonObserverDelegate policy) - => Add(type => type.IsValueToken(), policy); + => Add(type => type.IsValueToken(), policy, new RuleInfo(ValueItem, RuleText.CustomValue, JsonPathOutcome.Custom)); /// /// Writes the string, number, boolean and null items unchanged; object and array items get the next rule or the default policy. /// - public JsonArrayBuilder Unmasked() => MaskValue(JsonObserverValuePolicies.BlockList); - - internal JsonArrayBuilder MaskWhole(JsonObserverDelegate policy) => Add(_ => true, policy); + public JsonArrayBuilder Unmasked() => + Add( + type => type.IsValueToken(), + (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int _, ref PropertyPath propPath, JsonObserverValueDelegate _) => + JsonObserverValuePolicies.BlockList(ref reader, writer, context, ref propPath), + new RuleInfo(ValueItem, RuleText.Unmasked, JsonPathOutcome.Unchanged)); - internal static JsonObserverDelegate Build(JsonArrayBuilder builder) => builder.Build(); + internal JsonArrayBuilder MaskWhole(JsonObserverDelegate policy, string action) => + Add(_ => true, policy, new RuleInfo(AnyItem, action, JsonPathOutcome.Masked)); - private JsonArrayBuilder Array(JsonObserverDelegate policy) => Add(type => type == StartArray, policy); + internal static (JsonObserverDelegate Delegate, RuleSet Set) Build(JsonArrayBuilder builder) => builder.Build(); - private JsonArrayBuilder Obj(JsonObserverDelegate policy) => Add(type => type == StartObject, policy); + private JsonArrayBuilder Read(JsonObserverDelegate policy, string action) => + Add(type => type.IsValueToken(), policy, new RuleInfo(ValueItem, action, JsonPathOutcome.Read)); - private JsonObserverDelegate Build() => JsonObserverItem.ApplyArrayPolicy([.. _policies], _builderDefaultValuePolicy); + private (JsonObserverDelegate, RuleSet) Build() + { + JsonObserverItem[] items = [.. _policies]; + return (JsonObserverItem.ApplyArrayPolicy(items, _builderDefaultValuePolicy), new RuleSet(true, items, _builderDefaultValuePolicy)); + } - private JsonArrayBuilder Add(Func typeMatch, JsonObserverDelegate policy) + private JsonArrayBuilder Add(Func typeMatch, JsonObserverDelegate policy, RuleInfo info) { - _policies.Add(new JsonObserverItem((int _, ref PropertyPath _, JsonTokenType type) => (typeMatch(type), 1), policy)); + _policies.Add(new JsonObserverItem((int _, ref PropertyPath _, JsonTokenType type) => (typeMatch(type), 1), policy) { Info = info }); return this; } } diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs index 9e778ac..58dd4f4 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonObjBuilder.cs @@ -32,13 +32,18 @@ public PropertyMaskingStrategyBuilder Match(PropMatchingStrategy match) => public PropertyMaskingStrategyBuilder Match(params PropMatchingStrategy[] match) => new(this, new PropertyPathMatch(match), _builderDefaultValuePolicy); - internal static JsonObserverDelegate Build(JsonObjBuilder builder) => builder.Build(); - private JsonObserverDelegate Build() => JsonObserverItem.ApplyObjPolicy([.. _policies], _builderDefaultValuePolicy); + internal static (JsonObserverDelegate Delegate, RuleSet Set) Build(JsonObjBuilder builder) => builder.Build(); - private JsonObjBuilder AddAny(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy) => - Add((int depth, ref PropertyPath path, JsonTokenType _) => propNameMatch(depth, ref path), policy); + private (JsonObserverDelegate, RuleSet) Build() + { + JsonObserverItem[] items = [.. _policies]; + return (JsonObserverItem.ApplyObjPolicy(items, _builderDefaultValuePolicy), new RuleSet(false, items, _builderDefaultValuePolicy)); + } + + private JsonObjBuilder AddAny(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy, RuleInfo info) => + Add((int depth, ref PropertyPath path, JsonTokenType _) => propNameMatch(depth, ref path), policy, info); - private JsonObjBuilder AddValue(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy) => + private JsonObjBuilder AddValue(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy, RuleInfo info) => Add((int depth, ref PropertyPath path, JsonTokenType type) => { var (success, propDepth) = propNameMatch(depth, ref path); @@ -49,12 +54,11 @@ private JsonObjBuilder AddValue(JsonPropertyPathMatchDelegate propName } return (true, propDepth); - }, policy); + }, policy, info); - private JsonObjBuilder Add(JsonPropertyMatchDelegate propMatch, JsonObserverDelegate policy) + private JsonObjBuilder Add(JsonPropertyMatchDelegate propMatch, JsonObserverDelegate policy, RuleInfo info) { - var item = new JsonObserverItem(propMatch, policy); - _policies.Add(item); + _policies.Add(new JsonObserverItem(propMatch, policy) { Info = info }); return this; } @@ -83,43 +87,43 @@ public JsonObjBuilder MaskStr(Func strateg /// public JsonObjBuilder MaskStr(StringMaskingStrategy strategy) => - MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant)); + MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskStr", strategy.Constant)); /// public JsonObjBuilder ReadStr(Action strategy) - => MaskValue(JsonObserverItem.ReadStr(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadStr(strategy, _builderDefaultValuePolicy), RuleText.ReadStr); /// public JsonObjBuilder MaskInt(Func strategy) - => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy), "MaskInt(function)"); /// public JsonObjBuilder ReadInt(Action strategy) - => MaskValue(JsonObserverItem.ReadInt(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadInt(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadInt")); /// public JsonObjBuilder MaskLong(Func strategy) - => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy), "MaskLong(function)"); /// public JsonObjBuilder ReadLong(Action strategy) - => MaskValue(JsonObserverItem.ReadLong(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadLong(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadLong")); /// public JsonObjBuilder MaskDecimal(Func strategy) - => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy), "MaskDecimal(function)"); /// public JsonObjBuilder ReadDecimal(Action strategy) - => MaskValue(JsonObserverItem.ReadDecimal(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadDecimal(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadDecimal")); /// public JsonObjBuilder MaskBool(Func strategy) - => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy), "MaskBool(function)"); /// public JsonObjBuilder ReadBool(Action strategy) - => MaskValue(JsonObserverItem.ReadBool(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadBool(strategy, _builderDefaultValuePolicy), RuleText.ReadBool); /// public JsonObjBuilder MaskAny(Func strategy) @@ -127,19 +131,19 @@ public JsonObjBuilder MaskAny(Func strateg /// public JsonObjBuilder MaskAny(StringMaskingStrategy strategy) - => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant)); + => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskAny", strategy.Constant)); /// public JsonObjBuilder MaskAny(MaskTag tag) - => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag)); + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag)); /// public JsonObjBuilder MaskRawValue(Func strategy) - => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy), "MaskRawValue(function)"); /// public JsonObjBuilder ReadRaw(Action strategy) - => MaskValue(JsonObserverItem.ReadRaw(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadRaw(strategy, _builderDefaultValuePolicy), RuleText.ReadRaw); /// public JsonObjBuilder MaskValue(JsonObserverValueDelegate policy) => @@ -149,14 +153,15 @@ public JsonObjBuilder MaskValue(JsonObserverValueDelegate po /// public JsonObjBuilder MaskValue(JsonObserverDelegate policy) - => _builder.AddValue(_propNameMatch.AbsoluteMatch, policy); + => _builder.AddValue(_propNameMatch.AbsoluteMatch, policy, Info(RuleText.CustomValue, JsonPathOutcome.Custom)); /// public JsonObjBuilder Unmasked() => _builder.AddValue( _propNameMatch.AbsoluteMatch, (ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, ref PropertyPath propPath, JsonObserverValueDelegate _) => - JsonObserverValuePolicies.BlockList(ref reader, writer, context, ref propPath)); + JsonObserverValuePolicies.BlockList(ref reader, writer, context, ref propPath), + Info(RuleText.Unmasked, JsonPathOutcome.Unchanged)); /// /// Rules for the matched property when its value is an object; a value of another type gets the next matching rule @@ -166,27 +171,18 @@ public JsonObjBuilder Unmasked() => /// Policy for the object's values no rule matches; the enclosing one when null. public JsonObjBuilder Obj( Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) => - Obj(JsonObserverItem.Obj(init, defaultValuePolicy ?? _builderDefaultValuePolicy)); + JsonObserverValueDelegate? defaultValuePolicy = null) + { + var (policy, set) = JsonObserverItem.Obj(init, defaultValuePolicy ?? _builderDefaultValuePolicy); + return Container(StartObject, policy, Info("Obj(...)", JsonPathOutcome.Unchanged, set)); + } /// /// Custom handling of the matched property when its value is an object. /// /// Called with the reader on the object's start; it must write the object and move past it. - public JsonObjBuilder Obj(JsonObserverDelegate policy) - { - var match = _propNameMatch.AbsoluteMatch; - return _builder.Add((int depth, ref PropertyPath path, JsonTokenType type) => - { - var (success, nextDepth) = match(depth, ref path); - - if (!success || type != StartObject) - { - return (false, 0); - } - return (true, nextDepth); - }, policy); - } + public JsonObjBuilder Obj(JsonObserverDelegate policy) => + Container(StartObject, policy, Info("Obj(custom rule)", JsonPathOutcome.Custom)); /// /// Rules for the matched property when its value is an array; a value of another type gets the next matching rule @@ -196,29 +192,42 @@ public JsonObjBuilder Obj(JsonObserverDelegate policy) /// Policy for the array's values no rule matches; the enclosing one when null. public JsonObjBuilder Array( Action> init, - JsonObserverValueDelegate? defaultValuePolicy = null) => - Array(JsonObserverItem.Array(init, defaultValuePolicy ?? _builderDefaultValuePolicy)); + JsonObserverValueDelegate? defaultValuePolicy = null) + { + var (policy, set) = JsonObserverItem.Array(init, defaultValuePolicy ?? _builderDefaultValuePolicy); + return Container(StartArray, policy, Info("Array(...)", JsonPathOutcome.Unchanged, set)); + } /// /// Custom handling of the matched property when its value is an array. /// /// Called with the reader on the array's start; it must write the array and move past it. - public JsonObjBuilder Array(JsonObserverDelegate policy) + public JsonObjBuilder Array(JsonObserverDelegate policy) => + Container(StartArray, policy, Info("Array(custom rule)", JsonPathOutcome.Custom)); + + internal JsonObjBuilder MaskWhole(JsonObserverDelegate policy, string action) + => _builder.AddAny(_propNameMatch.AbsoluteMatch, policy, Info(action, JsonPathOutcome.Masked)); + + private JsonObjBuilder Read(JsonObserverDelegate policy, string action) + => _builder.AddValue(_propNameMatch.AbsoluteMatch, policy, Info(action, JsonPathOutcome.Read)); + + private JsonObjBuilder Container(JsonTokenType container, JsonObserverDelegate policy, RuleInfo info) { var match = _propNameMatch.AbsoluteMatch; return _builder.Add((int depth, ref PropertyPath path, JsonTokenType type) => { var (success, nextDepth) = match(depth, ref path); - if (!success || type != StartArray) + if (!success || type != container) { return (false, 0); } + return (true, nextDepth); - }, policy); + }, policy, info); } - internal JsonObjBuilder MaskWhole(JsonObserverDelegate policy) - => _builder.AddAny(_propNameMatch.AbsoluteMatch, policy); + private RuleInfo Info(string action, JsonPathOutcome outcome, RuleSet? child = null) => + new(_propNameMatch.Describe(), action, outcome, child); } } diff --git a/DragoAnt.System.Text.Json.Observer/Builders/JsonValuePolicyBuilder.cs b/DragoAnt.System.Text.Json.Observer/Builders/JsonValuePolicyBuilder.cs index 47c0e03..5977a37 100644 --- a/DragoAnt.System.Text.Json.Observer/Builders/JsonValuePolicyBuilder.cs +++ b/DragoAnt.System.Text.Json.Observer/Builders/JsonValuePolicyBuilder.cs @@ -33,13 +33,13 @@ public PropertyMaskingStrategyBuilder Match(params PropMatchingStrategy[] match) internal static JsonObserverItem[] BuildItems(JsonValuePolicyBuilder builder) => [.. builder._policies]; - private JsonValuePolicyBuilder AddAnyProp(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy) + private JsonValuePolicyBuilder AddAnyProp(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy, RuleInfo info) { - _policies.Add(new JsonObserverItem((int depth, ref PropertyPath path, JsonTokenType _) => propNameMatch(depth, ref path), policy)); + _policies.Add(new JsonObserverItem((int depth, ref PropertyPath path, JsonTokenType _) => propNameMatch(depth, ref path), policy) { Info = info }); return this; } - private JsonValuePolicyBuilder AddValueProp(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy) + private JsonValuePolicyBuilder AddValueProp(JsonPropertyPathMatchDelegate propNameMatch, JsonObserverDelegate policy, RuleInfo info) { var item = new JsonObserverItem((int depth, ref PropertyPath path, JsonTokenType type) => { @@ -51,7 +51,7 @@ private JsonValuePolicyBuilder AddValueProp(JsonPropertyPathMatchDeleg } return (true, nextDepth); - }, policy); + }, policy) { Info = info }; _policies.Add(item); return this; } @@ -98,7 +98,7 @@ public JsonValuePolicyBuilder MaskStr(Func /// or a function; a null result writes null. /// public JsonValuePolicyBuilder MaskStr(StringMaskingStrategy strategy) - => MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant)); + => MaskWhole(JsonObserverItem.ApplyStringPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskStr", strategy.Constant)); /// /// Hands a string or null value to and writes it unchanged. @@ -106,7 +106,7 @@ public JsonValuePolicyBuilder MaskStr(StringMaskingStrategy /// /// Receives the decoded value and the context. public JsonValuePolicyBuilder ReadStr(Action strategy) - => MaskValue(JsonObserverItem.ReadStr(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadStr(strategy, _builderDefaultValuePolicy), RuleText.ReadStr); /// /// Masks the whole value with , whatever its JSON type. The strategy receives the number @@ -115,7 +115,7 @@ public JsonValuePolicyBuilder ReadStr(Action strate /// /// Returns the replacement string; null writes null. public JsonValuePolicyBuilder MaskInt(Func strategy) - => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyIntPolicy(strategy), "MaskInt(function)"); /// /// Hands a number or null value to and writes it unchanged; a number that does not @@ -123,7 +123,7 @@ public JsonValuePolicyBuilder MaskInt(Func st /// /// Receives the value and the context. public JsonValuePolicyBuilder ReadInt(Action strategy) - => MaskValue(JsonObserverItem.ReadInt(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadInt(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadInt")); /// /// Masks the whole value with , whatever its JSON type. The strategy receives the number @@ -132,7 +132,7 @@ public JsonValuePolicyBuilder ReadInt(Action strategy) /// /// Returns the replacement string; null writes null. public JsonValuePolicyBuilder MaskLong(Func strategy) - => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyLongPolicy(strategy), "MaskLong(function)"); /// /// Hands a number or null value to and writes it unchanged; a number that does not @@ -140,7 +140,7 @@ public JsonValuePolicyBuilder MaskLong(Func /// /// Receives the value and the context. public JsonValuePolicyBuilder ReadLong(Action strategy) - => MaskValue(JsonObserverItem.ReadLong(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadLong(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadLong")); /// /// Masks the whole value with , whatever its JSON type. The strategy receives the number @@ -149,7 +149,7 @@ public JsonValuePolicyBuilder ReadLong(Action strateg /// /// Returns the replacement string; null writes null. public JsonValuePolicyBuilder MaskDecimal(Func strategy) - => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyDecimalPolicy(strategy), "MaskDecimal(function)"); /// /// Hands a number or null value to and writes it unchanged; a number out of the @@ -157,7 +157,7 @@ public JsonValuePolicyBuilder MaskDecimal(Func /// Receives the value, parsed with the invariant culture, and the context. public JsonValuePolicyBuilder ReadDecimal(Action strategy) - => MaskValue(JsonObserverItem.ReadDecimal(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadDecimal(strategy, _builderDefaultValuePolicy), RuleText.ReadNumber("ReadDecimal")); /// /// Masks the whole value with , whatever its JSON type. The strategy receives @@ -165,7 +165,7 @@ public JsonValuePolicyBuilder ReadDecimal(Action s /// /// Returns the replacement string; null writes null. public JsonValuePolicyBuilder MaskBool(Func strategy) - => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyBoolPolicy(strategy), "MaskBool(function)"); /// /// Hands a boolean or null value to and writes it unchanged. @@ -173,7 +173,7 @@ public JsonValuePolicyBuilder MaskBool(Func /// /// Receives the value and the context. public JsonValuePolicyBuilder ReadBool(Action strategy) - => MaskValue(JsonObserverItem.ReadBool(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadBool(strategy, _builderDefaultValuePolicy), RuleText.ReadBool); /// /// Masks the whole value with , whatever its JSON type: a string arrives decoded, @@ -196,7 +196,7 @@ public JsonValuePolicyBuilder MaskAny(Func /// or a function; a null result writes null. /// public JsonValuePolicyBuilder MaskAny(StringMaskingStrategy strategy) - => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant)); + => MaskWhole(JsonObserverItem.ApplyAnyPolicy(strategy, strategy.Constant), RuleText.Strategy("MaskAny", strategy.Constant)); /// /// Masks the whole value, whatever its JSON type, with the of the call @@ -204,7 +204,7 @@ public JsonValuePolicyBuilder MaskAny(StringMaskingStrategy /// /// How the value is masked, for example . public JsonValuePolicyBuilder MaskAny(MaskTag tag) - => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag)); + => MaskWhole(JsonObserverItem.ApplyTagPolicy(tag), RuleText.Tag(tag)); /// /// Like , but a string arrives as its raw JSON text, @@ -212,7 +212,7 @@ public JsonValuePolicyBuilder MaskAny(MaskTag tag) /// /// Returns the replacement string; null writes null. public JsonValuePolicyBuilder MaskRawValue(Func strategy) - => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy)); + => MaskWhole(JsonObserverItem.ApplyRawPolicy(strategy), "MaskRawValue(function)"); /// /// Hands a string, number, boolean or null value to as its raw JSON text and writes @@ -220,21 +220,21 @@ public JsonValuePolicyBuilder MaskRawValue(Func /// Receives the raw text (a string without quotes, escapes kept) and the context. public JsonValuePolicyBuilder ReadRaw(Action strategy) - => MaskValue(JsonObserverItem.ReadRaw(strategy, _builderDefaultValuePolicy)); + => Read(JsonObserverItem.ReadRaw(strategy, _builderDefaultValuePolicy), RuleText.ReadRaw); /// /// Writes the value of the matched property unchanged. Applies to strings, numbers, booleans and null; /// an object or array gets the next matching rule or the default policy. /// public JsonValuePolicyBuilder Unmasked() => - MaskValue(( + Value(( ref Utf8JsonReader reader, JsonWriter writer, TContext context, int depth, ref PropertyPath propPath, JsonObserverValueDelegate _) => - JsonObserverValuePolicies.BlockList(ref reader, writer, context, ref propPath)); + JsonObserverValuePolicies.BlockList(ref reader, writer, context, ref propPath), RuleText.Unmasked, JsonPathOutcome.Unchanged); /// /// Custom rule for a string, number, boolean or null value of the matched property; an object or array @@ -257,9 +257,19 @@ public JsonValuePolicyBuilder MaskValue(JsonObserverValueDelegate /// Called with the reader on the value; it must write exactly one value. public JsonValuePolicyBuilder MaskValue(JsonObserverDelegate policy) - => _builder.AddValueProp(_builder._relative ? _propNameMatch.RelativeMatch : _propNameMatch.AbsoluteMatch, policy); + => Value(policy, RuleText.CustomValue, JsonPathOutcome.Custom); + + internal JsonValuePolicyBuilder MaskWhole(JsonObserverDelegate policy, string action) + => _builder.AddAnyProp(Match, policy, Info(action, JsonPathOutcome.Masked)); + + private JsonValuePolicyBuilder Read(JsonObserverDelegate policy, string action) + => Value(policy, action, JsonPathOutcome.Read); + + private JsonValuePolicyBuilder Value(JsonObserverDelegate policy, string action, JsonPathOutcome outcome) + => _builder.AddValueProp(Match, policy, Info(action, outcome)); + + private JsonPropertyPathMatchDelegate Match => _builder._relative ? _propNameMatch.RelativeMatch : _propNameMatch.AbsoluteMatch; - internal JsonValuePolicyBuilder MaskWhole(JsonObserverDelegate policy) - => _builder.AddAnyProp(_builder._relative ? _propNameMatch.RelativeMatch : _propNameMatch.AbsoluteMatch, policy); + private RuleInfo Info(string action, JsonPathOutcome outcome) => new(_propNameMatch.Describe(), action, outcome); } } diff --git a/DragoAnt.System.Text.Json.Observer/Builders/RuleText.cs b/DragoAnt.System.Text.Json.Observer/Builders/RuleText.cs new file mode 100644 index 0000000..68134aa --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/Builders/RuleText.cs @@ -0,0 +1,24 @@ +using DragoAnt.System.Text.Json.Observer.Strategies; + +namespace DragoAnt.System.Text.Json.Observer.Builders; + +/// +/// How rule actions read in a . +/// +internal static class RuleText +{ + public const string Unmasked = "Unmasked()"; + public const string CustomValue = "MaskValue(custom rule)"; + public const string ReadStr = "ReadStr (a string or null is read and written as is; another type gets the default policy)"; + public const string ReadBool = "ReadBool (a boolean or null is read and written as is; another type gets the default policy)"; + public const string ReadRaw = "ReadRaw (a scalar is read and written as is; an object or array gets the default policy)"; + + public static string ReadNumber(string method) => + $"{method} (a number or null is read and written as is; another type gets the default policy)"; + + public static string Strategy(string method, string? constant) => + constant is null ? $"{method}(function)" : $"{method}(\"{constant}\")"; + + public static string Tag(MaskTag tag) => + tag.Key is null ? $"MaskAny(MaskTag.{tag.Kind})" : $"MaskAny(MaskTag.{tag.Kind}, key {tag.Key})"; +} diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs index c86c3d9..df1f786 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserver.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserver.cs @@ -72,8 +72,11 @@ public static JsonObserver Array( public static JsonObserver Any( Action> initObj, Action> initArray, - JsonObserverValueDelegate? defaultMasking = null) => - new(JsonObserverItem.Any(initObj, initArray, defaultMasking)); + JsonObserverValueDelegate? defaultMasking = null) + { + var (masking, obj, array) = JsonObserverItem.Any(initObj, initArray, defaultMasking); + return new JsonObserver(masking, new RuleExplainer(obj, array)); + } /// /// Creates an observer with a context for a root object that applies one policy to every value. @@ -81,7 +84,7 @@ public static JsonObserver Any( /// Policy for every value; when null. /// Type that read rules write extracted values to. public static JsonObserver Obj(JsonObserverValueDelegate? defaultMasking) => - new(JsonObserverItem.Obj(_ => { }, defaultMasking)); + Obj(_ => { }, defaultMasking); /// /// Creates an observer that also extracts values into a , for a root object. @@ -91,8 +94,11 @@ public static JsonObserver Obj(JsonObserverValueDelegateType that read rules write extracted values to. public static JsonObserver Obj( Action> init, - JsonObserverValueDelegate? defaultMasking = null) => - new(JsonObserverItem.Obj(init, defaultMasking)); + JsonObserverValueDelegate? defaultMasking = null) + { + var (masking, set) = JsonObserverItem.Obj(init, defaultMasking); + return new JsonObserver(masking, new RuleExplainer(set, null)); + } /// /// Creates an observer with a context for a root array that applies one policy to every value. @@ -100,7 +106,7 @@ public static JsonObserver Obj( /// Policy for every value; when null. /// Type that read rules write extracted values to. public static JsonObserver Array(JsonObserverValueDelegate? defaultMasking) => - new(JsonObserverItem.Array(_ => { }, defaultMasking)); + Array(_ => { }, defaultMasking); /// /// Creates an observer that also extracts values into a , for a root array. @@ -110,8 +116,11 @@ public static JsonObserver Array(JsonObserverValueDelegateType that read rules write extracted values to. public static JsonObserver Array( Action> init, - JsonObserverValueDelegate? defaultMasking = null) => - new(JsonObserverItem.Array(init, defaultMasking)); + JsonObserverValueDelegate? defaultMasking = null) + { + var (masking, set) = JsonObserverItem.Array(init, defaultMasking); + return new JsonObserver(masking, new RuleExplainer(null, set)); + } /// /// Creates an observer that masks against an expected structure: values of known properties are written as is, @@ -124,7 +133,7 @@ public static JsonObserver FromShape(JsonShape shape, JsonShapeOptions? options { ArgumentNullException.ThrowIfNull(shape); var walker = new ShapeWalker(shape, options ?? JsonShapeOptions.Default); - return new JsonObserver(new JsonObserver(walker.Invoke)); + return new JsonObserver(new JsonObserver(walker.Invoke, walker)); } private JsonObserver(JsonObserver masking) @@ -177,6 +186,10 @@ public MaskResult Mask(ReadOnlySpan utf8, IBufferWriter output, Json /// Status, bytes written and the input offset where reading stopped. public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter output, JsonObserverOptions? options = null) => _masking.Mask(utf8, output, JsonObserveringEmptyContext.Instance, options); + + /// + public JsonPathExplanation Explain(string path, JsonTokenType valueKind = JsonTokenType.String, JsonObserverOptions? options = null) + => _masking.Explain(path, valueKind, options); } /// @@ -187,13 +200,34 @@ public MaskResult Mask(in ReadOnlySequence utf8, IBufferWriter outpu public sealed class JsonObserver { private readonly JsonObserverDelegate _maskDelegate; + private readonly PathExplainer _explainer; private int _maxDepth = 6; - internal JsonObserver(JsonObserverDelegate maskDelegate) + internal JsonObserver(JsonObserverDelegate maskDelegate, PathExplainer explainer) { _maskDelegate = maskDelegate; + _explainer = explainer; } + /// + /// Tells which rule or policy handles the value at and what it does with it, without + /// masking anything: useful to check a configuration, to document it, or to find out why a value was masked. + /// + /// + /// A JSON path such as items[2].sku, $.order.card.number or $['a.b']; the first segment decides + /// whether the root is an object or an array. + /// + /// + /// JSON type of the value at the path: a scalar type, , or + /// / for a container; rules can differ by type. + /// + /// The call's options, for . + /// The deciding rule, its action, the outcome and the steps that lead there. + /// is not a JSON path. + /// is not a value type or a container start. + public JsonPathExplanation Explain(string path, JsonTokenType valueKind = JsonTokenType.String, JsonObserverOptions? options = null) + => _explainer.Explain(path, valueKind, (options ?? JsonObserverOptions.Default).PropertyNameCaseInsensitive); + private static ReadOnlySpan Utf8Bom => [0xEF, 0xBB, 0xBF]; /// diff --git a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs index 2d32968..980860b 100644 --- a/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs +++ b/DragoAnt.System.Text.Json.Observer/JsonObserverItem.cs @@ -14,21 +14,26 @@ namespace DragoAnt.System.Text.Json.Observer; /// Masking policy delegate. internal sealed class JsonObserverItem(JsonPropertyMatchDelegate propMatch, JsonObserverDelegate masking) { + /// + /// What the rule tests and does, for explanations. + /// + public RuleInfo Info { get; init; } = RuleInfo.Unknown; + /// /// Any payload object or array. /// /// Init masking for object. /// Init masking for array. /// Default policy for unknown scenarios. - public static JsonObserverDelegate Any( + public static (JsonObserverDelegate Delegate, RuleSet Obj, RuleSet Array) Any( Action> initObj, Action> initArray, JsonObserverValueDelegate? defaultValueMasking) { - var objMasking = Obj(initObj, defaultValueMasking); - var arrayMasking = Array(initArray, defaultValueMasking); + var (objMasking, objSet) = Obj(initObj, defaultValueMasking); + var (arrayMasking, arraySet) = Array(initArray, defaultValueMasking); - return ( + return (( ref Utf8JsonReader reader, JsonWriter writer, TContext context, @@ -58,7 +63,7 @@ public static JsonObserverDelegate Any( default: throw new JsonObserverException("Wrong path"); } - }; + }, objSet, arraySet); } /// @@ -66,7 +71,9 @@ public static JsonObserverDelegate Any( /// /// Init masking for object. /// Default masking for unknown scenarios. - public static JsonObserverDelegate Obj(Action> init, JsonObserverValueDelegate? defaultValueMasking) + public static (JsonObserverDelegate Delegate, RuleSet Set) Obj( + Action> init, + JsonObserverValueDelegate? defaultValueMasking) { var builder = new JsonObjBuilder(defaultValueMasking); init(builder); @@ -78,7 +85,9 @@ public static JsonObserverDelegate Obj(Action /// /// Masking condition builder. /// Default masking policy. - public static JsonObserverDelegate Array(Action> init, JsonObserverValueDelegate? defaultValuePolicy) + public static (JsonObserverDelegate Delegate, RuleSet Set) Array( + Action> init, + JsonObserverValueDelegate? defaultValuePolicy) { var builder = new JsonArrayBuilder(defaultValuePolicy); init(builder); diff --git a/DragoAnt.System.Text.Json.Observer/JsonPathExplanation.cs b/DragoAnt.System.Text.Json.Observer/JsonPathExplanation.cs new file mode 100644 index 0000000..aee7b27 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/JsonPathExplanation.cs @@ -0,0 +1,48 @@ +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// What an observer does with a value, as reported by . +/// +public enum JsonPathOutcome +{ + /// + /// The value is written as it is. + /// + Unchanged, + + /// + /// The value is replaced: masked, hashed, written as null, or an object or array masked whole. + /// + Masked, + + /// + /// The value is handed to the context by a read rule and written as it is. + /// + Read, + + /// + /// A custom rule or policy decides; the observer cannot tell what it writes. + /// + Custom, + + /// + /// A payload with this structure is not masked at all: its status is . + /// + Invalid, +} + +/// +/// Which rule or policy of an observer handles a JSON path, and what it does with the value there. +/// +/// The path explained, normalized, for example lines[0].qty. +/// What happens to the value. +/// The rule or policy that decides, for example Match("qty") or default policy AllowList. +/// What it does, for example MaskAny("***") or writes "***". +/// How the observer gets there, one entry per level of the path. +public sealed record JsonPathExplanation(string Path, JsonPathOutcome Outcome, string Rule, string Action, IReadOnlyList Steps) +{ + /// + /// One line, for example lines[0].qty: Masked by Match("qty") → MaskAny("***"). + /// + public override string ToString() => $"{Path}: {Outcome} by {Rule} → {Action}"; +} diff --git a/DragoAnt.System.Text.Json.Observer/PathExplainer.cs b/DragoAnt.System.Text.Json.Observer/PathExplainer.cs new file mode 100644 index 0000000..b5a9197 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/PathExplainer.cs @@ -0,0 +1,183 @@ +using System.Globalization; +using System.Text; + +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// One level of a path given to Explain: a property name, or an array index when is null. +/// +internal readonly record struct PathSegment(string? Name, int Index) +{ + public bool IsIndex => Name is null; +} + +/// +/// Explains which rule or policy of an observer handles a path. +/// +internal abstract class PathExplainer +{ + public JsonPathExplanation Explain(string path, JsonTokenType valueKind, bool propertyNameCaseInsensitive) + { + ArgumentNullException.ThrowIfNull(path); + if (valueKind is not (JsonTokenType.String or JsonTokenType.Number or JsonTokenType.True or JsonTokenType.False + or JsonTokenType.Null or JsonTokenType.StartObject or JsonTokenType.StartArray)) + { + throw new ArgumentOutOfRangeException(nameof(valueKind), valueKind, "Expected a value type, StartObject or StartArray."); + } + + var segments = Parse(path); + var normalized = Format(segments); + if (segments.Count == 0) + { + return new JsonPathExplanation("$", JsonPathOutcome.Unchanged, "root", "the root's rules apply to its members", []); + } + + var steps = new List(); + var (outcome, rule, action) = Explain(segments, valueKind, propertyNameCaseInsensitive, steps); + return new JsonPathExplanation(normalized, outcome, rule, action, steps); + } + + protected abstract (JsonPathOutcome Outcome, string Rule, string Action) Explain( + IReadOnlyList segments, + JsonTokenType valueKind, + bool propertyNameCaseInsensitive, + List steps); + + protected static JsonTokenType TokenAt(IReadOnlyList segments, int index, JsonTokenType valueKind) => + index == segments.Count - 1 ? valueKind + : segments[index + 1].IsIndex ? JsonTokenType.StartArray + : JsonTokenType.StartObject; + + protected static PropertyPath PathOf(IReadOnlyList segments, bool propertyNameCaseInsensitive) => + new(segments.Count, default) { PropertyNameCaseInsensitive = propertyNameCaseInsensitive }; + + protected static void Push(ref PropertyPath path, PathSegment segment) + { + if (segment.IsIndex) + { + path.AddArrayItem(segment.Index); + return; + } + + var name = segment.Name!; + var buffer = Encoding.UTF8.GetBytes(name); + path.AddPropertyName(buffer); + } + + protected static string Format(IReadOnlyList segments, int count = -1) + { + var text = new StringBuilder(); + count = count < 0 ? segments.Count : count; + for (var i = 0; i < count; i++) + { + if (segments[i].IsIndex) + { + text.Append('[').Append(segments[i].Index.ToString(CultureInfo.InvariantCulture)).Append(']'); + } + else + { + PropertyPath.AppendName(text, segments[i].Name!, first: i == 0); + } + } + + return text.ToString(); + } + + /// + /// Parses $.a.b[2]['c.d']; the leading $ and the first dot are optional. + /// + internal static List Parse(string path) + { + var segments = new List(); + var i = 0; + if (path.StartsWith('$')) + { + i = 1; + } + + var expectName = i == 0; + while (i < path.Length) + { + var c = path[i]; + if (c == '.') + { + if (expectName) + { + throw Invalid(path, i); + } + + i++; + expectName = true; + continue; + } + + if (c == '[') + { + i = ParseBracket(path, i, segments); + expectName = false; + continue; + } + + if (!expectName) + { + throw Invalid(path, i); + } + + var start = i; + while (i < path.Length && path[i] is not ('.' or '[')) + { + i++; + } + + segments.Add(new PathSegment(path[start..i], -1)); + expectName = false; + } + + if (expectName && path.Length > 0 && path[^1] == '.') + { + throw Invalid(path, path.Length - 1); + } + + return segments; + } + + private static int ParseBracket(string path, int i, List segments) + { + var start = i + 1; + if (start < path.Length && path[start] is '\'' or '"') + { + var quote = path[start]; + var name = new StringBuilder(); + var j = start + 1; + while (j < path.Length && path[j] != quote) + { + if (path[j] == '\\' && j + 1 < path.Length) + { + j++; + } + + name.Append(path[j++]); + } + + if (j + 1 >= path.Length || path[j + 1] != ']') + { + throw Invalid(path, i); + } + + segments.Add(new PathSegment(name.ToString(), -1)); + return j + 2; + } + + var end = path.IndexOf(']', start); + if (end < 0 || !int.TryParse(path.AsSpan(start, end - start), NumberStyles.None, CultureInfo.InvariantCulture, out var index)) + { + throw Invalid(path, i); + } + + segments.Add(new PathSegment(null, index)); + return end + 1; + } + + private static ArgumentException Invalid(string path, int at) => + new($"'{path}' is not a JSON path such as 'items[2].sku' or \"$['a.b']\" (position {at}).", nameof(path)); +} diff --git a/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs b/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs index 93263fc..22d682c 100644 --- a/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs +++ b/DragoAnt.System.Text.Json.Observer/PropertyPathMatch.cs @@ -25,6 +25,8 @@ private PropertyPathMatch(NameMatcher[] matches) _matches = matches; } + public string Describe() => $"Match({string.Join(", ", _matches.Select(m => m.Describe()))})"; + public (bool success, int depth) RelativeMatch(int depth, ref PropertyPath propPath) { var last = propPath.CurrentDepth; diff --git a/DragoAnt.System.Text.Json.Observer/RelativeValuePolicy.cs b/DragoAnt.System.Text.Json.Observer/RelativeValuePolicy.cs index c8d63f0..cfadd8e 100644 --- a/DragoAnt.System.Text.Json.Observer/RelativeValuePolicy.cs +++ b/DragoAnt.System.Text.Json.Observer/RelativeValuePolicy.cs @@ -8,6 +8,10 @@ internal sealed class RelativeValuePolicy( JsonObserverItem[] items, JsonObserverValueDelegate defaultValuePolicy) { + public JsonObserverItem[] Items => items; + + public JsonObserverValueDelegate DefaultValuePolicy => defaultValuePolicy; + public void Invoke(ref Utf8JsonReader reader, JsonWriter writer, TContext context, ref PropertyPath propPath) => policy(ref reader, writer, context, 0, ref propPath, defaultValuePolicy); diff --git a/DragoAnt.System.Text.Json.Observer/RuleExplainer.cs b/DragoAnt.System.Text.Json.Observer/RuleExplainer.cs new file mode 100644 index 0000000..04c5298 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/RuleExplainer.cs @@ -0,0 +1,140 @@ +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// Explains a rule-based observer by walking its rules the way the masking pass does, with a path built from the +/// explained one. +/// +internal sealed class RuleExplainer(RuleSet? obj, RuleSet? array) : PathExplainer +{ + protected override (JsonPathOutcome Outcome, string Rule, string Action) Explain( + IReadOnlyList segments, + JsonTokenType valueKind, + bool propertyNameCaseInsensitive, + List steps) + { + var set = segments[0].IsIndex ? array : obj; + if (set is null) + { + var root = segments[0].IsIndex ? "array" : "object"; + steps.Add($"$: the observer does not accept a root {root}"); + return (JsonPathOutcome.Invalid, "root", $"a root {root} makes the payload Invalid"); + } + + var path = PathOf(segments, propertyNameCaseInsensitive); + try + { + var effective = set.ValuePolicy ?? JsonObserverValuePolicies.Default; + var items = set.Items; + var depth = 0; + var chain = new List(); + for (var i = 0; i < segments.Count; i++) + { + Push(ref path, segments[i]); + var at = Format(segments, i + 1); + var token = TokenAt(segments, i, valueKind); + var last = i == segments.Count - 1; + var (item, nextDepth) = JsonObserverItem.MatchPolicy(items, depth, ref path, token); + if (item is not null) + { + var info = item.Info; + chain.Add(info.Match); + if (last || info.Child is null) + { + var (outcome, action) = last ? Resolve(info, token) : (info.Outcome, $"{info.Action} on the whole {Container(token)}"); + steps.Add($"{at}: {info.Match} → {action}"); + return (outcome, string.Join(" > ", chain), action); + } + + steps.Add($"{at}: {info.Match} → {info.Action}"); + items = info.Child.Items; + depth = nextDepth; + effective = info.Child.ValuePolicy ?? effective; + continue; + } + + if (!last) + { + if (effective.Target is RelativeValuePolicy relative) + { + var (relativeItem, _) = JsonObserverItem.MatchPolicy(relative.Items, 0, ref path, token); + if (relativeItem is not null) + { + var info = relativeItem.Info; + steps.Add($"{at}: relative {info.Match} → {info.Action} on the whole {Container(token)}"); + return (info.Outcome, $"relative {info.Match}", $"{info.Action} on the whole {Container(token)}"); + } + } + + steps.Add($"{at}: no rule; the {Container(token)} is descended with the same rules"); + continue; + } + + return DefaultPolicy(effective, ref path, token, at, steps); + } + + throw new InvalidOperationException("Unreachable: the last segment always returns."); + } + finally + { + path.Dispose(); + } + } + + private static (JsonPathOutcome, string, string) DefaultPolicy( + JsonObserverValueDelegate policy, + ref PropertyPath path, + JsonTokenType token, + string at, + List steps) + { + if (policy.Target is RelativeValuePolicy relative) + { + var (item, _) = JsonObserverItem.MatchPolicy(relative.Items, 0, ref path, token); + if (item is not null) + { + var (relativeOutcome, relativeAction) = Resolve(item.Info, token); + steps.Add($"{at}: relative {item.Info.Match} → {relativeAction}"); + return (relativeOutcome, $"relative {item.Info.Match}", relativeAction); + } + + steps.Add($"{at}: no relative rule"); + return DefaultPolicy(relative.DefaultValuePolicy, ref path, token, at, steps); + } + + var name = KnownPolicyName(policy); + var rule = name is null ? "custom default policy" : $"default policy {name}"; + var (outcome, action) = token is JsonTokenType.Null + ? (JsonPathOutcome.Unchanged, "keeps null") + : name switch + { + nameof(JsonObserverValuePolicies.AllowList) => (JsonPathOutcome.Masked, "writes \"***\""), + nameof(JsonObserverValuePolicies.BlockList) => (JsonPathOutcome.Unchanged, "writes the value as is"), + nameof(JsonObserverValuePolicies.NullList) => (JsonPathOutcome.Masked, "writes null"), + "LegacyAllowList" when token is JsonTokenType.True or JsonTokenType.False => (JsonPathOutcome.Unchanged, "writes the boolean as is"), + "LegacyAllowList" => (JsonPathOutcome.Masked, token is JsonTokenType.String ? "writes \"#str#*****\"" : "writes \"#number#*****\""), + _ => (JsonPathOutcome.Custom, "custom default policy decides"), + }; + if (token is JsonTokenType.StartObject or JsonTokenType.StartArray) + { + (outcome, action) = (JsonPathOutcome.Unchanged, $"the {Container(token)} is descended with the same rules"); + } + + steps.Add($"{at}: {rule} → {action}"); + return (outcome, rule, action); + } + + private static string? KnownPolicyName(JsonObserverValueDelegate policy) + { + var declaring = policy.Method.DeclaringType; + return declaring is { IsGenericType: true } && declaring.GetGenericTypeDefinition() == typeof(JsonObserverValuePolicies<>) + ? policy.Method.Name + : null; + } + + private static (JsonPathOutcome Outcome, string Action) Resolve(RuleInfo info, JsonTokenType token) => + token is JsonTokenType.Null && info.Action.StartsWith("MaskAny(", StringComparison.Ordinal) + ? (JsonPathOutcome.Unchanged, $"{info.Action} keeps null") + : (info.Outcome, info.Action); + + private static string Container(JsonTokenType token) => token is JsonTokenType.StartArray ? "array" : "object"; +} diff --git a/DragoAnt.System.Text.Json.Observer/RuleSet.cs b/DragoAnt.System.Text.Json.Observer/RuleSet.cs new file mode 100644 index 0000000..90d6a05 --- /dev/null +++ b/DragoAnt.System.Text.Json.Observer/RuleSet.cs @@ -0,0 +1,18 @@ +namespace DragoAnt.System.Text.Json.Observer; + +/// +/// The rules of one object or array, kept beside the delegate built from them so that a path can be explained. +/// +internal sealed record RuleSet(bool IsArray, JsonObserverItem[] Items, JsonObserverValueDelegate? ValuePolicy); + +/// +/// What a rule tests and what it does, for . +/// +/// The test, for example Match("card", "number"). +/// The action, for example MaskAny("***"). +/// What the action does to the value. +/// Rules for the matched object or array; null for a value rule or a custom container rule. +internal sealed record RuleInfo(string Match, string Action, JsonPathOutcome Outcome, RuleSet? Child = null) +{ + public static RuleInfo Unknown { get; } = new("rule", "custom rule", JsonPathOutcome.Custom); +} diff --git a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs index 433231b..3e09a58 100644 --- a/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs +++ b/DragoAnt.System.Text.Json.Observer/ShapeWalker.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; +using System.Text; using DragoAnt.System.Text.Json.Observer.Strategies; using static System.Text.Json.JsonTokenType; @@ -7,7 +8,7 @@ namespace DragoAnt.System.Text.Json.Observer; /// /// Masks a payload against a : known values are written as is, everything else is masked. /// -internal sealed class ShapeWalker +internal sealed class ShapeWalker : PathExplainer { private readonly JsonShape _root; private readonly JsonShape _unknown; @@ -181,6 +182,96 @@ private void MaskWhole(ref Utf8JsonReader reader, JsonWriter writer, ref Propert TagMasking.Mask(ref reader, writer, tag, ref propPath); } + protected override (JsonPathOutcome Outcome, string Rule, string Action) Explain( + IReadOnlyList segments, + JsonTokenType valueKind, + bool propertyNameCaseInsensitive, + List steps) + { + var ignoreCase = _ignoreCase ?? propertyNameCaseInsensitive; + var current = _root; + var unknownMember = false; + for (var i = 0; i < segments.Count; i++) + { + var segment = segments[i]; + var at = Format(segments, i + 1); + var container = segment.IsIndex ? JsonShapeKind.Array : JsonShapeKind.Object; + if (ReferenceEquals(current, JsonShape.UnknownPassThrough)) + { + steps.Add($"{at}: inside an unknown member (PassThrough)"); + continue; + } + + if (ReferenceEquals(current, JsonShape.UnknownDescend)) + { + steps.Add($"{at}: inside an unknown member (Descend)"); + continue; + } + + switch (current.Kind) + { + case JsonShapeKind.Object when !segment.IsIndex: + { + var known = current.FindMember(Encoding.UTF8.GetBytes(segment.Name!), ignoreCase); + steps.Add(known is null ? $"{at}: unknown member ({Unknown})" : $"{at}: known member {known.Name} ({known.Shape.Kind})"); + current = known?.Shape ?? _unknown; + unknownMember = known is null; + continue; + } + case JsonShapeKind.Map when !segment.IsIndex: + steps.Add($"{at}: dictionary value ({current.Item!.Kind})"); + current = current.Item!; + continue; + case JsonShapeKind.Array when segment.IsIndex: + steps.Add($"{at}: array item ({current.Item!.Kind})"); + current = current.Item!; + continue; + case JsonShapeKind.Masked: + return Masked(steps, at, $"shape Masked({current.Tag.Kind})", $"MaskTag.{current.Tag.Kind} on the whole value"); + default: + var expected = container == JsonShapeKind.Array ? "array" : "object"; + return Masked(steps, at, $"shape {current.Kind} where the path has an {expected}", "writes \"***\" for the whole value"); + } + } + + return Final(current, valueKind, steps, Format(segments), unknownMember); + } + + private string Unknown => ReferenceEquals(_unknown, JsonShape.UnknownDescend) ? "Descend" + : ReferenceEquals(_unknown, JsonShape.UnknownPassThrough) ? "PassThrough" + : "MaskWhole"; + + private (JsonPathOutcome, string, string) Final(JsonShape shape, JsonTokenType token, List steps, string at, bool unknownMember) + { + var isContainer = token is StartObject or StartArray; + var (outcome, rule, action) = shape switch + { + _ when ReferenceEquals(shape, JsonShape.UnknownPassThrough) => (JsonPathOutcome.Unchanged, "unknown member (PassThrough)", "writes the value as is"), + _ when ReferenceEquals(shape, JsonShape.UnknownDescend) && isContainer => (JsonPathOutcome.Unchanged, "unknown member (Descend)", "shows the names, masks every value inside"), + _ when ReferenceEquals(shape, JsonShape.UnknownDescend) => KeepNull(token, "unknown member (Descend)", "writes \"***\""), + { Kind: JsonShapeKind.Scalar } when !isContainer => (JsonPathOutcome.Unchanged, "shape Scalar", "writes the value as is"), + { Kind: JsonShapeKind.Masked } => KeepNull(token, $"shape Masked({shape.Tag.Kind})", $"MaskTag.{shape.Tag.Kind}"), + { Kind: JsonShapeKind.Object or JsonShapeKind.Map } when token is StartObject => (JsonPathOutcome.Unchanged, $"shape {shape.Kind}", "applies the shape to the members"), + { Kind: JsonShapeKind.Array } when token is StartArray => (JsonPathOutcome.Unchanged, "shape Array", "applies the item shape to every item"), + { Kind: JsonShapeKind.Object or JsonShapeKind.Map or JsonShapeKind.Array } when token is Null => (JsonPathOutcome.Unchanged, $"shape {shape.Kind}", "keeps null"), + _ when unknownMember => KeepNull(token, "unknown member (MaskWhole)", "writes \"***\" for the whole value"), + { Kind: JsonShapeKind.Opaque } => KeepNull(token, "shape Opaque", "writes \"***\" for the whole value"), + _ => KeepNull(token, $"shape {shape.Kind} does not fit a {token} value", "writes \"***\" for the whole value"), + }; + + steps.Add($"{at}: {rule} → {action}"); + return (outcome, rule, action); + } + + private (JsonPathOutcome, string, string) KeepNull(JsonTokenType token, string rule, string action) => + token is Null && _keepNulls ? (JsonPathOutcome.Unchanged, rule, "keeps null") : (JsonPathOutcome.Masked, rule, action); + + private static (JsonPathOutcome, string, string) Masked(List steps, string at, string rule, string action) + { + steps.Add($"{at}: {rule} → {action}"); + return (JsonPathOutcome.Masked, rule, action); + } + private static void CopyScalar(ref Utf8JsonReader reader, JsonWriter writer) { switch (reader.TokenType) From ab9ce30c2dda0a5796dd3ed2f07f996ac1edeb62 Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:16:07 +0200 Subject: [PATCH 12/13] Document the 2.0.0 core API additions and fixes CHANGELOG lists the sequence input, zero-allocation bytes API, Explain, classified tags, strategy context, JsonWriter span overloads, path indices, case sensitivity and shape metadata, plus the nested Array/Obj and relative-null fixes. The README gains compiled examples for a custom strategy and for Explain, the new option row and the 0 B allocation figure. --- CHANGELOG.md | 16 ++++- .../package.readme.md | 2 +- README.md | 72 +++++++++++++++++-- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70b0c70..48dd34e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **UTF-8 name matching:** property names are compared on their UTF-8 bytes, without creating strings. - **Verbatim pass-through:** unmasked numbers and strings are copied as written (`1.50`, `1e400`, `-0`, 20-digit integers). - `PropertyPath.Length`. +- **`ReadOnlySequence` input:** `Mask(in ReadOnlySequence, …)` and `Read(in ReadOnlySequence, …)` mask a payload held in several buffers, for example from a `PipeReader`, without copying it into one; the output is byte for byte what the span overload writes, however the bytes are split. +- **No per-call allocation on the bytes API:** the writers are reused per thread, so a warm `Mask`/`Read` of bytes with constant or tag rules allocates nothing (pinned by a test for the span, sequence, ignore-nulls and read paths). +- **`Explain(path)`:** `JsonObserver.Explain("lines[0].qty", JsonTokenType.Number)` returns a `JsonPathExplanation` naming the rule or policy that handles the value, its action, the outcome (`Unchanged`, `Masked`, `Read`, `Custom`, `Invalid`) and one step per level, for rule-based and shape observers. +- **Classified tags:** `MaskTag` carries an optional `Key` (a data classification, a redactor name) and `MaskKind.Custom`, so a strategy maps its own taxonomy without casting enum values; `MaskTag.Custom(key)`, `TryGetKey`. +- **Strategies see where a value is:** `Utf8MaskStrategy.Mask(in Utf8MaskContext, JsonWriter)` receives the value, its JSON type, the tag, the options, the property name and the whole path without allocating. Both `Mask` overloads are virtual; a strategy overrides the one it needs. +- **`JsonWriter` span overloads:** `WriteStringValue(ReadOnlySpan)`, `WritePropertyName(ReadOnlySpan)`, `WriteBase64StringValue(ReadOnlySpan)` and `WriteNumberValue(double)`. +- **Array indices in paths:** `PropertyPath.ToString()` renders `items[2].sku` (names that need it as `['a.b']`); `TryGetArrayIndex`, `IsArrayItem` and `TryGetPropertyNameUtf8` give zero-allocation access. +- **Case sensitivity:** `JsonObserverOptions.PropertyNameCaseInsensitive` (default `true`) makes rules, `PropMatches` tests and shapes match names exactly when set to `false`, the way the serializer does; `JsonShapeOptions.PropertyNameCaseInsensitive` and `JsonShapeOptions.FromSerializerOptions(...)` set it for one shape observer; `PropertyPath.PropertyNameCaseInsensitive` tells a custom rule. +- **Metadata on shapes:** `JsonShape.Members` lists `JsonShapeProperty` items with the `JsonPropertyInfo`, CLR member, property and declaring type, `IsRequired`, `IsNullable` and custom attributes; nodes carry their `JsonTypeInfo`/`ClrType`; nodes and properties have `Annotations` for integrations; `FromTypeInfo` takes an `annotate` callback, and `FindMember` looks a property up by its UTF-8 name. On .NET 8, source-generated metadata has no attributes or reference-type nullability. - **New package `DragoAnt.System.Text.Json.Observer.Http`:** `JsonBodyLoggingHandler` logs masked `HttpClient` request and response bodies; register it with `AddJsonBodyLogging`, pick maskers per body model type with `IJsonBodyMaskerProvider`, and attach model types per request with `WithBodyLogging()`. ### Changed — breaking @@ -26,11 +35,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 4. **Every `Mask*` rule masks the whole value whatever its JSON type.** `MaskStr`, `MaskRawValue`, `MaskInt`, `MaskLong`, `MaskDecimal` and `MaskBool` no longer pass a value of another type to the default policy (where `BlockList` exposed it), and no longer descend into an object or array: a container is skipped unread and the function receives `null`. `MaskStr` hands a number or boolean to its function as its literal (`"12.50"`, `"true"`). As these rules now match containers too, a rule written before an `Obj(...)` or `Array(...)` rule for the same name takes precedence over it. 5. **A string cut by `MaxValueBytes` reports `Truncated`** (with `FailedAtByte` `-1`), not `Masked`. A masking function receives a value longer than `MaxValueBytes` cut to that length. 6. **Number read rules no longer fail the body:** `ReadInt`, `ReadLong` and `ReadDecimal` receive `null` for a number that does not fit the type, and the token is written unchanged. -7. **`PropertyPath` is a `ref struct` valid only during the call** it is passed to: `GetPropertyName`, `GetPropertyNameReverse`, `Length` and `ToString` remain; its constructor, `MaxLength` and `Dispose` are internal. Custom rules compiled against 1.x must be rebuilt. +7. **`PropertyPath` is a `ref struct` valid only during the call** it is passed to: `GetPropertyName`, `GetPropertyNameReverse`, `Length` and `ToString` remain; its constructor, `MaxLength` and `Dispose` are internal. Custom rules compiled against 1.x must be rebuilt. `ToString` writes array items as `[index]` (`a.b[0].c`, formerly `a.b..c`). 8. **`JsonWriter` can no longer be derived from outside the library**, `JsonWriter.FromUtf8JsonWriter` and `JsonWriter.Empty` are removed, and `WriteCommentValue` is gone (comments are never written). 9. **Internal now:** `JsonObserverException`, `PropertyPathMatch`, `JsonPropertyMatchDelegate`, `JsonPropertyPathMatchDelegate`, and the constructors of `JsonObjBuilder`, `JsonArrayBuilder`, `JsonValuePolicyBuilder` and their rule builders (start rules with `Match`). 10. **A UTF-8 byte order mark at the start of the input is skipped.** +### Fixed + +- **Rules of an `Obj(...)` inside a property's `Array(...)` now apply** at any depth (`root.Match("lines").Array(l => l.Obj(…))`, and `Array(a => a.Array(b => b.Obj(…)))`). They looked for their names one or more levels too deep, so under `BlockList` those values were written in clear and under `AllowList` the whole item was masked. Only an `Obj(...)` directly under a root `Array(...)` worked. +- **Relative rules receive `null` like absolute ones:** `MaskStr`, `MaskRawValue`, `MaskInt`, `MaskLong`, `MaskDecimal`, `MaskBool` and the read rules inside `Relative(...)` are called for a JSON `null`, as the rule-kinds table documents; `MaskAny` and `MaskAny(MaskTag)` keep `null` without calling the function. + ### Observer.Http 2.0.0 — breaking (since the preview builds) - `JsonBodyOutcome.Timeout` is replaced by `Canceled`: every cancellation, an `HttpClient.Timeout` included, is logged as `Canceled`. diff --git a/DragoAnt.System.Text.Json.Observer/package.readme.md b/DragoAnt.System.Text.Json.Observer/package.readme.md index 5a30d8e..d4edf1a 100644 --- a/DragoAnt.System.Text.Json.Observer/package.readme.md +++ b/DragoAnt.System.Text.Json.Observer/package.readme.md @@ -70,7 +70,7 @@ Console.WriteLine($"{result.Status} {Encoding.UTF8.GetString(output.WrittenSpan) ## More -- [Documentation](https://github.com/DragoAnt/Extensions.System.Text.Json#readme): rule kinds, allow-lists from your types (`JsonShape`), options, performance. +- [Documentation](https://github.com/DragoAnt/Extensions.System.Text.Json#readme): rule kinds, custom mask strategies, `Explain(path)`, allow-lists from your types (`JsonShape`), `ReadOnlySequence` input, options, performance. - [Changelog](https://github.com/DragoAnt/Extensions.System.Text.Json/blob/main/CHANGELOG.md), including the breaking changes from 1.x. - `DragoAnt.System.Text.Json.Observer.Http` logs masked `HttpClient` request and response bodies. - [Issues](https://github.com/DragoAnt/Extensions.System.Text.Json/issues) diff --git a/README.md b/README.md index 1cc8f82..d22ac1e 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,68 @@ Every `Mask*` rule masks the **whole value whatever its JSON type** — a sensit | `Unmasked()` | — writes a string, number, boolean or `null` unchanged | | | `ReadStr` / `ReadInt` / `ReadLong` / `ReadDecimal` / `ReadBool` / `ReadRaw` | hands the value to the context and writes it unchanged; a number that does not fit arrives as `null` | | -A strategy is a constant string, a `Regex` whose matches become `*`, or a function of the value and the context; a `null` result writes `null`. A value longer than `MaxValueBytes` reaches the function cut to that length. `Hash` uses `JsonObserverOptions.HashKey`, or a random key per process when it is empty. +The table holds for absolute and relative rules alike. A strategy is a constant string, a `Regex` whose matches become `*`, or a function of the value and the context; a `null` result writes `null`. A value longer than `MaxValueBytes` reaches the function cut to that length. `Hash` uses `JsonObserverOptions.HashKey`, or a random key per process when it is empty. + +### Custom mask strategies + +A `MaskTag` can carry a `Key` — a data classification, a redactor name — that only your `Utf8MaskStrategy` interprets; the built-in strategy falls back to the tag's kind (`MaskTag.Custom(key)` becomes `"***"`). Override `Mask(in Utf8MaskContext, JsonWriter)` to also see the property name and the path of the value, without allocations. + +```csharp +using System.Text; +using DragoAnt.System.Text.Json.Observer; +using DragoAnt.System.Text.Json.Observer.Strategies; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj(Relative(rules => rules + .Match("email").MaskAny(MaskTag.Custom("pii")) + .Match("password").MaskAny(MaskTag.Full), + BlockList)); +var options = new JsonObserverOptions(MaskStrategy: new LabelStrategy()); + +Console.WriteLine(masker.Mask("""{"user":{"email":"a@b.c","password":"s3cret"},"items":[{"email":"x@y.z"}]}""", options)); +// Output: +// {"user":{"email":"","password":"***"},"items":[{"email":""}]} + +sealed class LabelStrategy : Utf8MaskStrategy +{ + public override void Mask(in Utf8MaskContext context, JsonWriter writer) + { + if (context.Tag.TryGetKey(out var label)) + { + writer.WriteStringValue($"<{label} at {context.Path.ToString()}>"); + return; + } + + Default.Mask(context, writer); + } +} +``` + +`Utf8MaskContext` has `Value`, `TokenType`, `Tag`, `Options`, `PropertyName` (UTF-8), `IsArrayItem` and `Path`; `JsonWriter` takes `ReadOnlySpan` values too, so a char-based redactor writes its result without an intermediate string. + +### Explain a path + +`Explain` tells which rule or policy handles a path and what it does — handy to check a configuration, to document it, or to find out why a value was masked. It walks the rules exactly as the masking pass does. + +```csharp +using System.Text.Json; +using DragoAnt.System.Text.Json.Observer; +using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; + +var masker = JsonObserver.Obj( + root => root.Match("lines").Array(lines => lines.Obj(line => line.Match("sku").Unmasked())), + Relative(rules => rules.Match("password").MaskAny("***"), AllowList)); + +Console.WriteLine(masker.Explain("lines[2].sku")); +Console.WriteLine(masker.Explain("lines[2].qty", JsonTokenType.Number)); +Console.WriteLine(masker.Explain("user.password")); +// Output: +// lines[2].sku: Unchanged by Match("lines") > object item > Match("sku") → Unmasked() +// lines[2].qty: Masked by default policy AllowList → writes "***" +// user.password: Masked by relative Match("password") → MaskAny("***") +``` + +The result also lists one step per level (`Steps`) and the `Outcome`: `Unchanged`, `Masked`, `Read`, `Custom` or `Invalid`. Observers built from a `JsonShape` explain against the shape. ### Allow-list from your types @@ -156,7 +217,9 @@ Console.WriteLine(masker.Mask("""{"name":"Alice","card":"4111111111111111","adde sealed record Customer(string Name, string Card); ``` -`JsonShapeOptions` choose what happens to unknown members (`MaskWhole`, `Descend`, `PassThrough`) and whether `null` stays. On .NET 8, source-generated metadata carries no attributes, so classify by name there. +`JsonShapeOptions` choose what happens to unknown members (`MaskWhole`, `Descend`, `PassThrough`) and whether `null` stays; `JsonShapeOptions.FromSerializerOptions(options)` also matches names with the serializer's `PropertyNameCaseInsensitive`. On .NET 8, source-generated metadata carries no attributes, so classify by name there. + +Every node and member keeps its metadata for integrations: `JsonShape.Members` lists `JsonShapeProperty` items with the `JsonPropertyInfo`, the CLR member, `PropertyType`, `IsRequired`, `IsNullable` and `GetCustomAttributes()`, and nodes and members carry `Annotations` that an integration fills — for example from the `annotate` callback of `FromTypeInfo`. --- @@ -187,7 +250,7 @@ Console.WriteLine(result.Status); ### UTF-8 API for the hot path -`Mask(ReadOnlySpan, IBufferWriter, JsonObserverOptions?)` masks bytes into a writer you reuse; the string API produces exactly the same output for the same text. +`Mask(ReadOnlySpan, IBufferWriter, JsonObserverOptions?)` masks bytes into a writer you reuse; the string API produces exactly the same output for the same text. A payload held in several buffers, for example from a `PipeReader`, goes to `Mask(in ReadOnlySequence, …)` as is: the output is the same however the bytes are split. ```csharp using System.Buffers; @@ -217,12 +280,13 @@ Console.WriteLine($"{result.Status} {Encoding.UTF8.GetString(output.WrittenSpan) | `HashKey`, `MaskStrategy` | random per process, built-in | used by `MaskTag` rules | | `IgnoreNulls` | `false` | drops `null` properties and items, and objects and arrays left empty by that | | `Indented` | `false` | indented output | +| `PropertyNameCaseInsensitive` | `true` | match rule names and shapes ignoring case; pass the serializer's setting to match names as deserialization does | Input may contain comments and trailing commas; a UTF-8 byte order mark is skipped. Comments are not written. ## Performance -Masking walks the tokens once. With constant-string or tag rules, the UTF-8 API allocates a small constant amount per call — about 240 B in the repository's allocation test, the same for 1 KB and 64 KB bodies; a masking function receives a decoded `string`, which it allocates. The [benchmark report](./docs/benchmarks/benchmarks.md) compares speed and memory with a DOM masker and [JsonMasking](https://github.com/ThiagoBarradas/jsonmasking): for an 8 KB flat body the observer took 71 µs against 902 µs, and allocated 665 B against 468 KB. See the [OSS analogs comparison](./docs/comparisons/analogs.md) for how other libraries handle cut-off JSON. +Masking walks the tokens once. With constant-string or tag rules, the UTF-8 API allocates nothing per call once warm — the repository's allocation test pins 0 B for 1 KB and 64 KB bodies, spans and multi-segment sequences; a masking function receives a decoded `string`, which it allocates. The [benchmark report](./docs/benchmarks/benchmarks.md) compares speed and memory with a DOM masker and [JsonMasking](https://github.com/ThiagoBarradas/jsonmasking): for an 8 KB flat body the observer took 71 µs against 902 µs, and allocated 665 B against 468 KB. See the [OSS analogs comparison](./docs/comparisons/analogs.md) for how other libraries handle cut-off JSON. ## HTTP client body logging (`DragoAnt.System.Text.Json.Observer.Http`) From 6d7013e6011664ef655329a5fe739be53779df4f Mon Sep 17 00:00:00 2001 From: VasiliyF <5789590+vfofanov@users.noreply.github.com> Date: Sun, 4 Oct 2026 02:36:36 +0200 Subject: [PATCH 13/13] Drop the fixed nested-array pitfall and stale allocation figures The core API merge fixes rules of an Obj inside a property's Array, so the skill no longer documents it as a known issue. Warm UTF-8 calls with constant or tag rules now allocate 0 B; the skills and docs said ~240 B. The allocation test takes the minimum over five measured rounds, so a one-off tier-up allocation under a loaded test host no longer fails it. --- .../AllocationTests.cs | 15 ++++++---- docs/benchmarks/benchmarks.md | 2 +- docs/comparisons/analogs.md | 4 +-- skills/json-observer-masking/SKILL.md | 1 - skills/json-observer-masking/examples.md | 3 +- skills/json-observer-masking/pitfalls.md | 29 +------------------ skills/json-observer-masking/recipes.md | 2 +- 7 files changed, 16 insertions(+), 40 deletions(-) diff --git a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs index ee5ceb2..aff8ad3 100644 --- a/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs +++ b/DragoAnt.System.Text.Json.Observer.Tests.Shared/AllocationTests.cs @@ -62,14 +62,19 @@ void Call() Call(); } + // A one-off runtime allocation (tier-up under a loaded test host) lands in one round; a real per-call cost lands in all. const int calls = 50; - var before = GC.GetAllocatedBytesForCurrentThread(); - for (var i = 0; i < calls; i++) + var perCall = long.MaxValue; + for (var round = 0; round < 5 && perCall > 0; round++) { - Call(); - } + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < calls; i++) + { + Call(); + } - var perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / calls; + perCall = Math.Min(perCall, (GC.GetAllocatedBytesForCurrentThread() - before) / calls); + } perCall.Should().Be(0, $"{api} {shape} {size} B allocates {perCall} B per call"); } diff --git a/docs/benchmarks/benchmarks.md b/docs/benchmarks/benchmarks.md index 4427ce3..92cf011 100644 --- a/docs/benchmarks/benchmarks.md +++ b/docs/benchmarks/benchmarks.md @@ -34,7 +34,7 @@ Figures from the raw table below (bytes path = `ReadOnlySpan` into a reuse 2. **No Large Object Heap:** DOM maskers allocate 0.7–3.9 MB on 64 KB bodies, straight into the LOH; the observer stays far below the 85,000-byte threshold. 3. **Linear speed:** time grows linearly with body size, at 2.0–3.3× a bare unmasked reader-writer copy. -> These numbers were measured before the final 2.0 changes. Since then, rules with a constant or tag mask no longer decode the value, and the repository's allocation test (`AllocationTests`) measures about 240 B per call on the bytes path for 1 KB to 64 KB bodies, nested and arrays included. Re-run the benchmarks to refresh this report: +> These numbers were measured before the final 2.0 changes. Since then, rules with a constant or tag mask no longer decode the value, and the repository's allocation test (`AllocationTests`) pins 0 B per warm call on the bytes path for 1 KB to 64 KB bodies, nested and arrays included. Re-run the benchmarks to refresh this report: > > ```sh > dotnet run -c Release --project DragoAnt.System.Text.Json.Observer.Benchmarks diff --git a/docs/comparisons/analogs.md b/docs/comparisons/analogs.md index 13d270b..8f961c7 100644 --- a/docs/comparisons/analogs.md +++ b/docs/comparisons/analogs.md @@ -13,7 +13,7 @@ A common question is: **"Should we switch to an existing open-source library, or 4. **Serialization Mismatch:** Libraries like [Json.Masker](https://github.com/myarichuk/Json.Masker) operate during object serialization. At the HTTP handler/middleware layer, bodies arrive as raw byte streams; deserializing them into C# objects just to re-serialize them with masking adds enormous CPU and memory overhead. 5. **Lack of JSON Body Support:** Microsoft's official [Microsoft.Extensions.Compliance.Redaction](https://github.com/dotnet/extensions) redacts discrete string values by classification, but does not parse or traverse JSON bodies. -**[DragoAnt.System.Text.Json.Observer 2.0](https://github.com/DragoAnt/Extensions.System.Text.Json)** is the **only** high-performance, single forward-pass streaming engine (`Utf8JsonReader` → `Utf8JsonWriter`) in .NET. On its bytes API (`ReadOnlySpan` → `IBufferWriter`), it allocates a small constant amount per call (about 240 B with constant or tag rules), stays fail-closed on truncated bodies, and runs at 2–3× the raw token-copy floor. +**[DragoAnt.System.Text.Json.Observer 2.0](https://github.com/DragoAnt/Extensions.System.Text.Json)** is the **only** high-performance, single forward-pass streaming engine (`Utf8JsonReader` → `Utf8JsonWriter`) in .NET. On its bytes API (`ReadOnlySpan` → `IBufferWriter`), it allocates nothing per warm call with constant or tag rules, stays fail-closed on truncated bodies, and runs at 2–3× the raw token-copy floor. --- @@ -141,7 +141,7 @@ When downstream log forwarders (e.g. Datadog, Elastic, Loki, CloudWatch) receive ```csharp ReadOnlySpan utf8Json = ...; var result = observer.Mask(utf8Json, bufferWriter); - // about 240 B per call with constant or tag rules, whatever the body size + // 0 B per warm call with constant or tag rules, whatever the body size ``` 2. **Pre-Encoded UTF-8 Property Matching:** Rules are compiled once into pre-encoded UTF-8 byte sequences. During traversal, property names are compared directly on `ReadOnlySpan` via case-insensitive SIMD/ASCII routines without allocating `string` instances. diff --git a/skills/json-observer-masking/SKILL.md b/skills/json-observer-masking/SKILL.md index 3e7366f..9a3c1b1 100644 --- a/skills/json-observer-masking/SKILL.md +++ b/skills/json-observer-masking/SKILL.md @@ -50,7 +50,6 @@ Console.WriteLine(masker.Mask("""{"user":"alice","password":"s3cret","card":{"nu - "Everything became `***`" → you used the default `AllowList`; pass `BlockList` (to the factory, or as `Relative`'s second argument). - "The secret is still visible" → the rule is absolute but the field is nested, or the name differs (`Password` vs `passwd`); use `Relative` and a `PropMatches`. -- **Known 2.0.0 issue:** rules of an `Obj(...)` inside a property's `Array(...)` never match — under `BlockList` that value stays in clear. Use `Match("lines", anyItem, "qty")` instead ([pitfalls.md](./pitfalls.md#rules-inside-a-nested-array-do-not-match)). - A rule written before an `Obj(...)` rule for the same name wins and masks the whole object. - `JsonObserver.Obj(...)` on a root array returns `Invalid`; use `JsonObserver.Any(...)` when the root can be either. - Comments in the input are accepted and never written; a UTF-8 BOM is skipped. diff --git a/skills/json-observer-masking/examples.md b/skills/json-observer-masking/examples.md index 3ce75dc..e37fc59 100644 --- a/skills/json-observer-masking/examples.md +++ b/skills/json-observer-masking/examples.md @@ -131,8 +131,7 @@ Console.WriteLine(masker.Mask(""" `JsonObserver.Obj(...)` expects a root object and `JsonObserver.Array(...)` a root array; the other root is `Invalid`. `JsonObserver.Any(obj, array, policy)` accepts both. In an array builder every rule applies to every item; `Obj(...)` handles the items that are objects. -**An array item is one level of a property path.** A multi-name `Match` crosses one level per name, so `Match("lines", "sku")` never reaches `{"lines":[{"sku":…}]}`; put a match-anything test where the item is: `Match("lines", AnyItem, "sku")` with `AnyItem = new PropMatchingStrategy(_ => true)`. Use that form for **objects inside a nested array**: in 2.0.0, rules of an `Obj(...)` placed inside a property's `Array(...)` do not match (see [pitfalls.md](./pitfalls.md#rules-inside-a-nested-array-do-not-match)). - +**An array item is one level of a property path.** A multi-name `Match` crosses one level per name, so `Match("lines", "sku")` never reaches `{"lines":[{"sku":…}]}`; put a match-anything test where the item is: `Match("lines", AnyItem, "sku")` with `AnyItem = new PropMatchingStrategy(_ => true)`. ```csharp using DragoAnt.System.Text.Json.Observer; using DragoAnt.System.Text.Json.Observer.Strategies; diff --git a/skills/json-observer-masking/pitfalls.md b/skills/json-observer-masking/pitfalls.md index d0d1753..5ac10ef 100644 --- a/skills/json-observer-masking/pitfalls.md +++ b/skills/json-observer-masking/pitfalls.md @@ -47,33 +47,6 @@ Console.WriteLine(fixedRules.Mask(json)); // {"login":{"newPassword":"***"},"users":[{"password":"***"}]} ``` -## Rules inside a nested array do not match - -**Known issue in 2.0.0:** the rules of an `Obj(...)` placed inside a property's `Array(...)` — `root.Match("lines").Array(l => l.Obj(line => line.Match("qty")…))` — never match. Under `BlockList` the value stays in clear; under `AllowList` everything in the item is masked. An `Obj(...)` directly under a root `JsonObserver.Array(...)` works. - -**Fix:** address the items with a path that names the item level, or with relative rules. - -```csharp -using DragoAnt.System.Text.Json.Observer; -using DragoAnt.System.Text.Json.Observer.Strategies; -using static DragoAnt.System.Text.Json.Observer.JsonObserverValuePolicies; - -var anyItem = new PropMatchingStrategy(_ => true); -const string json = """{"lines":[{"sku":"A1","qty":2}]}"""; - -var broken = JsonObserver.Obj(root => root.Match("lines").Array(lines => lines.Obj(line => line.Match("qty").MaskAny("***"))), BlockList); -var byPath = JsonObserver.Obj(root => root.Match("lines", anyItem, "qty").MaskAny("***"), BlockList); -var byRelative = JsonObserver.Obj(Relative(rules => rules.Match("lines", anyItem, "qty").MaskAny("***"), BlockList)); - -Console.WriteLine(broken.Mask(json)); -Console.WriteLine(byPath.Mask(json)); -Console.WriteLine(byRelative.Mask(json)); -// Output: -// {"lines":[{"sku":"A1","qty":2}]} -// {"lines":[{"sku":"A1","qty":"***"}]} -// {"lines":[{"sku":"A1","qty":"***"}]} -``` - ## A whole object was replaced by `"***"` **Cause:** every `Mask*` rule matches containers too and masks them whole. A mask rule written **before** an `Obj(...)`/`Array(...)` rule for the same name wins, because the first matching rule wins. @@ -141,4 +114,4 @@ An observer compiles its rules when built and caches path buffers across calls. ## Expecting zero allocations -With constant or tag rules the UTF-8 API allocates a small constant amount per call (a few hundred bytes, whatever the body size). Masking functions receive a `string`, and the string API allocates the input and output strings. Set allocation budgets from measurements, not from "0 B". +With constant or tag rules the UTF-8 API allocates nothing per call once warm. Masking functions receive a `string`, and the string API allocates the input and output strings. Measure with `GC.GetAllocatedBytesForCurrentThread()` after a warm-up rather than assuming. diff --git a/skills/json-observer-masking/recipes.md b/skills/json-observer-masking/recipes.md index fd8271a..3fc75a1 100644 --- a/skills/json-observer-masking/recipes.md +++ b/skills/json-observer-masking/recipes.md @@ -69,7 +69,7 @@ static class Utf8Masking } ``` -In a real hot path, write `output.WrittenSpan` straight to the log sink instead of decoding it to a string. With constant or tag rules, each call allocates a small constant amount (a few hundred bytes, whatever the body size); a rule with a masking function allocates the `string` it receives. Measure with `GC.GetAllocatedBytesForCurrentThread()` rather than assuming. +In a real hot path, write `output.WrittenSpan` straight to the log sink instead of decoding it to a string. With constant or tag rules, a warm call allocates nothing; a rule with a masking function allocates the `string` it receives. Measure with `GC.GetAllocatedBytesForCurrentThread()` rather than assuming. ## Cut-off or invalid bodies