diff --git a/LICENSE b/LICENSE index bfc8d63..1bda40a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ MIT License Copyright (c) 2022 David Anthoff +Copyright (c) 2026 Jean-Francois Baffier Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index e537616..7354d32 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,90 @@ This package provides the `@testitem` macro for the test runner feature in VS Code. You can learn more over at the [documentation](https://www.julia-vscode.org/docs/stable/userguide/testitems/) for the entire test item ecosystem. Note that while the documentation is hosted as part of the Julia VS Code extension, the test item framework is not a VS Code extension specific technology and can be used entirely without VS Code. +## Preparation, warm-up and measurement regions + +Optional `@prepare`, `@warmup` and `@measure` annotations let a test item share +its correctness checks with external performance tools, without depending on a +particular measurement package: + +```julia +using TestItems + +@testitem "Sort a vector" begin + using TestItems: @prepare, @warmup, @measure + + @prepare begin + # Imports, input construction and other preparation. + values = [3, 1, 2] + end + + @warmup sort(values) + + @measure begin + sorted = sort(values) + end + + @test sorted == [1, 2, 3] + @test values == [3, 1, 2] +end +``` + +In a normal functional run, **every body executes once in source order**. +Explicit warm-up code is not skipped, and no hidden repetitions are added. The +annotations expand to their original expression: they add no closure, scope, +timer, runtime callback or runtime mode check. Bindings, imports, values, +exceptions, `return`, `break` and `continue` retain their ordinary behavior. +Package loading and macro expansion still have their normal Julia costs. + +Each annotation takes one expression, including a `begin ... end` block. +An external tool can use the annotations to exclude preparation and warm-up +from measurement. Keep assertions outside `@measure` if their cost should be excluded. +These annotations do not add any package dependencies or require changes to +TestItemRunner. They must be in scope in the item body, for example through the +normal package imports or an explicit `using TestItems`. + +### Interface for external tools + +`with_phase_transform` allows a consumer to replace phase syntax while fresh +item code is being expanded. For example, this observes the annotations without +changing their execution: + +```julia +using TestItems: with_phase_transform +using TestItemRunner + +regions = Symbol[] +transform = function (phase, body, source, caller) + push!(regions, phase) # Expansion-time observation, not runtime timing. + return body +end + +with_phase_transform(transform) do + TestItemRunner.run_tests("test") +end +``` + +The callback receives the phase (`:prepare`, `:warmup` or `:measure`), raw body +expression, call-site `LineNumberNode` and caller `Module`. It returns replacement +syntax **without** pre-escaping it; the phase macro escapes it into the caller's +scope. A consumer inserting calls to its own helpers should use `GlobalRef` and +preserve user bindings, control flow, values and exceptions. Measurement policy, +including handling failed or incomplete measurements, belongs to that consumer. +TestItems itself supplies no timer, repetitions or performance verdict. + +The context is task-local and restored on return or error. Nested contexts are +supported, and `with_phase_transform(nothing) do ... end` temporarily disables a +transform. The context is not automatically propagated to child tasks or other +processes. TestItemRunner's discovery does not expand or execute item bodies. + +This is an **expansion-time** interface. Wrap `include`, `include_string`, +`macroexpand` or a runner evaluating fresh bodies; wrapping an already compiled +function does not instrument it. In particular, an annotation written directly +inside the `do` block is expanded before the context is installed: + +```julia +with_phase_transform(transform) do + @measure sort([3, 1, 2]) # Already expanded; remains transparent. +end +``` + diff --git a/src/TestItems.jl b/src/TestItems.jl index 13e0604..528fe03 100644 --- a/src/TestItems.jl +++ b/src/TestItems.jl @@ -1,6 +1,9 @@ module TestItems export @testitem, @testmodule, @testsnippet +export @prepare, @warmup, @measure, with_phase_transform + +include("phases.jl") """ @testitem "name" [keyword_args...] begin diff --git a/src/phases.jl b/src/phases.jl new file mode 100644 index 0000000..ea238d0 --- /dev/null +++ b/src/phases.jl @@ -0,0 +1,85 @@ +const _PHASE_TRANSFORM_KEY = gensym(:TestItems_phase_transform) + +""" + with_phase_transform(f, transform) + with_phase_transform(transform) do + # Expand and evaluate fresh test-item code here. + end + +Call `f()` with an expansion-time phase transform in the current task. The +previous transform is restored on normal return and on exceptions. Pass `nothing` +to temporarily restore transparent expansion. The return value is that of `f()`. + +When a phase macro expands, `transform(phase, body, source, caller)` receives: + +- `phase`: one of `:prepare`, `:warmup` or `:measure`; +- `body`: the unevaluated expression supplied to the macro; +- `source::LineNumberNode`: the macro's call site; +- `caller::Module`: the module in which it expands. + +Return replacement Julia syntax, not a pre-escaped expression. The macro escapes +the replacement into the caller's scope. A consumer must preserve the intended +scope, values, control flow and exception behavior of the user code. + +The transform runs during **macro expansion**, never as an implicit runtime +callback. Wrap `include`, `include_string`, `macroexpand`, or a runner that +evaluates fresh item bodies. Already expanded or compiled code is unchanged; +wrapping a call to such a function does not instrument it. The transform is +task-local, is not installed globally, and is not automatically propagated to +other tasks or processes. + +The package supplies no timer, repetition policy or performance backend. +""" +function with_phase_transform(f::Function, transform) + task_local_storage(f, _PHASE_TRANSFORM_KEY, transform) +end + +function _expand_phase(phase::Symbol, body, source::LineNumberNode, caller::Module) + transform = get(task_local_storage(), _PHASE_TRANSFORM_KEY, nothing) + replacement = transform === nothing ? body : transform(phase, body, source, caller) + esc(replacement) +end + +""" + @prepare expression + @prepare begin + # Imports, input construction and other preparation. + end + +Mark preparation code. Normally this expands directly to `expression`, evaluated +once in the caller's existing scope, with its original value and exceptions. +Top-level imports and bindings remain usable by later phases and assertions. +No closure, local scope, timer or runtime callback is added. +""" +macro prepare(body) + _expand_phase(:prepare, body, __source__, __module__) +end + +""" + @warmup expression + @warmup begin + # Explicit warm-up operations. + end + +Mark explicit warm-up code. Normally its body runs once, in source order, just +as if the marker were absent. It is not skipped and no hidden repetitions are +added. A performance consumer can interpret this region through an explicit +expansion-time transform; functional runs keep the ordinary code semantics. +""" +macro warmup(body) + _expand_phase(:warmup, body, __source__, __module__) +end + +""" + @measure expression + @measure begin + # The operation selected for measurement by an external tool. + end + +Mark a measurement region. Without an explicit transform this is only an +annotation: it executes the original body once and returns its value, without +measuring anything. Bindings remain visible to subsequent correctness checks. +""" +macro measure(body) + _expand_phase(:measure, body, __source__, __module__) +end diff --git a/test/phases.jl b/test/phases.jl new file mode 100644 index 0000000..930bbcc --- /dev/null +++ b/test/phases.jl @@ -0,0 +1,265 @@ +@testitem "Phase annotations" begin + using TestItemRunner + + const PHASES = (:prepare, :warmup, :measure) + + unline(expr) = expr isa Expr ? Base.remove_linenums!(deepcopy(expr)) : expr + phasecall(phase, body; line=1) = Expr(:macrocall, Symbol("@", phase), + LineNumberNode(line, :phase_fixture), body) + expand(phase, body; kwargs...) = macroexpand(@__MODULE__, phasecall(phase, body; kwargs...)) + + function passed_assertions(testset::Test.DefaultTestSet) + passed = testset.n_passed + for result in testset.results + result isa Test.DefaultTestSet && (passed += passed_assertions(result)) + end + passed + end + + const RUNTIME_PHASES = Symbol[] + record_runtime_phase(phase) = (push!(RUNTIME_PHASES, phase); nothing) + + function ordinary_workload() + @prepare begin + body = [3, 1, 2] + transform = 40 + events = [:prepare] + end + @warmup begin + push!(events, :warmup) + sort(body) + end + @measure begin + push!(events, :measure) + result = sort(body) + end + (result, body, transform + 2, events) + end + + function ordinary_control_flow() + total = 0 + for x in 1:10 + @prepare x == 2 && continue + @warmup x == 5 && break + @measure total += x + end + @measure return total + error("unreachable") + end + + @testset "TestItems" begin + @testset "Markers erase to the original expression" begin + for phase in PHASES + for body in (:(x + 1), :(begin x = [1, 2]; sum(x) end), 42, :nothing) + @test unline(expand(phase, body)) == unline(body) + end + @test Core.eval(@__MODULE__, expand(phase, :(6 * 7))) == 42 + end + @test ordinary_workload() == ([1, 2, 3], [3, 1, 2], 42, + [:prepare, :warmup, :measure]) + @test ordinary_control_flow() == 8 + end + + @testset "Top-level imports and caller bindings" begin + caller = Module(gensym(:PhaseCaller)) + Base.include_string(caller, """ + using TestItems + @prepare begin + using Base: identity + values = [3, 1, 2] + end + @warmup warmed = identity(values) + @measure sorted = sort(warmed) + result = (sorted, values) + """, "top_level_phases.jl") + @test getfield(caller, :result) == ([1, 2, 3], [3, 1, 2]) + @test getfield(caller, :warmed) === getfield(caller, :values) + end + + @testset "Original exceptions propagate unchanged" begin + sentinel = ErrorException("original exception") + for phase in PHASES + caller = Module(gensym(:ThrowCaller)) + Core.eval(caller, :(using TestItems)) + Core.eval(caller, :(const sentinel = $sentinel)) + caught = try + Core.eval(caller, phasecall(phase, :(throw(sentinel)))) + catch error + error + end + @test caught === sentinel + end + end + + @testset "Explicit expansion-time consumer" begin + records = Any[] + transform = function (phase, body, source, caller) + push!(records, (; phase, source, caller)) + phase === :measure ? :($body + 1) : body + end + result = with_phase_transform(transform) do + expression = expand(:measure, :(40 + 1); line=27) + @test length(records) == 1 + Core.eval(@__MODULE__, expression) + end + @test result == 42 + @test length(records) == 1 # Evaluation did not call the transform. + @test records[1].phase == :measure + @test records[1].source.line == 27 + @test records[1].source.file == :phase_fixture + @test records[1].caller === @__MODULE__ + @test unline(expand(:measure, :(40 + 1))) == :(40 + 1) + + calls = Ref(0) + with_phase_transform((args...) -> (calls[] += 1; args[2])) do + @test ordinary_workload()[1] == [1, 2, 3] + end + @test calls[] == 0 # This function's markers had already expanded. + + with_phase_transform((args...) -> nothing) do + @test expand(:warmup, :(error("not evaluated"))) === nothing + end + end + + @testset "Nesting, disabling and failure restore the context" begin + seen = Symbol[] + outer = (phase, body, source, caller) -> (push!(seen, :outer); body) + inner = (phase, body, source, caller) -> (push!(seen, :inner); body) + sentinel = ErrorException("transform failed") + with_phase_transform(outer) do + expand(:prepare, :nothing) + with_phase_transform(inner) do + expand(:warmup, :nothing) + end + with_phase_transform(nothing) do + expand(:measure, :nothing) + end + caught = try + with_phase_transform((args...) -> throw(sentinel)) do + expand(:measure, :nothing) + end + catch error + error + end + @test (caught isa LoadError ? caught.error : caught) === sentinel + @test_throws ErrorException with_phase_transform(inner) do + error("body failed") + end + expand(:measure, :nothing) + end + @test seen == [:outer, :inner, :outer] + @test unline(expand(:prepare, :(1 + 1))) == :(1 + 1) + end + + @testset "Transforms are isolated between tasks" begin + seen_a, seen_b = Symbol[], Symbol[] + plain_in_b = Ref{Any}() + ready, resume = Channel{Nothing}(1), Channel{Nothing}(1) + a = @async with_phase_transform((phase, body, source, caller) -> + (push!(seen_a, phase); body)) do + expand(:prepare, :nothing) + put!(ready, nothing) + take!(resume) + expand(:measure, :nothing) + end + b = @async begin + take!(ready) + plain_in_b[] = unline(expand(:warmup, :(1 + 1))) + with_phase_transform((phase, body, source, caller) -> + (push!(seen_b, phase); body)) do + expand(:warmup, :nothing) + end + put!(resume, nothing) + end + wait(a) + wait(b) + @test plain_in_b[] == :(1 + 1) + @test seen_a == [:prepare, :measure] + @test seen_b == [:warmup] + + with_phase_transform((args...) -> error("must not leak into child")) do + child = @async expand(:measure, :(6 * 7)) + @test unline(fetch(child)) == :(6 * 7) + end + end + + @testset "Unmodified TestItemRunner" begin + mktempdir() do fixtures + write(joinpath(fixtures, "items.jl"), raw""" +using TestItems + +@testsnippet PhaseImports begin + using Test + using TestItems: @prepare, @warmup, @measure +end + +@testitem "shared phases retain bindings and order" default_imports=false setup=[PhaseImports] begin + @prepare begin + values = [3, 1, 2] + phases = [:prepare] + end + @warmup begin + push!(phases, :warmup) + sort(values) + end + @measure begin + push!(phases, :measure) + sorted = sort(values) + end + @test phases == [:prepare, :warmup, :measure] + @test sorted == [1, 2, 3] + @test values == [3, 1, 2] +end + +@testitem "measurement keeps exceptions and assertions" default_imports=false setup=[PhaseImports] begin + @prepare sentinel = ErrorException("original exception") + @warmup warm = 41 + caught = try + @measure throw(sentinel) + catch error + error + end + @test caught === sentinel + @test warm + 1 == 42 + @test (@measure 6 * 7) == 42 +end + """) + discovered = String[] + transforms = Symbol[] + observer = (phase, body, source, caller) -> + (push!(transforms, phase); body) + with_phase_transform(observer) do + TestItemRunner.run_tests(fixtures; filter=item -> begin + push!(discovered, item.name) + false + end) + end + @test length(discovered) == 2 + @test isempty(transforms) # Discovery did not expand or run the bodies. + + plain = TestItemRunner.run_tests(fixtures) + @test passed_assertions(plain) == 6 + @test isempty(transforms) + + consumer = function (phase, body, source, caller) + observer(phase, body, source, caller) + Expr(:block, Expr(:call, GlobalRef(@__MODULE__, :record_runtime_phase), + QuoteNode(phase)), body) + end + observed = with_phase_transform(consumer) do + TestItemRunner.run_tests(fixtures) + end + @test passed_assertions(observed) == 6 + @test count(==(:prepare), transforms) == 2 + @test count(==(:warmup), transforms) == 2 + @test count(==(:measure), transforms) == 3 + @test count(==(:prepare), RUNTIME_PHASES) == 2 + @test count(==(:warmup), RUNTIME_PHASES) == 2 + @test count(==(:measure), RUNTIME_PHASES) == 3 + before = copy(RUNTIME_PHASES) + TestItemRunner.run_tests(fixtures) + @test RUNTIME_PHASES == before # The consumer did not leak into later runs. + end + end + end +end