From eecaf3eae3fed50bad877adb92b8056fdab2b058 Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Sun, 13 Sep 2026 15:25:50 +0200 Subject: [PATCH 1/3] Fix assignment of timing stats Previously if a test errored when stats was enabled the exception would propagate and running the tests would hang. --- src/testset.jl | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/testset.jl b/src/testset.jl index 6478e83..bb54413 100644 --- a/src/testset.jl +++ b/src/testset.jl @@ -525,7 +525,7 @@ function testset_beginend(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, d Random.seed!(ReTest.test_seed[]) end let - ts.timed = @stats $stats $(esc(tests)) + @stats $stats ts $(esc(tests)) end catch err err isa InterruptException && rethrow() @@ -580,7 +580,7 @@ function testset_forloop(mod::Module, isfinal::Bool, pat::Pattern, id::Int64, Test.TESTSET_DEPTH => get_testset_depth() + 1, try let - ts.timed = @stats $stats $(esc(tests)) + @stats $stats ts $(esc(tests)) end setresult!($marks, ts.subject, !anyfailed(ts)) catch err @@ -648,7 +648,7 @@ get_timed!(ts) = isempty(ts.timed) ? set_timed!(ts) : ts # adapted from @timed in Julia/base/timing.jl # also, @timed inserts a `while false; end` compiler heuristic, which destroys perfs here -macro stats(yes, ex) +macro stats(yes, ts, ex) quote if $yes local stats = Base.gc_num() @@ -656,18 +656,23 @@ macro stats(yes, ex) local rss = Sys.maxrss() local compile_time = cumulative_compile_time_ns() end - local val = $(esc(ex)) - if $yes - elapsedtime = time_ns() - elapsedtime - local diff = Base.GC_Diff(Base.gc_num(), stats) - rss = Sys.maxrss() - rss - compile_time = cumulative_compile_time_ns() - compile_time - # COMPAT: on Julia 1.1, the form `(time=..., bytes=..., ...)` doesn't work - # (macro name mangling with `#`, e.g. (#115#time = ..., )) - NamedTuple{(:time, :bytes, :gctime, :rss, :compile_time)}( - (elapsedtime/1e9, diff.allocd, diff.total_time/1e9, rss, compile_time)) - else - NamedTuple() + + try + $(esc(ex)) + finally + if $yes + elapsedtime = time_ns() - elapsedtime + Base.cumulative_compile_timing(false) + compile_time = cumulative_compile_time_ns() - compile_time + local diff = Base.GC_Diff(Base.gc_num(), stats) + rss = Sys.maxrss() - rss + # COMPAT: on Julia 1.1, the form `(time=..., bytes=..., ...)` doesn't work + # (macro name mangling with `#`, e.g. (#115#time = ..., )) + $(esc(ts)).timed = + NamedTuple{(:time, :bytes, :gctime, :rss, :compile_time)}( + (elapsedtime/1e9, diff.allocd, diff.total_time/1e9, rss, + compile_time)) + end end end end From 9e19d51c0544a15c61c05e83dc5a0b1a35b94fcd Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Sun, 13 Sep 2026 15:26:49 +0200 Subject: [PATCH 2/3] Fix compilation stats - Compilation timing needs to be enabled explicitly. - We were previously overcounting the compilation time by counting recompilation twice. --- src/testset.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/testset.jl b/src/testset.jl index bb54413..c71f997 100644 --- a/src/testset.jl +++ b/src/testset.jl @@ -654,6 +654,7 @@ macro stats(yes, ts, ex) local stats = Base.gc_num() local elapsedtime = time_ns() local rss = Sys.maxrss() + Base.cumulative_compile_timing(true) local compile_time = cumulative_compile_time_ns() end @@ -677,6 +678,8 @@ macro stats(yes, ts, ex) end end -cumulative_compile_time_ns() = sum(Base.cumulative_compile_time_ns()) +# `Base.cumulative_compile_time_ns()` returns (compilation, recompilation), where the +# second is the part of the first spent recompiling. +cumulative_compile_time_ns() = Base.cumulative_compile_time_ns()[1] end # module From dfe7ddb8348aae49c02e5edc8bd035e7764fbfcc Mon Sep 17 00:00:00 2001 From: JamesWrigley Date: Sun, 13 Sep 2026 16:16:12 +0200 Subject: [PATCH 3/3] Implement a REPL mode for running tests --- Project.toml | 17 ++- docs/Project.toml | 8 +- docs/src/index.md | 105 +++++++++++---- ext/ReTestREPLExt.jl | 113 +++++++++++++++++ src/ReTest.jl | 77 ++++++++++- src/repl.jl | 296 +++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 243 +++++++++++++++++++++++++++++++++++ 7 files changed, 825 insertions(+), 34 deletions(-) create mode 100644 ext/ReTestREPLExt.jl create mode 100644 src/repl.jl diff --git a/Project.toml b/Project.toml index da389a1..4f7b618 100644 --- a/Project.toml +++ b/Project.toml @@ -1,24 +1,39 @@ name = "ReTest" uuid = "e0db7c4e-2690-44b9-bad6-7687da720f89" -version = "0.4.1" +version = "0.4.2" authors = ["Rafael Fourquet "] [deps] Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" InlineTest = "bd334432-b1e7-49c7-a2dc-dd9149e4ebd6" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +Preferences = "21216c6a-2e73-6563-6e65-726566657250" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +ReplMaker = "b873ce64-0db9-51f5-a568-4457d8e49576" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TestEnv = "1e6cf692-eddd-4d53-88a5-2d735e33781b" + +[weakdeps] +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [sources] InlineTest = {path = "InlineTest"} +[extensions] +ReTestREPLExt = "REPL" + [compat] InlineTest = "=0.2.0" PrecompileTools = "1.2.1" +Preferences = "1.6" +REPL = "1" +ReplMaker = "0.2.8" Revise = "3.1" +TOML = "1" +TestEnv = "1.102" julia = "1.13" [extras] diff --git a/docs/Project.toml b/docs/Project.toml index 1d16967..02c93e0 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,9 +3,9 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" InlineTest = "bd334432-b1e7-49c7-a2dc-dd9149e4ebd6" ReTest = "e0db7c4e-2690-44b9-bad6-7687da720f89" +[sources] +InlineTest = {path = "../InlineTest"} +ReTest = {path = ".."} + [compat] Documenter = "1" - -[sources] -ReTest = { path = ".." } -InlineTest = { path = "../InlineTest" } diff --git a/docs/src/index.md b/docs/src/index.md index b918653..392a5a8 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -125,40 +125,52 @@ end end # module ``` -We can now load tests either via `using MyPackageTests`, if `LOAD_PATH` is configured -appropriately, or via `include`, and run whichever tests we want: +We can now load tests either via `using MyPackageTests; +MyPackageTests.runtests()` as above, or with the ReTest REPL mode. When `ReTest` +is loaded in an interactive REPL, pressing `}` at the start of an empty `julia>` +line enters the `retest>` mode. Here are some example commands: + +| Input | Equivalent call | +|:-----------------------|:-----------------------------------| +| `run` (`r`) | `retest()` | +| `run foo bar` | `retest("foo", "bar")` | +| `run foo -slow` | `retest("foo", "-slow")` | +| `run "two words"` | `retest("two words")` | +| `run 3 -4` | `retest(3, -4)` | +| `run :label` | `retest(:label)` | +| `run foo _dv2` | `retest("foo", :_dv2)` | +| `run foo verbose=2` | `retest("foo"; verbose=2)` | +| `run tag=[:a,:b]` | `retest(; tag=[:a, :b])` | +| `dry-run foo` (`dr`) | `retest("foo"; dry=true)` | +| `run-failed` (`rf`) | `retest(fail)` | +| `load` | (see below) | +| `set verbose 2` | (see [Preferences](@ref)) | +| `unset verbose` | (see [Preferences](@ref)) | +| `help` (`?`) | (prints a summary of the commands) | + +For `run`, every argument which isn't a number, a `:label`, a `_`-prefixed +shorthand or a `key=value` keyword is passed as a pattern to +[`retest`](@ref). The value of a keyword is evaluated as Julia code, unless it +is quoted, in which case it is passed as a string. + +The `load` command activates the test environment of the active project with +[TestEnv.jl](https://github.com/JuliaTesting/TestEnv.jl) and loads its tests +from `test/Tests.jl` with `load()`. The package itself doesn't +have to be loaded beforehand, as the test file can load it once the test +environment is active. For example, in the project of a +package `Foo`: ```julia -julia> include("test/MyPackageTests.jl"); +julia> using ReTest # assumes ReTest.jl is installed in your base environment -julia> using ReTest # to use the `retest` function +retest> load # activates Foo's test environment and loads `FooTests` -julia> retest(dry=true, verbose=2) # just list tests, showing nested ones -MyPackage -1| greet +retest> dr # display all testsets (dry-run) -Main.MyPackageTests -1| more greet -2| concatenation -3| stuff +retest> r stats=true # run all tests with stats -julia> retest("greet", verbose=2) # run only tests related to `greet()` - Pass -MyPackage: - greet | 1 - -Main.MyPackageTests: - more greet | 1 - concatenation | 1 - -Overall | 2 - -julia> MyPackageTests.runtests(3) # run only testset with ID 3 in MyPackageTests - Pass -3| stuff | 1 +retest> rf # run last failing tests (run-failed) ``` -Here it is for basic usage! - ## API @@ -364,6 +376,45 @@ instead of `@test true`, it could be useful to wrap it in its own testset, so th it can be filtered out. +## Preferences +The default value of the `stats`, `verbose`, `id`, `marks`, and `spin` keywords +of [`retest`](@ref) can be set persistently for a project with +[`Preferences.jl`](https://github.com/JuliaPackaging/Preferences.jl), either by +manually editing a preferences file or by using the `set` and `unset` commands +of the REPL mode. `set` prints the current preferences when called +without arguments: +```julia +retest> set verbose inf + +retest> set spin false + +retest> unset spin # remove the `spin` preference + +retest> set +stats = false +verbose = Inf +id = nothing +marks = true +spin = true +``` + +This writes the values in the `LocalPreferences.toml` file of the active project, +where they can also be edited by hand: +```toml +[ReTest] +verbose = inf +spin = false +``` + +These preferences are read each time `retest` is called so changing one takes +effect immediately. These particular preferences also will not trigger +recompilation of `ReTest`. They are overridden by the keywords explicitly passed +to `retest`. + +The preferences file the REPL mode uses will be in the `test/` directory when +that contains a project file, or the package root directory otherwise. + + ## Running tests in parallel with `Distributed` Currently, the tests are automatically run in parallel whenever there are diff --git a/ext/ReTestREPLExt.jl b/ext/ReTestREPLExt.jl new file mode 100644 index 0000000..e6deb1c --- /dev/null +++ b/ext/ReTestREPLExt.jl @@ -0,0 +1,113 @@ +module ReTestREPLExt + +import REPL +import TOML +using Base.CoreLogging: CoreLogging +using ReplMaker: initrepl, FunctionCompletionProvider +using ReTest: ReTest +using TestEnv: TestEnv + + +function load_tests() + project = Base.active_project() + name = get(TOML.parsefile(project), "name", nothing) + if isnothing(name) + error("the active project ($project) is not a package") + end + # must be computed before activating, which changes the active project + testdir = joinpath(dirname(project), "test") + testpath = joinpath(testdir, "$(name)Tests.jl") + if !isfile(testpath) + error("no test file found at $testpath") + end + + @info "Switching to test environment..." + TestEnv.activate() + try + @info "Switched to $(Base.active_project())" + keep_preferences(project, testdir) + ReTest.load(testpath) + @info "Loaded $testpath" + catch + # leave the session as it was, so `load` can be tried again + Base.set_active_project(project) + undo_keep_preferences() + rethrow() + end +end + +# The LOAD_PATH entry added by `keep_preferences`, if any +const pushed_env = Ref{Union{Nothing, String}}(nothing) + +# Preferences set in the temporary test environment would be lost with it, so write +# them to the project `Pkg.test` reads them from, and put it on the load path to read +# them back (the active temporary project doesn't inherit from it) +function keep_preferences(project, testdir) + testproject = nothing + for name in Base.project_names + file = joinpath(testdir, name) + if isfile(file) + testproject = file + break + end + end + if !isnothing(testproject) + project = testproject + end + ReTest.preferences_project = project + env = dirname(project) + if !(env in Base.LOAD_PATH) + push!(Base.LOAD_PATH, env) + pushed_env[] = env + end +end + +function undo_keep_preferences() + ReTest.preferences_project = nothing + env = pushed_env[] + if !isnothing(env) + filter!(!=(env), Base.LOAD_PATH) + pushed_env[] = nothing + end +end + +# ReplMaker warns that `}` is already bound by the REPL's bracket insertion, which it +# preserves anyway when not at the start of the line: drop that message, keep the rest +struct DropKeyWarning{L<:CoreLogging.AbstractLogger} <: CoreLogging.AbstractLogger + parent::L +end + +CoreLogging.shouldlog(l::DropKeyWarning, args...) = CoreLogging.shouldlog(l.parent, args...) +CoreLogging.min_enabled_level(l::DropKeyWarning) = CoreLogging.min_enabled_level(l.parent) +CoreLogging.catch_exceptions(l::DropKeyWarning) = CoreLogging.catch_exceptions(l.parent) + +function CoreLogging.handle_message(l::DropKeyWarning, level, message, args...; kwargs...) + if level == CoreLogging.Warn && startswith(string(message), "REPL key ") + return nothing + end + CoreLogging.handle_message(l.parent, level, message, args...; kwargs...) +end + +function init_repl_mode(repl) + CoreLogging.with_logger(DropKeyWarning(CoreLogging.current_logger())) do + initrepl(ReTest.repl_parse; + repl, + prompt_text = "retest> ", + prompt_color = :magenta, + start_key = '}', + mode_name = :retest, + completion_provider = FunctionCompletionProvider(ReTest.repl_completions), + startup_text = false) + end +end + +function __init__() + ReTest.repl_load_tests_hook = load_tests + if isdefined(Base, :active_repl) && Base.active_repl isa REPL.LineEditREPL + init_repl_mode(Base.active_repl) + elseif isinteractive() + Base.atreplinit(init_repl_mode) + end +end + +end diff --git a/src/ReTest.jl b/src/ReTest.jl index e331e89..8b9bc9a 100644 --- a/src/ReTest.jl +++ b/src/ReTest.jl @@ -6,6 +6,7 @@ using Distributed using Base.Threads: nthreads import Base: == using Random: shuffle!, randstring +using Preferences: Preferences, load_preference # from Test: export Test, @@ -50,6 +51,7 @@ include("testset.jl") include("hijack.jl") include("watch.jl") include("patterns.jl") +include("repl.jl") using .Testset: Testset, Format, print_id @@ -609,11 +611,79 @@ const retest_defaults = ( def(kw::Symbol) = if isdefined(Main, :__retest_defaults__) # TODO: test __retest_defaults__ - get(Main.__retest_defaults__, kw, retest_defaults[kw]) + get(() -> preference(kw), Main.__retest_defaults__, kw) else - retest_defaults[kw] + preference(kw) end +# keywords whose default value can be set persistently with Preferences.jl +const PREFERENCES = (:stats, :verbose, :id, :marks, :spin) + +# Project file `set_preferences!` writes to; `nothing` means the active project. +# The REPL extension sets it after `load`, when the active project is a temporary +# test environment which would take the preferences down with it. +global preferences_project::Union{Nothing,String} = nothing + +# `force_compiletime_default` keeps the preference out of ReTest's compile-time +# dependencies, so that setting one doesn't trigger a recompilation +function preference(kw::Symbol) + default = retest_defaults[kw] + if !(kw in PREFERENCES) + return default + end + val = load_preference(@__MODULE__, String(kw), default; + force_compiletime_default=true) + check_preference(kw, val) +end + +function check_preference_name(kw::Symbol) + if !(kw in PREFERENCES) + throw(ArgumentError("unsupported preference `$kw`, must be one of: " * + join(PREFERENCES, ", "))) + end + kw +end + +function check_preference(kw::Symbol, val) + check_preference_name(kw) + valid = if kw === :verbose + val isa Real + elseif kw === :id + val isa Bool || isnothing(val) # `nothing` is the default + else # :spin, :stats, :marks + val isa Bool + end + if !valid + throw(ArgumentError("invalid value for the `$kw` preference: $(repr(val))")) + end + val +end + +# `missing` unsets a preference +function set_preferences!(; force::Bool=true, kwargs...) + prefs = Pair{String,Any}[] + for (kw, val) in kwargs + if ismissing(val) + check_preference_name(kw) + elseif isnothing(val) + # `Preferences` stores it as a directive blocking the preferences set in + # other environments of the load path, which we don't want to expose + check_preference_name(kw) + throw(ArgumentError("the `$kw` preference can't be set to `nothing`, " * + "pass `missing` to unset it")) + else + check_preference(kw, val) + end + push!(prefs, String(kw) => val) + end + project = preferences_project + if isnothing(project) + Preferences.set_preferences!(@__MODULE__, prefs...; force=force) + else + Preferences.set_preferences!(@__MODULE__, prefs...; + project_toml=project, force=force) + end +end """ retest(mod..., pattern...; @@ -680,6 +750,9 @@ Filtering `pattern`s can be specified to run only a subset of the tests. The default values of these keywords can be overriden by defining a dictionary or named tuple within `Main` called `__retest_defaults__`, whose keys are symbols. E.g. `__retest_defaults__ = (verbose=Inf, spin=false)`. +The default values of `stats`, `verbose`, `id`, `marks` and `spin` can also be set +persistently with the `set` command of the `retest>` REPL mode, which +`__retest_defaults__` takes precedence over. ### Filtering diff --git a/src/repl.jl b/src/repl.jl new file mode 100644 index 0000000..8ec5db2 --- /dev/null +++ b/src/repl.jl @@ -0,0 +1,296 @@ +# Parsing of the `retest>` REPL mode input, see ext/ReTestREPLExt.jl. These +# functions are here instead of in the extension to make testing easier. + +const REPL_COMMANDS = ["dry-run", "help", "load", "run", "run-failed", "set", "unset"] + +const REPL_ALIASES = Dict("r" => "run", "dr" => "dry-run", "rf" => "run-failed", + "?" => "help") + +# set by the REPL extension, which is the only place where TestEnv is loaded +global repl_load_tests_hook::Union{Nothing,Function} = nothing + +function repl_help() + print(""" + Press `}` at the start of an empty `julia>` line to enter `retest>` mode, and + backspace on an empty line to exit. Each line must start with a command: + + run (r) ... run the testsets matching the given arguments, which are + passed to `retest`; without arguments, run all tests + dry-run (dr) ... list matching testsets without running them + run-failed (rf) ... run testsets which failed in the previous run + load activate the test environment of the active project and + load its tests from test/Tests.jl (only once + per session) + set [key value] show the preferences, or set one persistently, e.g. + `set verbose 2` or `set spin false` + unset key... remove the given preferences, e.g. `unset spin` + help (?) show this help + + The arguments of `run` and friends are translated as follows: + + run foo bar testsets matching both "foo" and "bar" + run foo -slow testsets matching "foo" but not "slow" + run "two words" quoted pattern + run 3 -3 select/exclude the testset with ID 3 + run :label testsets tagged with :label + run _dv2 keyword shorthand (here dry=true, verbose=2) + run verbose=2 keyword argument + run tag=[:a,:b] keyword argument, whose value is Julia code unless quoted + """) +end + +struct ReplToken + text::String # for a `key=value` token, the value + quoted::Bool # whether `text` was given as a quoted string + key::Union{Nothing,String} # the `key` of a `key=value` token +end + +function repl_tokenize(line::AbstractString) + tokens = ReplToken[] + word = "" + inword = false + quoted = false + inquote = false + key = nothing + + for c in line + if inquote + if c == '"' + inquote = false + else + word *= c + end + elseif c == '"' + inquote = true + inword = true + quoted = true + elseif isspace(c) + if inword + push!(tokens, ReplToken(word, quoted, key)) + word = "" + inword = false + quoted = false + key = nothing + end + elseif c == '=' && isnothing(key) && !quoted && Base.isidentifier(word) + # `key=value`: what follows is the value, and `quoted` from now on + # records the quoting of the value alone + key = word + word = "" + else + word *= c + inword = true + end + end + + if inquote + throw(ArgumentError("unterminated string in retest input: $line")) + end + if inword + push!(tokens, ReplToken(word, quoted, key)) + end + + tokens +end + +# `load` and other means (e.g. `using MyPackageTests`) both register in TESTED_MODULES +function repl_tests_loaded() + update_TESTED_MODULES!() # drops stale entries, as `retest` does + + # ReTest's precompilation workload registers test modules, which don't count + any(TESTED_MODULES) do mod + !isnothing(mod) && Base.moduleroot(mod) !== @__MODULE__ + end +end + +function repl_load_tests() + hook = repl_load_tests_hook + if isnothing(hook) + error("`load` requires ReTest's REPL extension to be loaded") + end + + if !isnothing(preferences_project) + # only a previous `load` sets it, so `using MyPackageTests` doesn't get in the way + error("`load` can be used only once per session, as the active project is " * + "then a test environment; restart Julia to load the tests of another " * + "package") + end + + hook() +end + +# `run` and friends go through here, rather than silently running nothing +function repl_retest(args...; kwargs...) + if !repl_tests_loaded() + error("no tests are loaded: use the `load` command to activate the test " * + "environment of the active project and load its tests") + end + + retest(args...; kwargs...) +end + +# Only commands are completed, which appear only as the first word +function repl_completions(before_cursor::AbstractString) + if any(isspace, before_cursor) + return String[], "" + end + + filter(c -> startswith(c, before_cursor), REPL_COMMANDS), String(before_cursor) +end + +# before `load`, the active project is whatever the session started in, which isn't +# where a package's test preferences belong +function check_preferences_project() + if isnothing(preferences_project) + error("no tests are loaded: use the `load` command first, so that the " * + "preferences are read from and stored in the project of the tested " * + "package") + end +end + +function repl_set_preferences!(; kwargs...) + check_preferences_project() + set_preferences!(; kwargs...) +end + +function repl_show_preferences() + check_preferences_project() + for kw in PREFERENCES + println(kw, " = ", preference(kw)) + end +end + +# the value of a preference, as a literal +function repl_value(tok::ReplToken) + text = tok.text + if tok.quoted + return text + end + + if text == "true" + true + elseif text == "false" + false + elseif text == "nothing" + nothing + else + int = tryparse(Int, text) + if !isnothing(int) + return int + end + + float = tryparse(Float64, text) # also handles "inf" + if !isnothing(float) + return float + end + + text + end +end + +# a preference name is a bare identifier, not a pattern or a `key=value` +function repl_prefname(tok::ReplToken) + if tok.quoted || !isnothing(tok.key) || !Base.isidentifier(tok.text) + throw(ArgumentError("invalid preference name: $(repr(tok.text))")) + end + + check_preference_name(Symbol(tok.text)) +end + +function repl_set(tokens) + if isempty(tokens) + return Expr(:call, repl_show_preferences) + end + + if length(tokens) != 2 + throw(ArgumentError("set expects a preference name and a value, " * + "e.g. `set verbose 2`")) + end + + key, val = tokens + + Expr(:call, repl_set_preferences!, + Expr(:parameters, Expr(:kw, repl_prefname(key), repl_value(val)))) +end + +function repl_unset(tokens) + if isempty(tokens) + throw(ArgumentError("unset expects one or more preference names, " * + "e.g. `unset spin`")) + end + + # `missing` makes `set_preferences!` delete the preference + prefs = map(tok -> Expr(:kw, repl_prefname(tok), missing), tokens) + + Expr(:call, repl_set_preferences!, Expr(:parameters, prefs...)) +end + +function repl_token!(args, kws, tok::ReplToken) + text = tok.text + if !isnothing(tok.key) + # an unquoted value is Julia code + push!(kws, Expr(:kw, Symbol(tok.key), tok.quoted ? text : Meta.parse(text))) + return + end + if tok.quoted + push!(args, text) + return + end + + int = tryparse(Int, text) + if !isnothing(int) + push!(args, int) + elseif occursin(r"^_\w+$", text) + push!(args, QuoteNode(Symbol(text))) + elseif startswith(text, ':') && length(text) > 1 + push!(args, QuoteNode(Symbol(text[2:end]))) + else + push!(args, text) + end +end + +function repl_parse(line::AbstractString) + tokens = repl_tokenize(line) + if isempty(tokens) + return nothing + end + + cmd = popfirst!(tokens) + if cmd.quoted || !isnothing(cmd.key) + throw(ArgumentError("expected a command, got the pattern \"$(cmd.text)\"; " * + "type ? for help")) + end + + args = [] + kws = [] + name = get(REPL_ALIASES, cmd.text, cmd.text) + + if name == "help" + return Expr(:call, repl_help) + elseif name == "load" + if !isempty(tokens) + throw(ArgumentError("$(cmd.text) doesn't take arguments")) + end + return Expr(:call, repl_load_tests) + elseif name == "set" + return repl_set(tokens) + elseif name == "unset" + return repl_unset(tokens) + elseif name == "dry-run" + push!(kws, Expr(:kw, :dry, true)) + elseif name == "run-failed" + push!(args, fail) + elseif name != "run" # `run` passes all its arguments as-is to `retest` + throw(ArgumentError("unknown command \"$(cmd.text)\"; type ? for help")) + end + + for tok in tokens + repl_token!(args, kws, tok) + end + + if isempty(kws) + Expr(:call, repl_retest, args...) + else + Expr(:call, repl_retest, Expr(:parameters, kws...), args...) + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 6660fd0..11a769c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1932,3 +1932,246 @@ Test.@testset "record(::ReTestSet, ::LogTestFailure)" begin @test fails2 == 0 @test errors2 == 0 end + +Test.@testset "REPL mode parsing" begin + parse = ReTest.repl_parse + # `run` and friends are parsed into a call to `repl_retest`, which checks that + # tests have been loaded before delegating to `retest` + call(args...) = Expr(:call, ReTest.repl_retest, args...) + callkw(kws, args...) = Expr(:call, ReTest.repl_retest, + Expr(:parameters, (Expr(:kw, k, v) for (k, v) in kws)...), + args...) + + @test parse("") === nothing + @test parse(" ") === nothing + @test_throws ArgumentError parse("foo") # unknown command + @test_throws ArgumentError parse("\"run\"") # a command is never quoted + @test_throws ArgumentError parse("verbose=2") # nor a keyword + @test_throws ArgumentError parse("\"unterminated") + + @test parse("help") == Expr(:call, ReTest.repl_help) + @test parse("load") == Expr(:call, ReTest.repl_load_tests) + @test_throws ArgumentError parse("load MyPkg") + # every advertised command is handled (`unset` needs an argument), and every + # alias parses like the command it stands for + @test all(c -> parse(c) !== nothing, filter(!=("unset"), ReTest.REPL_COMMANDS)) + @test all(a -> parse(a) == parse(ReTest.REPL_ALIASES[a]), keys(ReTest.REPL_ALIASES)) + + @test parse("run") == call() + @test parse("run foo -slow a/b c|d") == call("foo", "-slow", "a/b", "c|d") + @test parse("run \"two words\"") == call("two words") + @test parse("run 3 -4") == call(3, -4) + @test parse("run :label _dv2") == call(QuoteNode(:label), QuoteNode(:_dv2)) + @test parse("run dry-run") == call("dry-run") # a command name is just a pattern here + @test parse("dry-run foo") == callkw((:dry => true,), "foo") + @test parse("run-failed foo") == call(fail, "foo") + + # the value of a keyword is Julia code, unless it's quoted + @test parse("run foo verbose=2 tag=[:a,:b]") == + callkw((:verbose => 2, :tag => :([:a, :b])), "foo") + @test parse("run desc=\"two words\"") == callkw((:desc => "two words",)) + # only a leading identifier introduces a keyword + @test parse("run -verbose=2 \"verbose=2\"") == call("-verbose=2", "verbose=2") + + setpref(kw, val) = Expr(:call, ReTest.repl_set_preferences!, + Expr(:parameters, Expr(:kw, kw, val))) + @test parse("set") == Expr(:call, ReTest.repl_show_preferences) + @test parse("set verbose 2") == setpref(:verbose, 2) + @test parse("set verbose inf") == setpref(:verbose, Inf) + @test parse("set spin false") == setpref(:spin, false) + @test parse("unset spin") == setpref(:spin, missing) + @test parse("unset spin verbose") == + Expr(:call, ReTest.repl_set_preferences!, + Expr(:parameters, Expr(:kw, :spin, missing), + Expr(:kw, :verbose, missing))) + @test_throws ArgumentError parse("set verbose") # a name and a value are needed + @test_throws ArgumentError parse("set verbose 2 3") + @test_throws ArgumentError parse("unset") + # `repl_prefname` rejects whatever isn't a bare, known preference name + @test_throws ArgumentError parse("set \"verbose\" 2") + @test_throws ArgumentError parse("set verbose=2 3") + @test_throws ArgumentError parse("unset dry") + + @test ReTest.repl_completions("") == (ReTest.REPL_COMMANDS, "") + @test ReTest.repl_completions("run") == (["run", "run-failed"], "run") + @test ReTest.repl_completions("foo") == (String[], "foo") + @test ReTest.repl_completions("dry-run r") == (String[], "") # only the first word + + # `load` refuses to run twice, which `preferences_project` records + hook = ReTest.repl_load_tests_hook + modules = copy(ReTest.TESTED_MODULES) + try + ReTest.repl_load_tests_hook = () -> :loaded + @test ReTest.repl_load_tests() === :loaded + # tests registered by other means don't prevent `load` + @test ReTest.repl_tests_loaded() + @test ReTest.repl_load_tests() === :loaded + # but a previous successful `load` does, as the active project is then a + # test environment + ReTest.preferences_project = "/nonexistent/Project.toml" + @test_throws ErrorException ReTest.repl_load_tests() + ReTest.preferences_project = nothing + ReTest.repl_load_tests_hook = nothing + @test_throws ErrorException ReTest.repl_load_tests() + + # `set` and `unset` require a project to have been loaded, as only then is + # there a sensible project to store the preferences in + @test_throws ErrorException ReTest.repl_set_preferences!(spin=false) + @test_throws ErrorException Core.eval(Main, parse("unset spin")) + @test_throws ErrorException Core.eval(Main, parse("set")) + + # `run` and friends require tests to have been loaded, by `load` or otherwise + empty!(ReTest.TESTED_MODULES) + @test !ReTest.repl_tests_loaded() + @test_throws ErrorException ReTest.repl_retest() + # the parsed expressions are evaluable as-is (here `repl_retest` still + # refuses to run, as no tests are loaded) + @test_throws ErrorException Core.eval(Main, parse("run b1 spin=false")) + append!(ReTest.TESTED_MODULES, modules) + empty!(M.RUN) + @test ReTest.repl_retest(M, "b1", spin=false) === nothing + @test M.RUN == ["b1"] + finally + ReTest.repl_load_tests_hook = hook + ReTest.preferences_project = nothing + append!(empty!(ReTest.TESTED_MODULES), modules) + end +end + +Test.@testset "preferences" begin + uuid = Base.PkgId(ReTest).uuid + + # sets the given preferences in a temporary environment added to LOAD_PATH + function with_preferences(f, prefs::String) + mktempdir() do dir + write(joinpath(dir, "Project.toml"), """ + [deps] + ReTest = "$uuid" + """) + write(joinpath(dir, "LocalPreferences.toml"), "[ReTest]\n" * prefs) + push!(LOAD_PATH, dir) + try + f() + finally + pop!(LOAD_PATH) + end + end + end + + # without a preference set, `def` returns the plain default (`id`'s is `nothing`, + # meaning that its value depends on other options) + alldefaults() = all(kw -> ReTest.def(kw) === ReTest.retest_defaults[kw], + ReTest.PREFERENCES) + @test alldefaults() + + with_preferences(""" + spin = false + verbose = 3 + stats = true + id = true + marks = false + """) do + @test ReTest.def(:spin) == false + @test ReTest.def(:verbose) == 3 + @test ReTest.def(:stats) == true + @test ReTest.def(:id) == true + @test ReTest.def(:marks) == false + # keywords which are not preferences keep their default + @test ReTest.def(:dry) == ReTest.retest_defaults[:dry] + end + # the preferences go away with their environment + @test alldefaults() + + with_preferences("verbose = inf\nid = false\n") do + @test ReTest.def(:verbose) == Inf # TOML's `inf` + @test ReTest.def(:id) == false + end + + # a value of the wrong type is rejected when read + with_preferences(""" + stats = 1 + id = 1 + marks = "no" + spin = "yes" + verbose = "2" + """) do + for kw in ReTest.PREFERENCES + @test_throws ArgumentError ReTest.def(kw) + end + end + + @test_throws ArgumentError ReTest.check_preference(:dry, true) + @test ReTest.check_preference(:verbose, true) == true + @test_throws ArgumentError ReTest.set_preferences!(dry=true) + @test_throws ArgumentError ReTest.set_preferences!(spin=1) + # `Preferences` gives `nothing` a meaning of its own, which we don't expose + @test_throws ArgumentError ReTest.set_preferences!(spin=nothing) + @test_throws ArgumentError ReTest.set_preferences!(id=nothing) # `id`'s default + + # `preferences_project` makes `set_preferences!` write elsewhere than the active + # project, which in `retest>` mode is a temporary test environment (see + # ext/ReTestREPLExt.jl); the environment is also added to LOAD_PATH, as the REPL + # extension does, so that the preferences are read back + function with_preferences_project(f, project::String) + mktempdir() do dir + projectfile = joinpath(dir, "Project.toml") + write(projectfile, project) + push!(LOAD_PATH, dir) + ReTest.preferences_project = projectfile + try + f(dir, projectfile) + finally + ReTest.preferences_project = nothing + pop!(LOAD_PATH) + end + end + end + + # the preferences file of the active project must stay untouched + active = joinpath(dirname(Base.active_project()), "LocalPreferences.toml") + active_before = isfile(active) ? read(active, String) : nothing + + deps = """ + [deps] + ReTest = "$uuid" + """ + + with_preferences_project(deps) do dir, projectfile + ReTest.set_preferences!(spin=false, verbose=2) + @test ReTest.def(:spin) == false + @test ReTest.def(:verbose) == 2 + @test isfile(joinpath(dir, "LocalPreferences.toml")) + # ReTest is already a dependency, so the project file needs no change + @test read(projectfile, String) == deps + + ReTest.set_preferences!(spin=missing) + @test ReTest.def(:spin) == ReTest.retest_defaults[:spin] + + # once a project is loaded, `set`/`unset` are allowed and write to it + Core.eval(Main, ReTest.repl_parse("set verbose 3")) + @test ReTest.def(:verbose) == 3 + Core.eval(Main, ReTest.repl_parse("unset verbose")) + @test ReTest.def(:verbose) == ReTest.retest_defaults[:verbose] + @test Core.eval(Main, ReTest.repl_parse("set")) === nothing + @test_throws ArgumentError Core.eval(Main, ReTest.repl_parse("set spin nothing")) + end + @test ReTest.def(:verbose) == ReTest.retest_defaults[:verbose] + + # an existing JuliaLocalPreferences.toml is written instead of LocalPreferences.toml + with_preferences_project(deps) do dir, _ + touch(joinpath(dir, "JuliaLocalPreferences.toml")) + ReTest.set_preferences!(verbose=4) + @test !isfile(joinpath(dir, "LocalPreferences.toml")) + @test ReTest.def(:verbose) == 4 + end + + # a project which doesn't depend on ReTest: it gets added to `[extras]`, without + # which `Base` refuses to read preferences of a non-dependency + with_preferences_project("") do dir, projectfile + ReTest.set_preferences!(verbose=5) + @test occursin("ReTest", read(projectfile, String)) + @test ReTest.def(:verbose) == 5 + end + + @test (isfile(active) ? read(active, String) : nothing) == active_before +end