diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index dd5dbfc7..81a9cf17 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -655,28 +655,12 @@ count generated "generated source (a subset of non-test, shown for reference)" # scc counts one partition per run, so the split has to be assembled here rather than read off a single table. { printf 'language\\tnon_test_code\\ttest_code\\ttotal_code\\n' - # $doc and $l are jaq bindings, not shell variables, so the program stays single-quoted. - # shellcheck disable=SC2016 printf '{"nontest":%s,"test":%s}' "$(coreutils cat "$out/non-test.json")" "$(coreutils cat "$out/test.json")" | - jaq -r '. as $doc - | (reduce $doc.nontest[] as $l ({}; .[$l.Name] = {nontest: $l.Code, test: 0})) - | (reduce $doc.test[] as $l (.; .[$l.Name] = {nontest: (.[$l.Name].nontest // 0), test: $l.Code})) - | to_entries - | map({lang: .key, nontest: .value.nontest, test: .value.test, total: (.value.nontest + .value.test)}) - | sort_by(-.total) - | .[] - | [.lang, .nontest, .test, .total] - | @tsv' + jaq -r -f .mise/sloc-languages.jq } >"$out/by-language.tsv" -# The test share is derived at print time rather than stored, so the tsv holds only measured counts. echo "== per-language test vs non-test ==" -goawk -F '\\t' 'NR == 1 { printf "%-30s %10s %10s %10s %7s\\n", "Language", $2, $3, $4, "test%"; next } - { nt += $2; t += $3; total += $4 - printf "%-30s %10d %10d %10d %7s\\n", substr($1, 1, 30), $2, $3, $4, - ($4 > 0 ? sprintf("%.1f", 100 * $3 / $4) : "-") } - END { printf "%-30s %10d %10d %10d %7s\\n", "TOTAL", nt, t, total, - (total > 0 ? sprintf("%.1f", 100 * t / total) : "-") }' "$out/by-language.tsv" +goawk -f .mise/sloc-table.awk "$out/by-language.tsv" # Every counted line must fall in exactly one partition, so a classification miss fails instead of passing. code() { jaq '[.[].Code] | add' <"$out/$1.json"; } diff --git a/.mise/config.dart.toml b/.mise/config.dart.toml index e0556e30..26e623a9 100644 --- a/.mise/config.dart.toml +++ b/.mise/config.dart.toml @@ -122,41 +122,21 @@ shell = "{{ vars.task_shell }}" depends = ["gen:ws-spec"] description = "Emit the Dart REST client via swagger_parser + build_runner (consumes generated/specs/rest.yaml)" dir = "generated/dart-rest" -# Regenerate the Dart REST client, applying two swagger_parser output fixups. +# Regenerate the Dart REST client, fixing up two of swagger_parser's output files before build_runner reads them. # `rm -rf lib` first so a renamed output (e.g. client_postfix change) can't leave orphaned files behind. The # goawk rewrites pipe through `tr -d '\r'` because goawk emits CRLF on Windows and dart format preserves # whatever line endings it finds. # -# Fixup 1 (storage.dart): swagger_parser renders the application/octet-stream storage endpoints unusably for the -# browser -- a `dart:io` `File` PUT body and a `void` GET response. goawk rewrites it before build_runner so putFile -# takes a Uint8List (dio sends Uint8List as raw bytes; a List would be JSON-encoded) and getFile returns the -# bytes. The END guard fails loudly if a swagger_parser bump moves those sites. -# -# Fixup 2 (rest_client.dart): with client_postfix = "" (which drops the `_client` suffix) swagger_parser emits the -# root client's sub-client imports as `clients/_.dart` -- a stray trailing underscore that export.dart does -# not have. goawk strips it. Idempotent: a no-op if upstream fixes the inconsistency. +# rest_client.dart: with client_postfix = "" (which drops the `_client` suffix) swagger_parser emits the root +# client's sub-client imports as `clients/_.dart` -- a stray trailing underscore that export.dart does not +# have. goawk strips it. Idempotent: a no-op if upstream fixes the inconsistency. run = """ dart pub get coreutils rm -rf lib dart run swagger_parser s=../../target/scratch coreutils mkdir -p "$s" -goawk 'BEGIN { q = sprintf("%c", 39) } -/^import / && /dart:io/ { print "import " q "dart:typed_data" q ";"; io++; next } -/^ Future getFile\\(\\{$/ { - print " @DioResponseType(ResponseType.bytes)" - print " Future> getFile({" - get++ - next -} -/@Body\\(\\) required File body,/ { sub(/File body/, "Uint8List body"); print; body++; next } -{ print } -END { - if (io != 1 || get != 1 || body != 1) { - printf("gen:dart-rest storage fixup miss: io=%d get=%d body=%d\\n", io, get, body) > "/dev/stderr" - exit 1 - } -}' lib/clients/storage.dart | coreutils tr -d '\\r' >"$s/storage.dart" +goawk -f ../../.mise/dart-rest-storage.awk lib/clients/storage.dart | coreutils tr -d '\\r' >"$s/storage.dart" coreutils mv "$s/storage.dart" lib/clients/storage.dart goawk '/^import / && /clients/ { gsub(/_\\.dart/, ".dart") } { print }' lib/rest_client.dart | coreutils tr -d '\\r' >"$s/rest_client.dart" diff --git a/.mise/config.toml b/.mise/config.toml index 926da141..79becfbd 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -2346,18 +2346,9 @@ project="et-$scenario-verify" # The values are read from a probe of each limited process rather than from the process itself. # A runner exits once its module has run, so by the time the exchange is verified there may be no container left to # read; the probe is the same service definition -- the same limits and, under k3s, the same plugin annotations -- -# with its command replaced by a sleep, and so lives as long as the check needs it. Under k3s it is a bare Pod cut from -# the Deployment's template, relabelled so the Deployment's ReplicaSet does not adopt it as a surplus replica. -probe_pod='select(.kind == "Deployment" and .metadata.name == strenv(app)) | .spec.template - | .apiVersion = "v1" | .kind = "Pod" | .metadata.name = strenv(app) + "-limits-probe" - | .metadata.labels.app = strenv(app) + "-limits-probe" | .spec.restartPolicy = "Never" - | .spec.containers[0].command = ["sleep", "600"] | del(.spec.containers[0].args) - | del(.spec.containers[0].readinessProbe)' -limited='select(.kind == "Deployment") - | select((.spec.template.metadata.annotations // {}) | keys | any_c(test("memory-qos|ulimits"))) - | .metadata.name' +# with its command replaced by a sleep, and so lives as long as the check needs it. # `--no-doc` throughout, because yq otherwise separates each document's lines with a `---` of its own. -apps=$(yq --no-doc "$limited" "$manifest") +apps=$(yq --no-doc --from-file .mise/verify-limits-apps.yq "$manifest") cleanup() { for app in $apps; do if [ "$format" = "k3s" ]; then @@ -2370,7 +2361,7 @@ cleanup() { trap cleanup EXIT for app in $apps; do if [ "$format" = "k3s" ]; then - app="$app" yq --no-doc "$probe_pod" "$manifest" | kubectl apply -n "$ns" -f - >/dev/null + app="$app" yq --no-doc --from-file .mise/verify-limits-probe.yq "$manifest" | kubectl apply -n "$ns" -f - >/dev/null kubectl wait -n "$ns" --for=condition=Ready "pod/$app-limits-probe" --timeout=120s >/dev/null else set -- run --detach --no-deps --name "$project-$app-limits-probe" --entrypoint sleep "$app" 600 @@ -2390,18 +2381,6 @@ in_container() { fi } -# One ` ` line per plugin annotation, from every Deployment of the manifest. -# `$app` is yq's own variable, bound by `as`, so it must reach yq unexpanded. -# shellcheck disable=SC2016 -annotations='select(.kind == "Deployment") | .metadata.name as $app - | (.spec.template.metadata.annotations // {}) | to_entries[] | $app + " " + .key + " " + .value' -# The same for each rlimit, its value the soft and hard limits the ulimits annotation lists. -# shellcheck disable=SC2016 -rlimits='select(.kind == "Deployment") | .metadata.name as $app - | (.spec.template.metadata.annotations // {}) | to_entries[] - | select(.key | contains("ulimits.noderesource.dev/")) | .value | from_yaml | .[] - | $app + " " + (.type | downcase | sub("rlimit_"; "")) + " " + (.soft | tostring) + " " + (.hard | tostring)' - checked=0 while read -r app key expected; do case "$key" in @@ -2415,7 +2394,7 @@ while read -r app key expected; do exit 1 fi checked=$((checked + 1)) -done < <(yq --no-doc "$annotations" "$manifest") +done < <(yq --no-doc --from-file .mise/verify-limits-annotations.yq "$manifest") while read -r app resource soft hard; do actual=$(in_container "$app" prlimit --pid 1 "--$resource" --output SOFT,HARD --noheadings | coreutils tr -s ' ') @@ -2425,7 +2404,7 @@ while read -r app resource soft hard; do exit 1 fi checked=$((checked + 1)) -done < <(yq --no-doc "$rlimits" "$manifest") +done < <(yq --no-doc --from-file .mise/verify-limits-rlimits.yq "$manifest") if [ "$checked" -eq 0 ]; then echo "verify-limits: $manifest names no limit for the runtime plugins to write, so there was nothing to check" >&2 @@ -2648,9 +2627,8 @@ usage = 'arg "" help="Dependency package name as it appears in Cargo.lo [tasks."gen:dockerignore"] description = "Regenerate .dockerignore from .gitignore + Docker-only excludes" # Docker reads only the root .dockerignore (never .gitignore or nested ignore files). -# A bare pattern there is root-anchored -- whereas a slash-less .gitignore pattern matches at any depth. # So mirror .gitignore, add the Docker-only excludes (`.git/`, plus the build recipe and every top-level Markdown -# file, so editing them doesn't bust the `COPY . .` cache), and `**/`-prefix each non-anchored pattern. +# file, so editing them doesn't bust the `COPY . .` cache), and rewrite each pattern for the way Docker anchors it. # `.dockerignore` itself is deliberately NOT excluded any more. Excluding it saved nothing (the file is tiny) and # meant it was absent from the build context, so `et-cli` -- which derives each scenario's per-Dockerfile ignore # file from it -- could not run inside an image built from this Dockerfile: @@ -2675,17 +2653,7 @@ trap 'coreutils rm -f "$staged"' EXIT { coreutils cat .gitignore printf '%s\n' '.git/' 'Dockerfile*' '/*.md' - } | goawk ' - /^[[:space:]]*#/ { print; next } - /^[[:space:]]*$/ { print; next } - { - neg = ""; line = $0 - if (substr(line, 1, 1) == "!") { neg = "!"; line = substr(line, 2) } - if (substr(line, 1, 1) == "/") { print neg substr(line, 2); next } - core = line; sub(/\/$/, "", core) - if (index(core, "/") > 0) print neg line; else print neg "**/" line - } - ' + } | goawk -f .mise/gitignore-to-dockerignore.awk } | coreutils tr -d '\r' >"$staged" coreutils mv -f "$staged" .dockerignore ''' @@ -2884,21 +2852,15 @@ run = "cargo run -q -p et-int-gen --bin et-int-gen -- generate bindings" depends = ["gen:ws-spec"] description = "Render generated/specs/rest.yaml as the Markdown reference generated/specs/rest.md (vimanam)" env = { RUMDL_CACHE_DIR = "{{ config_root }}/target/rumdl-cache" } -# Two constructs in vimanam's output that the Markdown rules reject are rewritten before formatting. -# It writes `
` between the paragraphs of a table cell, and an emphasis-only line where an operation has no request -# or response schema; they become a space and a plain sentence. rumdl's fixes and the Markdown formatter then settle -# the layout, so the committed file is already what both checks want. The work is staged under target/ and lands in -# one rename, leaving the committed file whole if any step fails. +# vimanam's output is tidied before formatting, then rumdl's fixes and the Markdown formatter settle the layout. +# The committed file is therefore already what both checks want. The work is staged under target/ and lands in one +# rename, leaving the committed file whole if any step fails. run = """ stage=target/int-gen/rest-md out=generated/specs/rest.md coreutils mkdir -p "$stage" vimanam generated/specs/rest.yaml --detail full --include-schemas --include-auth --no-report -o "$stage/raw.md" -goawk ' -{ gsub("
", " ") } -/^[*]No (request|response) schema available[*]$/ { gsub("[*]", ""); $0 = $0 "." } -{ print } -' "$stage/raw.md" >"$stage/tidy.md" +goawk -f .mise/rest-md-tidy.awk "$stage/raw.md" >"$stage/tidy.md" rumdl check --fix --fail-on never --silent --config config/rumdl.toml "$stage/tidy.md" dprint fmt --stdin "$out" <"$stage/tidy.md" >"$stage/rest.md" coreutils mv -f "$stage/rest.md" "$out" diff --git a/.mise/dart-rest-storage.awk b/.mise/dart-rest-storage.awk new file mode 100644 index 00000000..139c6f37 --- /dev/null +++ b/.mise/dart-rest-storage.awk @@ -0,0 +1,20 @@ +# Rewrites swagger_parser's Dart storage client so the browser can use it. +# swagger_parser renders the application/octet-stream storage endpoints unusably for the browser -- a `dart:io` +# `File` PUT body and a `void` GET response. This makes putFile take a Uint8List (dio sends Uint8List as raw bytes; a +# List would be JSON-encoded) and getFile return the bytes. The END guard fails loudly if a swagger_parser bump +# moves those sites. +/^import / && /dart:io/ { print "import 'dart:typed_data';"; io++; next } +/^ Future getFile\(\{$/ { + print " @DioResponseType(ResponseType.bytes)" + print " Future> getFile({" + get++ + next +} +/@Body\(\) required File body,/ { sub(/File body/, "Uint8List body"); print; body++; next } +{ print } +END { + if (io != 1 || get != 1 || body != 1) { + printf("gen:dart-rest storage fixup miss: io=%d get=%d body=%d\n", io, get, body) > "/dev/stderr" + exit 1 + } +} diff --git a/.mise/gitignore-to-dockerignore.awk b/.mise/gitignore-to-dockerignore.awk new file mode 100644 index 00000000..36d80eae --- /dev/null +++ b/.mise/gitignore-to-dockerignore.awk @@ -0,0 +1,17 @@ +# Rewrites .gitignore patterns as .dockerignore ones, which Docker anchors differently. +# A bare .dockerignore pattern is root-anchored, whereas a slash-less .gitignore pattern matches at any depth, so a +# pattern whose only slash is a trailing one gains a `**/` prefix. A leading slash anchors a pattern in both, and is +# dropped; a slash anywhere else anchors it in both as written. A `!` negation stays in front of whichever form +# results, and comments and blank lines pass through unchanged. +/^[[:space:]]*#/ { print; next } +/^[[:space:]]*$/ { print; next } +{ + neg = "" + line = $0 + if (substr(line, 1, 1) == "!") { neg = "!"; line = substr(line, 2) } + if (substr(line, 1, 1) == "/") { print neg substr(line, 2); next } + core = line + sub(/\/$/, "", core) + if (index(core, "/") > 0) print neg line + else print neg "**/" line +} diff --git a/.mise/rest-md-tidy.awk b/.mise/rest-md-tidy.awk new file mode 100644 index 00000000..73cda93c --- /dev/null +++ b/.mise/rest-md-tidy.awk @@ -0,0 +1,6 @@ +# Rewrites the two constructs in vimanam's Markdown that the Markdown rules reject. +# It writes `
` between the paragraphs of a table cell, and an emphasis-only line where an operation has no +# request or response schema; they become a space and a plain sentence. +{ gsub("
", " ") } +/^[*]No (request|response) schema available[*]$/ { gsub("[*]", ""); $0 = $0 "." } +{ print } diff --git a/.mise/sloc-languages.jq b/.mise/sloc-languages.jq new file mode 100644 index 00000000..1e52fdae --- /dev/null +++ b/.mise/sloc-languages.jq @@ -0,0 +1,12 @@ +# Joins scc's per-language counts for the non-test and test partitions into one tab-separated row per language. +# The input is `{"nontest": , "test": }`, each an scc json report. A row is the language, its non-test +# and test code lines and their total, and the rows run from the largest total down. +. as $doc +| (reduce $doc.nontest[] as $l ({}; .[$l.Name] = { nontest: $l.Code, test: 0 })) +| (reduce $doc.test[] as $l (.; .[$l.Name] = { nontest: (.[$l.Name].nontest // 0), test: $l.Code })) +| to_entries +| map({ lang: .key, nontest: .value.nontest, test: .value.test, total: (.value.nontest + .value.test) }) +| sort_by(-.total) +| .[] +| [.lang, .nontest, .test, .total] +| @tsv diff --git a/.mise/sloc-table.awk b/.mise/sloc-table.awk new file mode 100644 index 00000000..e42657de --- /dev/null +++ b/.mise/sloc-table.awk @@ -0,0 +1,12 @@ +# Prints the per-language tsv as an aligned table, adding each language's test share and a closing total. +# The share is derived here rather than stored, so the tsv holds only measured counts. +function share(part, whole) { return (whole > 0 ? sprintf("%.1f", 100 * part / whole) : "-") } +BEGIN { FS = "\t"; row = "%-30s %10d %10d %10d %7s\n" } +NR == 1 { printf "%-30s %10s %10s %10s %7s\n", "Language", $2, $3, $4, "test%"; next } +{ + nt += $2 + t += $3 + total += $4 + printf row, substr($1, 1, 30), $2, $3, $4, share($3, $4) +} +END { printf row, "TOTAL", nt, t, total, share(t, total) } diff --git a/.mise/verify-limits-annotations.yq b/.mise/verify-limits-annotations.yq new file mode 100644 index 00000000..938427a0 --- /dev/null +++ b/.mise/verify-limits-annotations.yq @@ -0,0 +1,6 @@ +# Prints one ` ` line per plugin annotation, from every Deployment of the manifest. +select(.kind == "Deployment") +| .metadata.name as $app +| (.spec.template.metadata.annotations // {}) +| to_entries[] +| $app + " " + .key + " " + .value diff --git a/.mise/verify-limits-apps.yq b/.mise/verify-limits-apps.yq new file mode 100644 index 00000000..313c8813 --- /dev/null +++ b/.mise/verify-limits-apps.yq @@ -0,0 +1,4 @@ +# Names every Deployment whose pod template carries a memory-qos or ulimits plugin annotation. +select(.kind == "Deployment") +| select((.spec.template.metadata.annotations // {}) | keys | any_c(test("memory-qos|ulimits"))) +| .metadata.name diff --git a/.mise/verify-limits-probe.yq b/.mise/verify-limits-probe.yq new file mode 100644 index 00000000..bb751a3c --- /dev/null +++ b/.mise/verify-limits-probe.yq @@ -0,0 +1,12 @@ +# Cuts a bare probe Pod, sleeping in place of its command, from the template of the Deployment `app` names. +# It is relabelled so the Deployment's ReplicaSet does not adopt it as a surplus replica. +select(.kind == "Deployment" and .metadata.name == strenv(app)) +| .spec.template +| .apiVersion = "v1" +| .kind = "Pod" +| .metadata.name = strenv(app) + "-limits-probe" +| .metadata.labels.app = strenv(app) + "-limits-probe" +| .spec.restartPolicy = "Never" +| .spec.containers[0].command = ["sleep", "600"] +| del(.spec.containers[0].args) +| del(.spec.containers[0].readinessProbe) diff --git a/.mise/verify-limits-rlimits.yq b/.mise/verify-limits-rlimits.yq new file mode 100644 index 00000000..0fb24c84 --- /dev/null +++ b/.mise/verify-limits-rlimits.yq @@ -0,0 +1,10 @@ +# Prints one ` ` line per rlimit a ulimits annotation lists, from every Deployment. +select(.kind == "Deployment") +| .metadata.name as $app +| (.spec.template.metadata.annotations // {}) +| to_entries[] +| select(.key | contains("ulimits.noderesource.dev/")) +| .value +| from_yaml +| .[] +| $app + " " + (.type | downcase | sub("rlimit_"; "")) + " " + (.soft | tostring) + " " + (.hard | tostring) diff --git a/CHECKS.md b/CHECKS.md index 62f6b842..ffa3befc 100644 --- a/CHECKS.md +++ b/CHECKS.md @@ -1,11 +1,11 @@ # Checks -- **Generated:** 2026-11-08 +- **Generated:** 2026-11-09 - **Checks:** 83 - **Tools:** 48 - **Shell-script checks:** 10 - **Checks with a repair pass:** 22 -- **Rules:** 374 custom local rules, or non-default strict linter settings +- **Rules:** 376 custom local rules, or non-default strict linter settings - **Meta rules:** 21 of those rules are rules about rules Every check is listed across every `MISE_ENV`, filed under the tool it runs. Each entry names the env whose config @@ -273,6 +273,11 @@ Rules, from `config/ast-grep/rules`: - `no-test-in-rust-docs` -- Doc comment in non-test code mentions tests. Document what the item does for every caller; why a test needs it belongs in the test that relies on it. A name containing it (`test`, `ws-test-server`, `ET_TEST_X`) is allowed. +- `no-tuple-identity-map` -- A `.map` closure that takes a tuple apart and puts the same names back in the same order + reads as an identity map, whatever borrow it changes on the way. Over `&(K, V)` items it turns each into `(&K, &V)`: + say so with field access, `.map(|pair| (&pair.0, &pair.1))`, so the conversion is the visible part. Over owned tuples + it is the identity and the call can go. This mirrors `DeepSource RS-W1018`. clippy's `map_identity` compares the item + types, so the borrowing form, which changes the type, gets past it. - `no-uds-listener` -- HTTP servers here listen on TCP only. A Unix-domain-socket listener gives requests no peer address, and the handlers take the client address from the socket peer rather than from client-supplied `Forwarded` / `X-Forwarded-For` headers -- so `ws_handler` refuses every request arriving on one. Bind a TCP address instead. @@ -1045,6 +1050,13 @@ Rules, from `config/semgrep`: - `no-maintainer-task-name-mentions` -- Do not name an install, prefetch or asset-publishing task outside the jobs, images and asset tables that invoke it. These run on a maintainer's machine or a runner, never from the code, so a source file naming one is describing an operation it has no part in. +- `no-multiline-inline-program` -- An awk, sed, jq or yq program in a mise task must fit on one line. One that needs + more is a program in its own right, and as a string inside a TOML string it is invisible to every checker: nothing + formats or lints it, a diff shows it as config, and its text has to survive TOML's escaping and then the shell's + quoting before its interpreter sees the first byte. Move it to a file beside the config in `.mise/`, named for the + task and with the extension of whatever reads it (`.awk`, `.jq`, `.yq`), and pass the path instead -- `goawk -f`, `jaq + -f`, `yq --from-file`. A program held in a shell variable and expanded into the call is the same program, and moves + the same way. - `no-nextest-mentions` -- Do not name the test runner outside its config and the jobs that invoke it. A test asserts what the code must do, and how it happens to be scheduled is the runner's business rather than the test's -- unless the schedule is load-bearing, which is a constraint worth stating and worth allowlisting for. diff --git a/config/ast-grep/rules/no-tuple-identity-map.yaml b/config/ast-grep/rules/no-tuple-identity-map.yaml new file mode 100644 index 00000000..cbb4b6bf --- /dev/null +++ b/config/ast-grep/rules/no-tuple-identity-map.yaml @@ -0,0 +1,17 @@ +id: no-tuple-identity-map +language: Rust +severity: error +message: | + A `.map` closure that takes a tuple apart and puts the same names back in the same order reads as an identity map, + whatever borrow it changes on the way. Over `&(K, V)` items it turns each into `(&K, &V)`: say so with field access, + `.map(|pair| (&pair.0, &pair.1))`, so the conversion is the visible part. Over owned tuples it is the identity and + the call can go. This mirrors `DeepSource RS-W1018`. clippy's `map_identity` compares the item types, so the + borrowing form, which changes the type, gets past it. +# Check-only, because the rewrite depends on the item type, which a syntax tree cannot see. +# Field access is right for borrowed tuples, while for owned ones it would return borrows of the closure's argument +# and fail to compile. A repeated metavariable has to bind the same name each time, so only the same names in the +# same order match. +rule: + any: + - pattern: $ITER.map(|($A, $B)| ($A, $B)) + - pattern: $ITER.map(|($A, $B, $C)| ($A, $B, $C)) diff --git a/config/ast-grep/tests/no-tuple-identity-map.yaml b/config/ast-grep/tests/no-tuple-identity-map.yaml new file mode 100644 index 00000000..f949f264 --- /dev/null +++ b/config/ast-grep/tests/no-tuple-identity-map.yaml @@ -0,0 +1,17 @@ +id: no-tuple-identity-map +valid: + - | + let map = serializer.collect_map(pairs.iter().map(|pair| (&pair.0, &pair.1))); + - | + let swapped = pairs.iter().map(|(key, value)| (value, key)); + - | + let keys = pairs.iter().map(|(key, _value)| key); + - | + let owned = pairs.iter().map(|(key, value)| (key.clone(), value)); + - | + let rotated = rows.iter().map(|(first, second, third)| (second, third, first)); +invalid: + - | + let map = serializer.collect_map(pairs.iter().map(|(key, value)| (key, value))); + - | + let same = rows.into_iter().map(|(first, second, third)| (first, second, third)); diff --git a/config/semgrep-tests/no-multiline-inline-program.toml.yaml b/config/semgrep-tests/no-multiline-inline-program.toml.yaml new file mode 100644 index 00000000..cd13f451 --- /dev/null +++ b/config/semgrep-tests/no-multiline-inline-program.toml.yaml @@ -0,0 +1,60 @@ +code: | + [tasks.example] + run = """ + # ruleid: no-multiline-inline-program + goawk ' + { print } + ' "$file" + + # ruleid: no-multiline-inline-program + printf '%s\\n' "$json" | jaq -r '. as $doc + | $doc.name' + + # ruleid: no-multiline-inline-program + goawk -F '\\t' -v label="a b" 'NR == 1 { next } + { print $2 }' "$file" + + # ruleid: no-multiline-inline-program + gh release view "$tag" --json assets --jq '.assets[] + | .name' + + # ruleid: no-multiline-inline-program + out=$(mise exec -- yq -p toml "select(.a) + | .b" "$file") + + # ruleid: no-multiline-inline-program + limited='select(.kind == "Deployment") + | .metadata.name' + apps=$(yq --no-doc "$limited" "$manifest") + + # ok: no-multiline-inline-program + host="$(rustc -vV | goawk '/^host:/ { print $2 }')" + + # ok: no-multiline-inline-program + version="$(goawk -F'"' '/^min_version/{print $2; exit}' "$file")" + + # ok: no-multiline-inline-program + jaq -r -f .mise/sloc-languages.jq "$file" + + # ok: no-multiline-inline-program + coreutils cat .mise/cov-branch-union.jq 'not a program + but a path to one' + + # ok: no-multiline-inline-program + jqfmt -f "$f" -op pipe 'not the tool + whose name it starts with' + + # ok: no-multiline-inline-program + read_model='s3() { + curl -fsS "$1" + }' + kubectl exec deploy/ws-server -- sh -c "$read_model" + + # ok: no-multiline-inline-program + trap 'status=$?; + exit "$status"' EXIT + + # ok: no-multiline-inline-program + echo "done" # goawk reads each 'line + """ + shell = "{{ vars.task_shell }}" diff --git a/config/semgrep/comment-summary-line.yaml b/config/semgrep/comment-summary-line.yaml index 8e3af0b3..16bdd4ba 100644 --- a/config/semgrep/comment-summary-line.yaml +++ b/config/semgrep/comment-summary-line.yaml @@ -13,6 +13,7 @@ rules: - "/**/*.sh" - "/**/*.toml" - "/**/*.yaml" + - "/**/*.yq" - "/**/.dockerignore" - "/**/.gitignore" - "/**/.semgrepignore" diff --git a/config/semgrep/no-multiline-inline-program.yaml b/config/semgrep/no-multiline-inline-program.yaml new file mode 100644 index 00000000..6357eff6 --- /dev/null +++ b/config/semgrep/no-multiline-inline-program.yaml @@ -0,0 +1,36 @@ +rules: + - id: no-multiline-inline-program + languages: [generic] + paths: + include: + - "/.mise/config*.toml" + # Flag an awk, sed, jq or yq program whose quoted text runs past the line it opens on, in either of two shapes. + # Inline: the tool in command position -- not part of a path or a longer word, so `cov-branch-assert.jq` and + # `jqfmt` never match -- then any flags and operands that close on the same line, then a quote that does not. + # The prefix stops at `#`, so a comment that names a tool and then quotes something is left alone. In a + # variable: a multi-line single-quoted assignment whose variable a later line of the same body hands to one of + # the tools; a multi-line string the tools never read, such as shell for another shell to run, is not a + # program of theirs. A one-line program stays inline, since the line limit already keeps it short. No autofix: + # the fix is a new file, which a rewrite of the matched text cannot create. + # + # The operand quantifiers are possessive because a word can otherwise be split into its runs in exponentially + # many ways. On a line that does not match, the engine tries them all and gives up on the file once it hits its + # backtracking limit, which reports nothing for every line of it, matches included. + pattern-either: + - pattern-regex: >- + (?mx) ^ [^#\n]*? (?- + (?msx) ^ [ \t]* (?: (?: export | local | readonly ) [ \t]+ )? ([A-Za-z_]\w*) = '[^'\n]*\n [^']* ' + (?: (?! ^ (?: """ | ''' ) $ ) . )*? + ^ [^#\n]*? (?- + An awk, sed, jq or yq program in a mise task must fit on one line. One that needs more is a program in its + own right, and as a string inside a TOML string it is invisible to every checker: nothing formats or lints + it, a diff shows it as config, and its text has to survive TOML's escaping and then the shell's quoting + before its interpreter sees the first byte. Move it to a file beside the config in `.mise/`, named for the + task and with the extension of whatever reads it (`.awk`, `.jq`, `.yq`), and pass the path instead -- + `goawk -f`, `jaq -f`, `yq --from-file`. A program held in a shell variable and expanded into the call is + the same program, and moves the same way. + severity: ERROR