From 08cef54b53c10287a8cc84caad34c78ac152cb8a Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Sun, 8 Nov 2026 20:24:21 -0300 Subject: [PATCH 1/3] image: packages sources, every recipe's git-checkout on its remote A fork's branch merged and deleted (brand's ui-hill-climb) failed melange's clone 25 minutes into a release, its error buried under the kernel building beside it. git ls-remote finds a missing branch, a missing tag or a tag moved off its expected commit in seconds. Co-Authored-By: Claude Opus 5.5 --- cmd/image/main.go | 1 + cmd/image/packages.go | 7 +- cmd/image/sources.go | 210 ++++++++++++++++++++++++++++++++++++++ cmd/image/sources_test.go | 33 ++++++ 4 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 cmd/image/sources.go create mode 100644 cmd/image/sources_test.go diff --git a/cmd/image/main.go b/cmd/image/main.go index e903bab..0d69af2 100644 --- a/cmd/image/main.go +++ b/cmd/image/main.go @@ -22,6 +22,7 @@ // image release-notes -version v -packages f a release's notes (cmd/image/notes.go) // image channel-index -version v -add|-withdraw a channel's index with a release added or withdrawn (cmd/image/channel.go) // image packages missing|prune the recipes packages/ lacks, a wave a line; builds no recipe makes out (cmd/image/packages.go) +// image packages sources every recipe's git-checkout branch and tag on its remote (cmd/image/sources.go) // image lock write [-all] | index the locks from apko's resolution, or Wolfi's lines bumped to its index (cmd/image/lock.go) // image cache pull|push the release builder's packages/, kernel, firmware and QEMU in S3 (cmd/image/cache.go) // image usb-disk the stick's disk from the signed release (cmd/image/usb.go) diff --git a/cmd/image/packages.go b/cmd/image/packages.go index b2d4589..0f492a4 100644 --- a/cmd/image/packages.go +++ b/cmd/image/packages.go @@ -219,8 +219,11 @@ func unmade(dir string, recipes []recipe) ([]string, error) { // a line per wave that builds side by side, or image packages prune, which removes every apk no // recipe makes now and prints the files it removed, for the index to be made again. func packagesCmd(args []string) error { - if len(args) != 1 || (args[0] != "missing" && args[0] != "prune") { - return errors.New("packages missing|prune") + if len(args) != 1 || (args[0] != "missing" && args[0] != "prune" && args[0] != "sources") { + return errors.New("packages missing|prune|sources") + } + if args[0] == "sources" { + return packagesSources("recipes") } recipes, err := readRecipes("recipes") if err != nil { diff --git a/cmd/image/sources.go b/cmd/image/sources.go new file mode 100644 index 0000000..f578f89 --- /dev/null +++ b/cmd/image/sources.go @@ -0,0 +1,210 @@ +package main + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" +) + +// The recipes' sources before a builder is launched: every git-checkout's branch is on its +// remote, and every tag is, at the commit the recipe expects. A branch merged and deleted, or a +// tag moved, fails melange's clone minutes into a release, buried among the builds beside it; +// git ls-remote says so in seconds, before anything is built. + +// checkoutRef is one git-checkout of a recipe, its ${{...}} substituted. +type checkoutRef struct { + Recipe, Repository, Branch, Tag, Commit string +} + +func (c checkoutRef) String() string { + ref := "branch " + c.Branch + if c.Tag != "" { + ref = "tag " + c.Tag + } + return fmt.Sprintf("%s: %s %s", c.Recipe, c.Repository, ref) +} + +var substitution = regexp.MustCompile(`\$\{\{([a-z]+)\.([A-Za-z0-9_-]+)\}\}`) + +// readCheckouts reads a recipe's git-checkouts, line by line as readRecipe does: the package's +// name and version, its vars and var-transforms, and each git-checkout's with block. +func readCheckouts(file string) ([]checkoutRef, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + vals := map[string]string{} + var transforms []map[string]string + var checkouts []map[string]string + var top string + var cur map[string]string // the git-checkout or var-transform being read + curIndent := 0 + s := bufio.NewScanner(f) + for s.Scan() { + line := s.Text() + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + indent := len(line) - len(strings.TrimLeft(line, " ")) + if indent == 0 { + top, cur = strings.TrimSuffix(trimmed, ":"), nil + continue + } + item := strings.HasPrefix(trimmed, "- ") + key, value, isKey := strings.Cut(strings.TrimPrefix(trimmed, "- "), ":") + value = strings.Trim(strings.TrimSpace(value), `"'`) + if cur != nil && (indent < curIndent || (indent == curIndent-2 && item)) { + cur = nil + } + switch { + case top == "package" && indent == 2 && isKey && (key == "name" || key == "version"): + if _, ok := vals["package."+key]; !ok { + vals["package."+key] = value + } + case top == "vars" && indent == 2 && isKey: + vals["vars."+key] = value + case top == "var-transforms" && item: + cur, curIndent = map[string]string{key: value}, indent+2 + transforms = append(transforms, cur) + case item && isKey && key == "uses" && value == "git-checkout": + cur, curIndent = map[string]string{}, indent+2 + checkouts = append(checkouts, cur) + case cur != nil && isKey && key != "with": + cur[key] = value + } + } + if err := s.Err(); err != nil { + return nil, err + } + subst := func(v string) string { + return substitution.ReplaceAllStringFunc(v, func(m string) string { + g := substitution.FindStringSubmatch(m) + if r, ok := vals[g[1]+"."+g[2]]; ok { + return r + } + return m + }) + } + // Each transform may read the ones before it, as melange applies them in order. + for _, t := range transforms { + re, err := regexp.Compile(t["match"]) + if err != nil { + return nil, fmt.Errorf("%s: var-transform %s: %w", file, t["to"], err) + } + vals["vars."+t["to"]] = re.ReplaceAllString(subst(t["from"]), t["replace"]) + } + name := strings.TrimSuffix(filepath.Base(file), ".yaml") + var refs []checkoutRef + for _, c := range checkouts { + r := checkoutRef{Recipe: name, Repository: subst(c["repository"]), Branch: subst(c["branch"]), + Tag: subst(c["tag"]), Commit: c["expected-commit"]} + for _, v := range []string{r.Repository, r.Branch, r.Tag} { + if strings.Contains(v, "${{") { + return nil, fmt.Errorf("%s: git-checkout: %s is not one this check substitutes", file, v) + } + } + if r.Repository == "" || (r.Branch == "") == (r.Tag == "") { + return nil, fmt.Errorf("%s: git-checkout: a repository and one of branch or tag", file) + } + refs = append(refs, r) + } + return refs, nil +} + +// checkRef is what is wrong with c on its remote, or "": ls-remote's lines are "\t", and +// an annotated tag's commit is its peeled ^{} line. +func checkRef(ctx context.Context, c checkoutRef) string { + ref := "refs/heads/" + c.Branch + if c.Tag != "" { + ref = "refs/tags/" + c.Tag + } + cmd := exec.CommandContext(ctx, "git", "ls-remote", c.Repository, ref, ref+"^{}") + cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") + out, err := cmd.Output() + if err != nil { + msg := err.Error() + var ee *exec.ExitError + if errors.As(err, &ee) && len(ee.Stderr) > 0 { + msg = strings.TrimSpace(string(ee.Stderr)) + } + return "git ls-remote: " + msg + } + got := map[string]string{} + for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if sha, name, ok := strings.Cut(l, "\t"); ok { + got[name] = sha + } + } + sha, ok := got[ref+"^{}"] + if !ok { + sha, ok = got[ref] + } + switch { + case !ok && c.Tag != "": + return "no such tag on the remote" + case !ok: + return "no such branch on the remote (merged and deleted? the recipe should name the branch it was merged into)" + case c.Tag != "" && c.Commit != "" && sha != c.Commit: + return fmt.Sprintf("the tag is at %s, the recipe expects %s", sha, c.Commit) + } + return "" +} + +// packagesSources checks every recipe's git-checkouts against their remotes, a few at once, and +// fails with each one that would fail melange's clone. +func packagesSources(recipesDir string) error { + paths, err := filepath.Glob(filepath.Join(recipesDir, "*.yaml")) + if err != nil { + return err + } + var refs []checkoutRef + for _, p := range paths { + r, err := readCheckouts(p) + if err != nil { + return err + } + refs = append(refs, r...) + } + var ( + mu sync.Mutex + failures []string + wg sync.WaitGroup + sem = make(chan struct{}, 16) + ) + for _, c := range refs { + wg.Add(1) + go func() { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if msg := checkRef(ctx, c); msg != "" { + mu.Lock() + failures = append(failures, fmt.Sprintf("%s: %s", c, msg)) + mu.Unlock() + } + }() + } + wg.Wait() + if len(failures) > 0 { + sort.Strings(failures) + for _, f := range failures { + fmt.Fprintln(os.Stderr, "sources:", f) + } + return fmt.Errorf("%d of %d git-checkouts would fail melange's clone", len(failures), len(refs)) + } + fmt.Printf("sources: %d git-checkouts on their remotes\n", len(refs)) + return nil +} diff --git a/cmd/image/sources_test.go b/cmd/image/sources_test.go new file mode 100644 index 0000000..f4b7981 --- /dev/null +++ b/cmd/image/sources_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "path/filepath" + "regexp" + "testing" +) + +// Every recipe's git-checkouts read to a repository and a concrete ref, the var-transforms +// applied: a recipe this check cannot read would otherwise be a release's surprise again. +func TestReadCheckoutsOfEveryRecipe(t *testing.T) { + paths, err := filepath.Glob("../../recipes/*.yaml") + if err != nil || len(paths) == 0 { + t.Fatalf("recipes: %v", err) + } + byRecipe := map[string][]checkoutRef{} + for _, p := range paths { + refs, err := readCheckouts(p) + if err != nil { + t.Fatal(err) + } + for _, r := range refs { + byRecipe[r.Recipe] = append(byRecipe[r.Recipe], r) + } + } + // openssh's tag is a var-transform of its version; mesa checks out a tag and a branch. + if got := byRecipe["openssh"]; len(got) != 1 || !regexp.MustCompile(`^V_\d+_\d+_P\d+$`).MatchString(got[0].Tag) { + t.Errorf("openssh: %+v, want a tag V___P", got) + } + if got := byRecipe["mesa"]; len(got) < 2 { + t.Errorf("mesa: %+v, want its translator's tag and the headers' branch", got) + } +} From 290cadfc37c1f01887e57222b3ef2e43aca4c4b2 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Sun, 8 Nov 2026 20:24:21 -0300 Subject: [PATCH 2/3] Taskfile: sources checked before a release, failed recipes last in its log task check, task release and release:check run packages sources first. A recipe whose build fails is recorded, and release:check ends with each one and the end of its log, after cache push: infra's release workflow shows the log's last lines, which were the kernel's. Co-Authored-By: Claude Opus 5.5 --- Taskfile.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 6e627e1..a1938f6 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -49,7 +49,7 @@ tasks: --signing-key melange.rsa {{.WOLFI}} {{.LOCAL}} --env-file build/x86-64-v3.env --out-dir .cache/out/{{.NAME}} --apk-cache-dir .cache/apk > .cache/logs/{{.NAME}}.log 2>&1 - || { tail -40 .cache/logs/{{.NAME}}.log; exit 1; } + || { echo {{.NAME}} >> .cache/failed; tail -40 .cache/logs/{{.NAME}}.log; exit 1; } - >- flock .cache/packages.lock sh -c ' cp .cache/out/{{.NAME}}/x86_64/*.apk packages/x86_64/ @@ -59,11 +59,13 @@ tasks: "mv packages/x86_64/APKINDEX.new.tar.gz packages/x86_64/APKINDEX.tar.gz && chown -R {{.UID}}:{{.UID}} packages .cache/out/{{.NAME}} .cache/logs"' check: - desc: "What a pull request must pass, with no Docker and no keys: gofmt, go vet and the Go tests, which compare the recipes with origin/main." + desc: "What a pull request must pass, with no Docker and no keys: gofmt, go vet, the Go tests, which compare the recipes with origin/main, and every recipe's git-checkout on its remote." cmds: - test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; } - go vet ./... - go test ./internal/... ./cmd/... + # A fork's branch merged and deleted fails a release minutes in; here it fails the next PR. + - go run ./cmd/image packages sources kernel:config: desc: Merge and assert the desktop kernel's configuration, without compiling it. @@ -460,6 +462,8 @@ tasks: - sh: '[ "{{.JOB}}" = release:build ] || [ "{{.JOB}}" = release:check ]' msg: JOB is release:build or release:check cmds: + # What would fail melange's clone fails here, not on a builder 25 minutes in. + - go run ./cmd/image packages sources - >- aws ec2 run-instances --launch-template LaunchTemplateName=construct-builder --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=construct-builder},{Key=construct:role,Value=builder},{Key=construct:commit,Value={{.COMMIT}}},{Key=construct:job,Value={{.JOB}}}{{if eq .NEW "true"}},{Key=construct:args,Value=NEW=true}{{end}}]' @@ -480,6 +484,14 @@ tasks: msg: "the builder builds main's commits only" cmds: - echo "check of $(git rev-parse --short HEAD)" + # Declared before cache push, so it runs after it: the failed recipes and the end of their + # logs are the last lines of the job's log, which infra's release workflow shows, instead of + # whatever the kernel, building beside them, printed for twenty minutes after. + - defer: >- + {{if .EXIT_CODE}}for r in $(sort -u .cache/failed 2>/dev/null); do + echo "FAILED: recipe $r (recipes/$r.log)"; tail -30 .cache/logs/$r.log | sed 's/^/ /'; + done{{end}} + - go run ./cmd/image packages sources - aws secretsmanager get-secret-value --secret-id construct/melange-key --query SecretString --output text > melange.rsa - chmod 600 melange.rsa - go run ./cmd/image cache pull @@ -541,6 +553,7 @@ tasks: JOBS: '{{.JOBS | default 4}}' cmds: - mkdir -p .cache + - rm -f .cache/failed - go run ./cmd/image packages missing > .cache/waves.txt - >- while read -r wave; do From 40f56ca8f9c2125f252acd12447fbe0351d89735 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Sun, 8 Nov 2026 20:47:26 -0300 Subject: [PATCH 3/3] image: TestUpdateCDN closes its connections before goleak looks http.Get's default transport keeps its connections alive past the test, and TestMain's goleak found their read and write loops on CI (os#46). Co-Authored-By: Claude Opus 5.5 --- cmd/image/cdn_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/image/cdn_test.go b/cmd/image/cdn_test.go index d9aaee7..505aa91 100644 --- a/cmd/image/cdn_test.go +++ b/cmd/image/cdn_test.go @@ -51,9 +51,14 @@ func TestUpdateCDN(t *testing.T) { t.Fatal(err) } defer c.srv.Close() + // Its own transport, its connections closed before TestMain's goleak looks: the default one's + // keep-alive connections outlive the test, and their goroutines were found as leaks. + tr := &http.Transport{} + defer tr.CloseIdleConnections() + client := &http.Client{Transport: tr} base := strings.Replace(c.url, "10.0.2.2", "127.0.0.1", 1) get := func(path string) string { - resp, err := http.Get(base + path) + resp, err := client.Get(base + path) if err != nil { t.Fatal(err) } @@ -79,7 +84,7 @@ func TestUpdateCDN(t *testing.T) { // Only what an index lists is a file: a version no release has, a file in another release's // slot, a release directory's other files, and a path out of it are all nothing. for _, p := range []string{"/testing/construct_20261004.3.efi", "/assets/v1/construct_20261004.2.efi", "/assets/v2/SHA256SUMS", "/assets/v2/../release.pem"} { - resp, err := http.Get(base + p) + resp, err := client.Get(base + p) if err != nil { t.Fatal(err) }