Release errors: recipe sources checked first, failed recipes last in the log

Taskfile.yml

@@ -49,7 +49,7 @@ tasks:
         --signing-key melange.rsa {{.WOLFI}} {{.LOCAL}} --env-file build/x86-64-v3.env
         --out-dir .cache/out/{{.NAME}} --apk-cache-dir .cache/apk
         > .cache/logs/{{.NAME}}.log 2>&1
-        || { tail -40 .cache/logs/{{.NAME}}.log; exit 1; }
+        || { echo {{.NAME}} >> .cache/failed; tail -40 .cache/logs/{{.NAME}}.log; exit 1; }
       - >-
         flock .cache/packages.lock sh -c '
         cp .cache/out/{{.NAME}}/x86_64/*.apk packages/x86_64/
@@ -59,11 +59,13 @@ tasks:
         "mv packages/x86_64/APKINDEX.new.tar.gz packages/x86_64/APKINDEX.tar.gz && chown -R {{.UID}}:{{.UID}} packages .cache/out/{{.NAME}} .cache/logs"'
 
   check:
-    desc: "What a pull request must pass, with no Docker and no keys: gofmt, go vet and the Go tests, which compare the recipes with origin/main."
+    desc: "What a pull request must pass, with no Docker and no keys: gofmt, go vet, the Go tests, which compare the recipes with origin/main, and every recipe's git-checkout on its remote."
     cmds:
       - test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; }
       - go vet ./...
       - go test ./internal/... ./cmd/...
+      # A fork's branch merged and deleted fails a release minutes in; here it fails the next PR.
+      - go run ./cmd/image packages sources
 
   kernel:config:
     desc: Merge and assert the desktop kernel's configuration, without compiling it.
@@ -460,6 +462,8 @@ tasks:
       - sh: '[ "{{.JOB}}" = release:build ] || [ "{{.JOB}}" = release:check ]'
         msg: JOB is release:build or release:check
     cmds:
+      # What would fail melange's clone fails here, not on a builder 25 minutes in.
+      - go run ./cmd/image packages sources
       - >-
         aws ec2 run-instances --launch-template LaunchTemplateName=construct-builder
         --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=construct-builder},{Key=construct:role,Value=builder},{Key=construct:commit,Value={{.COMMIT}}},{Key=construct:job,Value={{.JOB}}}{{if eq .NEW "true"}},{Key=construct:args,Value=NEW=true}{{end}}]'
@@ -480,6 +484,14 @@ tasks:
         msg: "the builder builds main's commits only"
     cmds:
       - echo "check of $(git rev-parse --short HEAD)"
+      # Declared before cache push, so it runs after it: the failed recipes and the end of their
+      # logs are the last lines of the job's log, which infra's release workflow shows, instead of
+      # whatever the kernel, building beside them, printed for twenty minutes after.
+      - defer: >-
+          {{if .EXIT_CODE}}for r in $(sort -u .cache/failed 2>/dev/null); do
+          echo "FAILED: recipe $r (recipes/$r.log)"; tail -30 .cache/logs/$r.log | sed 's/^/  /';
+          done{{end}}
+      - go run ./cmd/image packages sources
       - aws secretsmanager get-secret-value --secret-id construct/melange-key --query SecretString --output text > melange.rsa
       - chmod 600 melange.rsa
       - go run ./cmd/image cache pull
@@ -541,6 +553,7 @@ tasks:
       JOBS: '{{.JOBS | default 4}}'
     cmds:
       - mkdir -p .cache
+      - rm -f .cache/failed
       - go run ./cmd/image packages missing > .cache/waves.txt
       - >-
         while read -r wave; do

cmd/image/cdn_test.go

@@ -51,9 +51,14 @@ func TestUpdateCDN(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer c.srv.Close()
+	// Its own transport, its connections closed before TestMain's goleak looks: the default one's
+	// keep-alive connections outlive the test, and their goroutines were found as leaks.
+	tr := &http.Transport{}
+	defer tr.CloseIdleConnections()
+	client := &http.Client{Transport: tr}
 	base := strings.Replace(c.url, "10.0.2.2", "127.0.0.1", 1)
 	get := func(path string) string {
-		resp, err := http.Get(base + path)
+		resp, err := client.Get(base + path)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -79,7 +84,7 @@ func TestUpdateCDN(t *testing.T) {
 	// Only what an index lists is a file: a version no release has, a file in another release's
 	// slot, a release directory's other files, and a path out of it are all nothing.
 	for _, p := range []string{"/testing/construct_20261004.3.efi", "/assets/v1/construct_20261004.2.efi", "/assets/v2/SHA256SUMS", "/assets/v2/../release.pem"} {
-		resp, err := http.Get(base + p)
+		resp, err := client.Get(base + p)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/image/main.go

@@ -22,6 +22,7 @@
 //	image release-notes -version v -packages f  a release's notes (cmd/image/notes.go)
 //	image channel-index -version v -add|-withdraw  a channel's index with a release added or withdrawn (cmd/image/channel.go)
 //	image packages missing|prune                the recipes packages/ lacks, a wave a line; builds no recipe makes out (cmd/image/packages.go)
+//	image packages sources                      every recipe's git-checkout branch and tag on its remote (cmd/image/sources.go)
 //	image lock write [-all] | index             the locks from apko's resolution, or Wolfi's lines bumped to its index (cmd/image/lock.go)
 //	image cache pull|push                       the release builder's packages/, kernel, firmware and QEMU in S3 (cmd/image/cache.go)
 //	image usb-disk                              the stick's disk from the signed release (cmd/image/usb.go)

cmd/image/packages.go

@@ -219,8 +219,11 @@ func unmade(dir string, recipes []recipe) ([]string, error) {
 // a line per wave that builds side by side, or image packages prune, which removes every apk no
 // recipe makes now and prints the files it removed, for the index to be made again.
 func packagesCmd(args []string) error {
-	if len(args) != 1 || (args[0] != "missing" && args[0] != "prune") {
-		return errors.New("packages missing|prune")
+	if len(args) != 1 || (args[0] != "missing" && args[0] != "prune" && args[0] != "sources") {
+		return errors.New("packages missing|prune|sources")
+	}
+	if args[0] == "sources" {
+		return packagesSources("recipes")
 	}
 	recipes, err := readRecipes("recipes")
 	if err != nil {

cmd/image/sources.go

@@ -0,0 +1,210 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+// The recipes' sources before a builder is launched: every git-checkout's branch is on its
+// remote, and every tag is, at the commit the recipe expects. A branch merged and deleted, or a
+// tag moved, fails melange's clone minutes into a release, buried among the builds beside it;
+// git ls-remote says so in seconds, before anything is built.
+
+// checkoutRef is one git-checkout of a recipe, its ${{...}} substituted.
+type checkoutRef struct {
+	Recipe, Repository, Branch, Tag, Commit string
+}
+
+func (c checkoutRef) String() string {
+	ref := "branch " + c.Branch
+	if c.Tag != "" {
+		ref = "tag " + c.Tag
+	}
+	return fmt.Sprintf("%s: %s %s", c.Recipe, c.Repository, ref)
+}
+
+var substitution = regexp.MustCompile(`\$\{\{([a-z]+)\.([A-Za-z0-9_-]+)\}\}`)
+
+// readCheckouts reads a recipe's git-checkouts, line by line as readRecipe does: the package's
+// name and version, its vars and var-transforms, and each git-checkout's with block.
+func readCheckouts(file string) ([]checkoutRef, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	vals := map[string]string{}
+	var transforms []map[string]string
+	var checkouts []map[string]string
+	var top string
+	var cur map[string]string // the git-checkout or var-transform being read
+	curIndent := 0
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		line := s.Text()
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		indent := len(line) - len(strings.TrimLeft(line, " "))
+		if indent == 0 {
+			top, cur = strings.TrimSuffix(trimmed, ":"), nil
+			continue
+		}
+		item := strings.HasPrefix(trimmed, "- ")
+		key, value, isKey := strings.Cut(strings.TrimPrefix(trimmed, "- "), ":")
+		value = strings.Trim(strings.TrimSpace(value), `"'`)
+		if cur != nil && (indent < curIndent || (indent == curIndent-2 && item)) {
+			cur = nil
+		}
+		switch {
+		case top == "package" && indent == 2 && isKey && (key == "name" || key == "version"):
+			if _, ok := vals["package."+key]; !ok {
+				vals["package."+key] = value
+			}
+		case top == "vars" && indent == 2 && isKey:
+			vals["vars."+key] = value
+		case top == "var-transforms" && item:
+			cur, curIndent = map[string]string{key: value}, indent+2
+			transforms = append(transforms, cur)
+		case item && isKey && key == "uses" && value == "git-checkout":
+			cur, curIndent = map[string]string{}, indent+2
+			checkouts = append(checkouts, cur)
+		case cur != nil && isKey && key != "with":
+			cur[key] = value
+		}
+	}
+	if err := s.Err(); err != nil {
+		return nil, err
+	}
+	subst := func(v string) string {
+		return substitution.ReplaceAllStringFunc(v, func(m string) string {
+			g := substitution.FindStringSubmatch(m)
+			if r, ok := vals[g[1]+"."+g[2]]; ok {
+				return r
+			}
+			return m
+		})
+	}
+	// Each transform may read the ones before it, as melange applies them in order.
+	for _, t := range transforms {
+		re, err := regexp.Compile(t["match"])
+		if err != nil {
+			return nil, fmt.Errorf("%s: var-transform %s: %w", file, t["to"], err)
+		}
+		vals["vars."+t["to"]] = re.ReplaceAllString(subst(t["from"]), t["replace"])
+	}
+	name := strings.TrimSuffix(filepath.Base(file), ".yaml")
+	var refs []checkoutRef
+	for _, c := range checkouts {
+		r := checkoutRef{Recipe: name, Repository: subst(c["repository"]), Branch: subst(c["branch"]),
+			Tag: subst(c["tag"]), Commit: c["expected-commit"]}
+		for _, v := range []string{r.Repository, r.Branch, r.Tag} {
+			if strings.Contains(v, "${{") {
+				return nil, fmt.Errorf("%s: git-checkout: %s is not one this check substitutes", file, v)
+			}
+		}
+		if r.Repository == "" || (r.Branch == "") == (r.Tag == "") {
+			return nil, fmt.Errorf("%s: git-checkout: a repository and one of branch or tag", file)
+		}
+		refs = append(refs, r)
+	}
+	return refs, nil
+}
+
+// checkRef is what is wrong with c on its remote, or "": ls-remote's lines are "<sha>\t<ref>", and
+// an annotated tag's commit is its peeled ^{} line.
+func checkRef(ctx context.Context, c checkoutRef) string {
+	ref := "refs/heads/" + c.Branch
+	if c.Tag != "" {
+		ref = "refs/tags/" + c.Tag
+	}
+	cmd := exec.CommandContext(ctx, "git", "ls-remote", c.Repository, ref, ref+"^{}")
+	cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
+	out, err := cmd.Output()
+	if err != nil {
+		msg := err.Error()
+		var ee *exec.ExitError
+		if errors.As(err, &ee) && len(ee.Stderr) > 0 {
+			msg = strings.TrimSpace(string(ee.Stderr))
+		}
+		return "git ls-remote: " + msg
+	}
+	got := map[string]string{}
+	for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		if sha, name, ok := strings.Cut(l, "\t"); ok {
+			got[name] = sha
+		}
+	}
+	sha, ok := got[ref+"^{}"]
+	if !ok {
+		sha, ok = got[ref]
+	}
+	switch {
+	case !ok && c.Tag != "":
+		return "no such tag on the remote"
+	case !ok:
+		return "no such branch on the remote (merged and deleted? the recipe should name the branch it was merged into)"
+	case c.Tag != "" && c.Commit != "" && sha != c.Commit:
+		return fmt.Sprintf("the tag is at %s, the recipe expects %s", sha, c.Commit)
+	}
+	return ""
+}
+
+// packagesSources checks every recipe's git-checkouts against their remotes, a few at once, and
+// fails with each one that would fail melange's clone.
+func packagesSources(recipesDir string) error {
+	paths, err := filepath.Glob(filepath.Join(recipesDir, "*.yaml"))
+	if err != nil {
+		return err
+	}
+	var refs []checkoutRef
+	for _, p := range paths {
+		r, err := readCheckouts(p)
+		if err != nil {
+			return err
+		}
+		refs = append(refs, r...)
+	}
+	var (
+		mu       sync.Mutex
+		failures []string
+		wg       sync.WaitGroup
+		sem      = make(chan struct{}, 16)
+	)
+	for _, c := range refs {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
+			ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+			defer cancel()
+			if msg := checkRef(ctx, c); msg != "" {
+				mu.Lock()
+				failures = append(failures, fmt.Sprintf("%s: %s", c, msg))
+				mu.Unlock()
+			}
+		}()
+	}
+	wg.Wait()
+	if len(failures) > 0 {
+		sort.Strings(failures)
+		for _, f := range failures {
+			fmt.Fprintln(os.Stderr, "sources:", f)
+		}
+		return fmt.Errorf("%d of %d git-checkouts would fail melange's clone", len(failures), len(refs))
+	}
+	fmt.Printf("sources: %d git-checkouts on their remotes\n", len(refs))
+	return nil
+}

cmd/image/sources_test.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"path/filepath"
+	"regexp"
+	"testing"
+)
+
+// Every recipe's git-checkouts read to a repository and a concrete ref, the var-transforms
+// applied: a recipe this check cannot read would otherwise be a release's surprise again.
+func TestReadCheckoutsOfEveryRecipe(t *testing.T) {
+	paths, err := filepath.Glob("../../recipes/*.yaml")
+	if err != nil || len(paths) == 0 {
+		t.Fatalf("recipes: %v", err)
+	}
+	byRecipe := map[string][]checkoutRef{}
+	for _, p := range paths {
+		refs, err := readCheckouts(p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, r := range refs {
+			byRecipe[r.Recipe] = append(byRecipe[r.Recipe], r)
+		}
+	}
+	// openssh's tag is a var-transform of its version; mesa checks out a tag and a branch.
+	if got := byRecipe["openssh"]; len(got) != 1 || !regexp.MustCompile(`^V_\d+_\d+_P\d+$`).MatchString(got[0].Tag) {
+		t.Errorf("openssh: %+v, want a tag V_<major>_<minor>_P<patch>", got)
+	}
+	if got := byRecipe["mesa"]; len(got) < 2 {
+		t.Errorf("mesa: %+v, want its translator's tag and the headers' branch", got)
+	}
+}