From 72ccfc6c3d3d39849aa8df4e17910d7eb940af52 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Tue, 8 Sep 2026 21:48:42 -0700 Subject: [PATCH 01/74] Add offline worker report rendering --- .gitignore | 5 ++- README.md | 13 ++++++++ audit-worker-versions/main.go | 53 +++++++++++++++++++++++++++--- audit-worker-versions/main_test.go | 29 ++++++++++++++++ 4 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 audit-worker-versions/main_test.go diff --git a/.gitignore b/.gitignore index 09a76685e..fedf60eff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -mozilla-history -audit-worker-versions -!audit-worker-versions/main.go +# Locally built root executable. +/mozilla-history diff --git a/README.md b/README.md index 3d55383c6..5f8bd4bc4 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,19 @@ mozilla-history This will populate subdirectories `Clients`, `Hooks`, `Roles` and `WorkerPools` of the current directory. +## Rendering an Existing Worker Snapshot + +The worker-version report can be regenerated from a saved snapshot without +Taskcluster credentials or probe tasks: + +```sh +go run ./audit-worker-versions render \ + WorkerVersions/workers.json \ + WorkerVersions/README.md +``` + +Omit the output path to print the generated Markdown to standard output. + ## Automating the Process You can automate this reporting process by setting up a cron job to execute `run-report.sh` at regular intervals. diff --git a/audit-worker-versions/main.go b/audit-worker-versions/main.go index f7a4b8312..0feb63854 100644 --- a/audit-worker-versions/main.go +++ b/audit-worker-versions/main.go @@ -181,7 +181,7 @@ var ( ) var ( - outputDir = "WorkerVersions" + outputDir = "WorkerVersions" ) func FilenameEscape(raw string) (escaped string) { @@ -195,7 +195,6 @@ func EmptyDirectory(dir string) { } } - func WriteFile(path string, content []byte) { err := os.MkdirAll(filepath.Dir(path), 0755) if err != nil { @@ -204,7 +203,7 @@ func WriteFile(path string, content []byte) { err = os.WriteFile(path, content, 0644) if err != nil { - log.Fatalf("Error:\n%v", err) + log.Fatalf("Error:\n%v", err) } } @@ -217,6 +216,20 @@ func WriteFile(path string, content []byte) { // // Files are written to the WorkerVersions directory func main() { + if len(os.Args) >= 2 && os.Args[1] == "render" { + if len(os.Args) < 3 || len(os.Args) > 4 { + log.Fatal("Usage: audit-worker-versions render INPUT_JSON [OUTPUT_MARKDOWN]") + } + workers, err := readSnapshot(os.Args[2]) + fatalOnError(err) + contents := renderReadme(workers) + if len(os.Args) == 4 { + WriteFile(os.Args[3], []byte(contents)) + } else { + fmt.Print(contents) + } + return + } queue := tcqueue.NewFromEnv() @@ -404,6 +417,9 @@ func generateReadmeSection(title string, workers []WorkerInfo, filter func(Worke for k, v := range imagesets { images = append(images, kv{k, v}) } + sort.Slice(images, func(i, j int) bool { + return strings.Compare(images[i].Key, images[j].Key) < 0 + }) return map[string]interface{}{ "FullColumns": title == "Generic Worker", @@ -417,7 +433,10 @@ func generateReadmeSection(title string, workers []WorkerInfo, filter func(Worke func writeReadme(workers []WorkerInfo) { filename := filepath.Join(outputDir, "README.md") + WriteFile(filename, []byte(renderReadme(workers))) +} +func renderReadme(workers []WorkerInfo) string { sections := [5]map[string]interface{}{ generateReadmeSection("Generic Worker", workers, func(w WorkerInfo) bool { return w.Implementation == "generic-worker" }), generateReadmeSection("Docker Worker", workers, func(w WorkerInfo) bool { return w.Implementation == "docker-worker" }), @@ -426,8 +445,32 @@ func writeReadme(workers []WorkerInfo) { generateReadmeSection("Version not determined [^2]", workers, func(w WorkerInfo) bool { return w.isUnknown }), } - contents := renderTemplate(sections) - WriteFile(filename, []byte(contents)) + return renderTemplate(sections) +} + +func readSnapshot(filename string) ([]WorkerInfo, error) { + contents, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + workers := []WorkerInfo{} + if err := json.Unmarshal(contents, &workers); err != nil { + return nil, err + } + + // These flags are internal rendering state and are not serialized in the + // snapshot. Restore them from the persisted error value for offline renders. + for i := range workers { + switch workers[i].Details["error"] { + case "No artifacts found": + workers[i].hasNoArtifacts = true + case "Version not determined; task not (yet) claimed": + workers[i].isUnknown = true + } + } + + return workers, nil } func writeSnapshot(workers []WorkerInfo) { diff --git a/audit-worker-versions/main_test.go b/audit-worker-versions/main_test.go new file mode 100644 index 000000000..7b60f308a --- /dev/null +++ b/audit-worker-versions/main_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestReadSnapshotRestoresRenderingState(t *testing.T) { + filename := filepath.Join(t.TempDir(), "workers.json") + data := `[ + {"WorkerPoolID":"one/pool","Details":{"error":"No artifacts found"}}, + {"WorkerPoolID":"two/pool","Details":{"error":"Version not determined; task not (yet) claimed"}} + ]` + if err := os.WriteFile(filename, []byte(data), 0644); err != nil { + t.Fatal(err) + } + + workers, err := readSnapshot(filename) + if err != nil { + t.Fatal(err) + } + if !workers[0].hasNoArtifacts { + t.Error("no-artifacts state was not restored") + } + if !workers[1].isUnknown { + t.Error("unknown-version state was not restored") + } +} From eed63994ff332b199aeeabe4351952530776940e Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Tue, 8 Sep 2026 21:51:55 -0700 Subject: [PATCH 02/74] Link worker pools and improve report navigation --- audit-worker-versions/main.go | 17 ++++++++++++--- audit-worker-versions/main_test.go | 33 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/audit-worker-versions/main.go b/audit-worker-versions/main.go index 0feb63854..42adef0c6 100644 --- a/audit-worker-versions/main.go +++ b/audit-worker-versions/main.go @@ -7,6 +7,7 @@ import ( "io" "log" "net/http" + "net/url" "os" "path/filepath" "regexp" @@ -40,6 +41,16 @@ type WorkerInfo struct { TotalCapacity int } +func (w WorkerInfo) WorkerPoolURL() string { + parts := strings.SplitN(w.WorkerPoolID, "/", 2) + if len(parts) != 2 { + return "" + } + return "https://firefox-ci-tc.services.mozilla.com/provisioners/" + + url.PathEscape(parts[0]) + "/worker-types/" + url.PathEscape(parts[1]) + + "?sortBy=Last%20Active&sortDirection=desc" +} + func (w *WorkerInfo) String() string { revision := "" engine := "" @@ -130,7 +141,7 @@ var ( Total: ` + "`" + `{{ .Count }}` + "`" + ` {{ if gt (len .Versions) 1 }} -Count by version: +### Count by version | Version | Count | | :--- | ---: | @@ -139,7 +150,7 @@ Count by version: {{ end }} {{- end }} {{ if gt (len .Images) 1 }} -Count by image: +### Count by image | Version | Count | | :--- | ---: | @@ -151,7 +162,7 @@ Count by image: | Worker Pool | Implementation | Version {{ if .FullColumns }}| Engine | Revision | OS | Arch | GO {{ end }}| Total Workers | Total Capacity | | --- | --- | --- {{ if .FullColumns }}| --- | --- | --- | --- | --- {{ end }}| ---: | ---: | {{ range .Filtered -}} -| **{{ .WorkerPoolID }}** | {{ .Implementation }} | {{ or .Version .Details.error }} {{ if $.FullColumns }}| {{ or .Details.engine "-" }} | {{ or (slice .Details.revision 0 10) "-" }} | {{ or .Details.os "-" }} | {{ or .Details.arch "-" }} | {{ or .Details.go "-" }} {{ end }}| {{ .TotalWorkers }} | {{ .TotalCapacity }} | +| [**{{ .WorkerPoolID }}**]({{ .WorkerPoolURL }}) | {{ .Implementation }} | {{ or .Version .Details.error }} {{ if $.FullColumns }}| {{ or .Details.engine "-" }} | {{ or (slice .Details.revision 0 10) "-" }} | {{ or .Details.os "-" }} | {{ or .Details.arch "-" }} | {{ or .Details.go "-" }} {{ end }}| {{ .TotalWorkers }} | {{ .TotalCapacity }} | {{end}} {{- end -}} {{end}} diff --git a/audit-worker-versions/main_test.go b/audit-worker-versions/main_test.go index 7b60f308a..be535cd00 100644 --- a/audit-worker-versions/main_test.go +++ b/audit-worker-versions/main_test.go @@ -3,9 +3,42 @@ package main import ( "os" "path/filepath" + "strings" "testing" ) +func TestWorkerPoolURL(t *testing.T) { + worker := WorkerInfo{WorkerPoolID: "releng-hardware/gecko-t-win7-32-hw"} + want := "https://firefox-ci-tc.services.mozilla.com/provisioners/releng-hardware/worker-types/gecko-t-win7-32-hw?sortBy=Last%20Active&sortDirection=desc" + if got := worker.WorkerPoolURL(); got != want { + t.Fatalf("WorkerPoolURL() = %q, want %q", got, want) + } +} + +func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { + workers := []WorkerInfo{{ + WorkerPoolID: "example/pool", + Implementation: "generic-worker", + Version: "1.2.3", + Details: map[string]string{"revision": "1234567890"}, + }, { + WorkerPoolID: "example/other-pool", + Implementation: "generic-worker", + Version: "2.0.0", + Details: map[string]string{"revision": "1234567890"}, + }} + + got := renderReadme(workers) + for _, want := range []string{ + "### Count by version", + "[**example/pool**](https://firefox-ci-tc.services.mozilla.com/provisioners/example/worker-types/pool?sortBy=Last%20Active&sortDirection=desc)", + } { + if !strings.Contains(got, want) { + t.Errorf("rendered README does not contain %q", want) + } + } +} + func TestReadSnapshotRestoresRenderingState(t *testing.T) { filename := filepath.Join(t.TempDir(), "workers.json") data := `[ From c63b4d2fd5380be583fc370c05b8c7209e9b14e3 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Tue, 8 Sep 2026 22:23:03 -0700 Subject: [PATCH 03/74] Extract report rendering and improve navigation --- README.md | 10 + audit-worker-versions/main.go | 155 ---------------- audit-worker-versions/report.go | 171 ++++++++++++++++++ .../{main_test.go => report_test.go} | 23 +++ docs/app.js | 155 ++++++++++++++++ docs/index-local.html | 17 ++ 6 files changed, 376 insertions(+), 155 deletions(-) create mode 100644 audit-worker-versions/report.go rename audit-worker-versions/{main_test.go => report_test.go} (68%) create mode 100644 docs/app.js create mode 100644 docs/index-local.html diff --git a/README.md b/README.md index 5f8bd4bc4..5fbcf3121 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,16 @@ go run ./audit-worker-versions render \ Omit the output path to print the generated Markdown to standard output. +To preview the generated report in the website, serve the repository root: + +```sh +python3 -m http.server 8000 +``` + +Then open . The local preview uses +`WorkerVersions/README.md` and `docs/history.json` from the checkout. The +published page continues to load the current report from GitHub. + ## Automating the Process You can automate this reporting process by setting up a cron job to execute `run-report.sh` at regular intervals. diff --git a/audit-worker-versions/main.go b/audit-worker-versions/main.go index 42adef0c6..aabba5cef 100644 --- a/audit-worker-versions/main.go +++ b/audit-worker-versions/main.go @@ -1,20 +1,17 @@ package main import ( - "bytes" "encoding/json" "fmt" "io" "log" "net/http" - "net/url" "os" "path/filepath" "regexp" "sort" "strings" "sync" - "text/template" "time" "github.com/taskcluster/httpbackoff/v3" @@ -41,16 +38,6 @@ type WorkerInfo struct { TotalCapacity int } -func (w WorkerInfo) WorkerPoolURL() string { - parts := strings.SplitN(w.WorkerPoolID, "/", 2) - if len(parts) != 2 { - return "" - } - return "https://firefox-ci-tc.services.mozilla.com/provisioners/" + - url.PathEscape(parts[0]) + "/worker-types/" + url.PathEscape(parts[1]) + - "?sortBy=Last%20Active&sortDirection=desc" -} - func (w *WorkerInfo) String() string { revision := "" engine := "" @@ -133,59 +120,6 @@ func GetImageset(wp *tcworkermanager.WorkerPoolFullDefinition) string { return sortedImages } -var ( - // Templates for README file - readmeTpl string = ` -{{- define "row" -}} -## {{ .Title }} - -Total: ` + "`" + `{{ .Count }}` + "`" + ` -{{ if gt (len .Versions) 1 }} -### Count by version - -| Version | Count | -| :--- | ---: | -{{ range .Versions -}} -| {{ .Key }} | {{ .Value }} | -{{ end }} -{{- end }} -{{ if gt (len .Images) 1 }} -### Count by image - -| Version | Count | -| :--- | ---: | -{{ range .Images -}} -| {{ .Key }} | {{ .Value }} | -{{ end }} -{{- end }} -{{if .Count }} -| Worker Pool | Implementation | Version {{ if .FullColumns }}| Engine | Revision | OS | Arch | GO {{ end }}| Total Workers | Total Capacity | -| --- | --- | --- {{ if .FullColumns }}| --- | --- | --- | --- | --- {{ end }}| ---: | ---: | -{{ range .Filtered -}} -| [**{{ .WorkerPoolID }}**]({{ .WorkerPoolURL }}) | {{ .Implementation }} | {{ or .Version .Details.error }} {{ if $.FullColumns }}| {{ or .Details.engine "-" }} | {{ or (slice .Details.revision 0 10) "-" }} | {{ or .Details.os "-" }} | {{ or .Details.arch "-" }} | {{ or .Details.go "-" }} {{ end }}| {{ .TotalWorkers }} | {{ .TotalCapacity }} | -{{end}} -{{- end -}} -{{end}} - -# Worker Pool Versions - -{{ range . }} -{{ template "row" . }} -{{ end }} - -[^1]: Those are the pools whose tasks were claimed and resolved by a worker as expected, but the worker did not publish either artifact ` + "`public/logs/live_backing.log` nor `public/logs/chain_of_trust.log`" + `, which is the source used to identify the worker implementation. - -[^2]: Probing task remains pending after two hours. Those are the pools that were not able to start any worker to claim the task within two hours. -` -) - -func renderTemplate(data interface{}) string { - t := template.Must(template.New("").Parse(readmeTpl)) - var content bytes.Buffer - t.Execute(&content, data) - return content.String() -} - var ( // set during build with `-ldflags "-X main.revision=$(git rev-parse HEAD)"` revision string = "" @@ -395,95 +329,6 @@ func inspect(queue *tcqueue.Queue, taskIDs []string) { writeSnapshot(workers) } -func generateReadmeSection(title string, workers []WorkerInfo, filter func(WorkerInfo) bool) map[string]interface{} { - filtered := make([]WorkerInfo, 0) - versions := make(map[string]int) - imagesets := make(map[string]int) - - for _, w := range workers { - if filter(w) { - filtered = append(filtered, w) - versions[w.Version]++ - imagesets[w.Imageset]++ - } - } - - sort.Slice(filtered, func(i, j int) bool { - return strings.Compare(filtered[i].WorkerPoolID, filtered[j].WorkerPoolID) < 0 - }) - - type kv struct { - Key string - Value int - } - var sortedVersions []kv - for k, v := range versions { - sortedVersions = append(sortedVersions, kv{k, v}) - } - sort.Slice(sortedVersions, func(i, j int) bool { - return strings.Compare(sortedVersions[i].Key, sortedVersions[j].Key) < 0 - }) - - var images []kv - for k, v := range imagesets { - images = append(images, kv{k, v}) - } - sort.Slice(images, func(i, j int) bool { - return strings.Compare(images[i].Key, images[j].Key) < 0 - }) - - return map[string]interface{}{ - "FullColumns": title == "Generic Worker", - "Filtered": filtered, - "Count": len(filtered), - "Versions": sortedVersions, - "Images": images, - "Title": title, - } -} - -func writeReadme(workers []WorkerInfo) { - filename := filepath.Join(outputDir, "README.md") - WriteFile(filename, []byte(renderReadme(workers))) -} - -func renderReadme(workers []WorkerInfo) string { - sections := [5]map[string]interface{}{ - generateReadmeSection("Generic Worker", workers, func(w WorkerInfo) bool { return w.Implementation == "generic-worker" }), - generateReadmeSection("Docker Worker", workers, func(w WorkerInfo) bool { return w.Implementation == "docker-worker" }), - generateReadmeSection("Script Worker", workers, func(w WorkerInfo) bool { return strings.Contains(w.Implementation, "Scriptworker") }), - generateReadmeSection("No artifacts found [^1]", workers, func(w WorkerInfo) bool { return w.hasNoArtifacts }), - generateReadmeSection("Version not determined [^2]", workers, func(w WorkerInfo) bool { return w.isUnknown }), - } - - return renderTemplate(sections) -} - -func readSnapshot(filename string) ([]WorkerInfo, error) { - contents, err := os.ReadFile(filename) - if err != nil { - return nil, err - } - - workers := []WorkerInfo{} - if err := json.Unmarshal(contents, &workers); err != nil { - return nil, err - } - - // These flags are internal rendering state and are not serialized in the - // snapshot. Restore them from the persisted error value for offline renders. - for i := range workers { - switch workers[i].Details["error"] { - case "No artifacts found": - workers[i].hasNoArtifacts = true - case "Version not determined; task not (yet) claimed": - workers[i].isUnknown = true - } - } - - return workers, nil -} - func writeSnapshot(workers []WorkerInfo) { filename := filepath.Join(outputDir, "workers.json") diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go new file mode 100644 index 000000000..edf8b075d --- /dev/null +++ b/audit-worker-versions/report.go @@ -0,0 +1,171 @@ +package main + +import ( + "bytes" + "encoding/json" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "text/template" +) + +type count struct { + Key string + Value int +} + +type reportSection struct { + Title string + Description string + Count int + Versions []count + Images []count + Filtered []WorkerInfo + FullColumns bool +} + +func (w WorkerInfo) WorkerPoolURL() string { + parts := strings.SplitN(w.WorkerPoolID, "/", 2) + if len(parts) != 2 { + return "" + } + return "https://firefox-ci-tc.services.mozilla.com/provisioners/" + + url.PathEscape(parts[0]) + "/worker-types/" + url.PathEscape(parts[1]) + + "?sortBy=Last%20Active&sortDirection=desc" +} + +const readmeTpl = ` +{{- define "row" -}} +## {{ .Title }} +{{ if .Description }} +{{ .Description }} +{{ end }} + +Total: ` + "`" + `{{ .Count }}` + "`" + ` +{{ if gt (len .Versions) 1 }} +### Count by version + +| Version | Count | +| :--- | ---: | +{{ range .Versions -}} +| {{ .Key }} | {{ .Value }} | +{{ end }} +{{- end }} +{{ if gt (len .Images) 1 }} +### Count by image + +| Version | Count | +| :--- | ---: | +{{ range .Images -}} +| {{ .Key }} | {{ .Value }} | +{{ end }} +{{- end }} +{{if .Count }} +### Worker pools + +| Worker Pool | Implementation | Version {{ if .FullColumns }}| Engine | Revision | OS | Arch | GO {{ end }}| Total Workers | Total Capacity | +| --- | --- | --- {{ if .FullColumns }}| --- | --- | --- | --- | --- {{ end }}| ---: | ---: | +{{ range .Filtered -}} +| [**{{ .WorkerPoolID }}**]({{ .WorkerPoolURL }}) | {{ .Implementation }} | {{ or .Version .Details.error }} {{ if $.FullColumns }}| {{ or .Details.engine "-" }} | {{ or (slice .Details.revision 0 10) "-" }} | {{ or .Details.os "-" }} | {{ or .Details.arch "-" }} | {{ or .Details.go "-" }} {{ end }}| {{ .TotalWorkers }} | {{ .TotalCapacity }} | +{{end}} +{{- end -}} +{{end}} + +# Worker Pool Versions + +{{ range . }} +{{ template "row" . }} +{{ end }} +` + +func renderTemplate(data interface{}) string { + t := template.Must(template.New("").Parse(readmeTpl)) + var content bytes.Buffer + if err := t.Execute(&content, data); err != nil { + panic(err) + } + return strings.TrimSpace(content.String()) + "\n" +} + +func sortedCounts(values map[string]int) []count { + counts := make([]count, 0, len(values)) + for key, value := range values { + counts = append(counts, count{Key: key, Value: value}) + } + sort.Slice(counts, func(i, j int) bool { + return strings.Compare(counts[i].Key, counts[j].Key) < 0 + }) + return counts +} + +func generateReadmeSection(title, description string, workers []WorkerInfo, filter func(WorkerInfo) bool) reportSection { + filtered := make([]WorkerInfo, 0) + versions := make(map[string]int) + imagesets := make(map[string]int) + + for _, worker := range workers { + if filter(worker) { + filtered = append(filtered, worker) + versions[worker.Version]++ + imagesets[worker.Imageset]++ + } + } + + sort.Slice(filtered, func(i, j int) bool { + return strings.Compare(filtered[i].WorkerPoolID, filtered[j].WorkerPoolID) < 0 + }) + + return reportSection{ + Title: title, + Description: description, + Count: len(filtered), + Versions: sortedCounts(versions), + Images: sortedCounts(imagesets), + Filtered: filtered, + FullColumns: title == "Generic Worker", + } +} + +func writeReadme(workers []WorkerInfo) { + filename := filepath.Join(outputDir, "README.md") + WriteFile(filename, []byte(renderReadme(workers))) +} + +func renderReadme(workers []WorkerInfo) string { + sections := [5]reportSection{ + generateReadmeSection("Generic Worker", "", workers, func(w WorkerInfo) bool { return w.Implementation == "generic-worker" }), + generateReadmeSection("Docker Worker", "", workers, func(w WorkerInfo) bool { return w.Implementation == "docker-worker" }), + generateReadmeSection("Script Worker", "", workers, func(w WorkerInfo) bool { return strings.Contains(w.Implementation, "Scriptworker") }), + generateReadmeSection("No artifacts found", "These pools claimed and resolved the probe task, but did not publish `public/logs/live_backing.log` or `public/logs/chain_of_trust.log`, which are used to identify the worker implementation.", workers, func(w WorkerInfo) bool { return w.hasNoArtifacts }), + generateReadmeSection("Version not determined", "These pools did not claim the probe task within two hours, so their worker implementation and version could not be determined.", workers, func(w WorkerInfo) bool { return w.isUnknown }), + } + + return renderTemplate(sections) +} + +func readSnapshot(filename string) ([]WorkerInfo, error) { + contents, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + workers := []WorkerInfo{} + if err := json.Unmarshal(contents, &workers); err != nil { + return nil, err + } + + // These flags are internal rendering state and are not serialized in the + // snapshot. Restore them from the persisted error value for offline renders. + for i := range workers { + switch workers[i].Details["error"] { + case "No artifacts found": + workers[i].hasNoArtifacts = true + case "Version not determined; task not (yet) claimed": + workers[i].isUnknown = true + } + } + + return workers, nil +} diff --git a/audit-worker-versions/main_test.go b/audit-worker-versions/report_test.go similarity index 68% rename from audit-worker-versions/main_test.go rename to audit-worker-versions/report_test.go index be535cd00..9f8f7aecb 100644 --- a/audit-worker-versions/main_test.go +++ b/audit-worker-versions/report_test.go @@ -31,6 +31,7 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { got := renderReadme(workers) for _, want := range []string{ "### Count by version", + "### Worker pools", "[**example/pool**](https://firefox-ci-tc.services.mozilla.com/provisioners/example/worker-types/pool?sortBy=Last%20Active&sortDirection=desc)", } { if !strings.Contains(got, want) { @@ -39,6 +40,28 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { } } +func TestRenderReadmeExplainsIncompleteProbesInline(t *testing.T) { + workers := []WorkerInfo{ + {WorkerPoolID: "example/no-artifact", Details: map[string]string{"error": "No artifacts found"}, hasNoArtifacts: true}, + {WorkerPoolID: "example/pending", Details: map[string]string{"error": "Version not determined; task not (yet) claimed"}, isUnknown: true}, + } + + got := renderReadme(workers) + for _, want := range []string{ + "## No artifacts found\n", + "did not publish `public/logs/live_backing.log`", + "## Version not determined\n", + "did not claim the probe task within two hours", + } { + if !strings.Contains(got, want) { + t.Errorf("rendered README does not contain %q", want) + } + } + if strings.Contains(got, "[^1]") || strings.Contains(got, "[^2]") { + t.Error("rendered README contains obsolete footnote markers") + } +} + func TestReadSnapshotRestoresRenderingState(t *testing.T) { filename := filepath.Join(t.TempDir(), "workers.json") data := `[ diff --git a/docs/app.js b/docs/app.js new file mode 100644 index 000000000..275f113b6 --- /dev/null +++ b/docs/app.js @@ -0,0 +1,155 @@ +// This works for both the Mozilla and community history GitHub Pages sites. +let basePath = String(window.location.pathname) +if (!basePath.includes('-history')) basePath = '/mozilla-history/' + +const localPreview = new URLSearchParams(window.location.search).has('local') +const readmeUrl = localPreview + ? new URL('../WorkerVersions/README.md', window.location.href) + : `https://raw.githubusercontent.com/taskcluster${basePath}master/WorkerVersions/README.md` +const historyUrl = new URL('history.json', window.location.href) + +async function loadReadme() { + const response = await fetch(readmeUrl) + const markdown = await response.text() + showdown.setFlavor('github') + document.getElementById('content').innerHTML = new showdown.Converter({ + ghCompatibleHeaderId: true, + tables: true, + }).makeHtml(markdown) +} + +function renderHistoryTables(data) { + const dates = Object.keys(data).sort((a, b) => b.localeCompare(a)) + + const tableByDate = (title, property) => { + let columns = [] + dates.forEach(date => { + const values = data[date][property] + Object.keys(values).forEach(key => { + if (!columns.includes(key)) columns.push(key) + }) + }) + columns = columns.sort((a, b) => a.localeCompare(b)) + + let table = `

${title}

` + table += `${columns.map(key => ``).join('')}` + dates.forEach(date => { + table += ` + ${columns.map(column => data[date][property][column] || '-').map(value => ``).join('')} + ` + }) + table += '
Date${key || 'unknown'}
${date}${value}
' + return table + } + + let html = '

History

' + html += tableByDate('Worker implementations', 'implementations') + html += tableByDate('Worker versions', 'versions') + document.getElementById('history-content').innerHTML = html +} + +function renderGraphs(data) { + const dates = Object.keys(data).sort() + const implementationTraces = [ + ['Generic worker', 'generic-worker'], + ['Docker worker', 'docker-worker'], + ['Unknown', ''], + ].map(([name, implementation]) => ({ + x: dates, + y: dates.map(date => data[date].implementations[implementation] || 0), + name, + type: 'line', + })) + + const allVersions = [...new Set(dates.map(date => Object.keys(data[date].versions)).flat())].sort() + const versionTraces = allVersions.map(version => ({ + x: dates, + y: dates.map(date => data[date].versions[version] || 0), + name: version, + type: 'line', + })) + + const layout = { barmode: 'group' } + Plotly.newPlot('graph-implementations', implementationTraces, { + ...layout, + title: 'Worker Implementations', + }) + Plotly.newPlot('graph-versions', versionTraces, { + ...layout, + title: 'Worker Versions', + }) +} + +async function loadHistory() { + const response = await fetch(historyUrl) + const data = await response.json() + renderHistoryTables(data) + renderGraphs(data) +} + +function slugify(value) { + return value + .toLowerCase() + .trim() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') +} + +function buildTableOfContents() { + const headings = document.querySelectorAll([ + '#content h2', + '#content h3', + '#history-content h2', + '#history-content h3', + '#graphs-content h2', + '#graphs-content h3', + ].join(', ')) + const toc = document.getElementById('toc-list') + const usedIds = new Set() + let sectionId = '' + let sectionItem = null + let subsectionList = null + + headings.forEach(heading => { + const baseId = slugify(heading.textContent) + const candidateId = heading.tagName === 'H3' && sectionId + ? `${sectionId}-${baseId}` + : baseId + let id = candidateId + let suffix = 2 + while (usedIds.has(id)) { + id = `${candidateId}-${suffix}` + suffix += 1 + } + usedIds.add(id) + heading.id = id + + const item = document.createElement('li') + const link = document.createElement('a') + link.href = `#${id}` + link.textContent = heading.textContent + item.appendChild(link) + + if (heading.tagName === 'H2') { + sectionId = id + sectionItem = item + subsectionList = null + toc.appendChild(item) + return + } + + if (!subsectionList) { + subsectionList = document.createElement('ul') + sectionItem.appendChild(subsectionList) + } + subsectionList.appendChild(item) + }) +} + +async function init() { + await Promise.all([loadReadme(), loadHistory()]) + buildTableOfContents() + document.querySelectorAll('table').forEach(table => table.classList.add('sortable')) +} + +init() diff --git a/docs/index-local.html b/docs/index-local.html new file mode 100644 index 000000000..93947ef89 --- /dev/null +++ b/docs/index-local.html @@ -0,0 +1,17 @@ + + + + + + Local Worker Version Report + + +

Opening the worker version report with local data…

+ + + + From b96c389f1022687e1b46eb2a221f2a15722f6edf Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Tue, 8 Sep 2026 22:29:54 -0700 Subject: [PATCH 04/74] Document worker report data sources --- WorkerVersions/README.md | 12 ++++++++++++ audit-worker-versions/report.go | 4 ++++ audit-worker-versions/report_test.go | 5 +++++ 3 files changed, 21 insertions(+) diff --git a/WorkerVersions/README.md b/WorkerVersions/README.md index b4aa887f5..39999e79c 100644 --- a/WorkerVersions/README.md +++ b/WorkerVersions/README.md @@ -9,6 +9,8 @@ Total: `424` Count by version: +_Source: version information parsed from the log artifact produced when each worker claims an intentionally malformed probe task. The task is expected to fail with a malformed-payload exception; known worker implementations and versions are identified from their distinct log output._ + | Version | Count | | :--- | ---: | | 100.0.1 | 6 | @@ -32,6 +34,8 @@ Count by version: Count by image: +_Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images; unknown means no supported image reference was found._ + | Version | Count | | :--- | ---: | | projects/taskcluster-imaging/global/images/gw-fxci-gcp-l1-2404-amd64-headless-googlecompute-2026-09-09 | 128 | @@ -488,6 +492,8 @@ Total: `2` Count by version: +_Source: version information parsed from the log artifact produced when each worker claims an intentionally malformed probe task. The task is expected to fail with a malformed-payload exception; known worker implementations and versions are identified from their distinct log output._ + | Version | Count | | :--- | ---: | | 38.0.5 | 1 | @@ -496,6 +502,8 @@ Count by version: Count by image: +_Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images; unknown means no supported image reference was found._ + | Version | Count | | :--- | ---: | | ami-03e4f8db63254ce7e,ami-0a6e926238859761c,ami-0b5dd0bbb670ec80e | 1 | @@ -568,6 +576,8 @@ Total: `4` Count by image: +_Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images; unknown means no supported image reference was found._ + | Version | Count | | :--- | ---: | | projects/taskcluster-imaging/global/images/generic-2204-wayland-vm-gcp-googlecompute-2023-09-22t17-39-37z | 2 | @@ -589,6 +599,8 @@ Total: `30` Count by image: +_Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images; unknown means no supported image reference was found._ + | Version | Count | | :--- | ---: | | unknown | 20 | diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index edf8b075d..10dd88177 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -47,6 +47,8 @@ Total: ` + "`" + `{{ .Count }}` + "`" + ` {{ if gt (len .Versions) 1 }} ### Count by version +_Source: version information parsed from the log artifact produced when each worker claims an intentionally malformed probe task. The task is expected to fail with a malformed-payload exception; known worker implementations and versions are identified from their distinct log output._ + | Version | Count | | :--- | ---: | {{ range .Versions -}} @@ -56,6 +58,8 @@ Total: ` + "`" + `{{ .Count }}` + "`" + ` {{ if gt (len .Images) 1 }} ### Count by image +_Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images; unknown means no supported image reference was found._ + | Version | Count | | :--- | ---: | {{ range .Images -}} diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index 9f8f7aecb..ea5d1747d 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -20,17 +20,22 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { WorkerPoolID: "example/pool", Implementation: "generic-worker", Version: "1.2.3", + Imageset: "image-one", Details: map[string]string{"revision": "1234567890"}, }, { WorkerPoolID: "example/other-pool", Implementation: "generic-worker", Version: "2.0.0", + Imageset: "image-two", Details: map[string]string{"revision": "1234567890"}, }} got := renderReadme(workers) for _, want := range []string{ "### Count by version", + "intentionally malformed probe task", + "expected to fail with a malformed-payload exception", + "live Worker Manager launch configuration", "### Worker pools", "[**example/pool**](https://firefox-ci-tc.services.mozilla.com/provisioners/example/worker-types/pool?sortBy=Last%20Active&sortDirection=desc)", } { From 6d212e5b360fecb86a70c77f42b2513655a915bf Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Tue, 8 Sep 2026 23:08:19 -0700 Subject: [PATCH 05/74] Clarify worker report totals and implementations --- audit-worker-versions/report.go | 2 +- audit-worker-versions/report_test.go | 1 + docs/app.js | 10 +++++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index 10dd88177..4dc6db4cd 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -43,7 +43,7 @@ const readmeTpl = ` {{ .Description }} {{ end }} -Total: ` + "`" + `{{ .Count }}` + "`" + ` +Total worker pools: ` + "`" + `{{ .Count }}` + "`" + ` {{ if gt (len .Versions) 1 }} ### Count by version diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index ea5d1747d..002897460 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -32,6 +32,7 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { got := renderReadme(workers) for _, want := range []string{ + "Total worker pools: `2`", "### Count by version", "intentionally malformed probe task", "expected to fail with a malformed-payload exception", diff --git a/docs/app.js b/docs/app.js index 275f113b6..1699e3aa7 100644 --- a/docs/app.js +++ b/docs/app.js @@ -21,7 +21,7 @@ async function loadReadme() { function renderHistoryTables(data) { const dates = Object.keys(data).sort((a, b) => b.localeCompare(a)) - const tableByDate = (title, property) => { + const tableByDate = (title, property, description = '') => { let columns = [] dates.forEach(date => { const values = data[date][property] @@ -31,7 +31,7 @@ function renderHistoryTables(data) { }) columns = columns.sort((a, b) => a.localeCompare(b)) - let table = `

${title}

` + let table = `

${title}

${description}` table += `${columns.map(key => ``).join('')}` dates.forEach(date => { table += ` @@ -43,7 +43,11 @@ function renderHistoryTables(data) { } let html = '

History

' - html += tableByDate('Worker implementations', 'implementations') + html += tableByDate( + 'Worker implementations', + 'implementations', + '

Worker implementation is inferred from distinctive content in the log artifact produced by the intentionally malformed probe task. Docker Worker identifies the legacy docker-worker implementation; it does not indicate that a generic-worker pool is configured to accept docker-worker-style payloads.

', + ) html += tableByDate('Worker versions', 'versions') document.getElementById('history-content').innerHTML = html } From d10a9c87c2df656957589d457d96a2c740b14c44 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Tue, 8 Sep 2026 23:34:45 -0700 Subject: [PATCH 06/74] Improve graph tooltips and responsive layout --- docs/app.js | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/app.js b/docs/app.js index 1699e3aa7..6be6e7308 100644 --- a/docs/app.js +++ b/docs/app.js @@ -54,26 +54,30 @@ function renderHistoryTables(data) { function renderGraphs(data) { const dates = Object.keys(data).sort() + const lineTrace = (name, y) => ({ + x: dates, + y, + name, + type: 'scatter', + mode: 'lines', + hovertemplate: '%{fullData.name}
Date: %{x}
Worker pools: %{y}', + }) const implementationTraces = [ ['Generic worker', 'generic-worker'], ['Docker worker', 'docker-worker'], ['Unknown', ''], - ].map(([name, implementation]) => ({ - x: dates, - y: dates.map(date => data[date].implementations[implementation] || 0), + ].map(([name, implementation]) => lineTrace( name, - type: 'line', - })) + dates.map(date => data[date].implementations[implementation] || 0), + )) const allVersions = [...new Set(dates.map(date => Object.keys(data[date].versions)).flat())].sort() - const versionTraces = allVersions.map(version => ({ - x: dates, - y: dates.map(date => data[date].versions[version] || 0), - name: version, - type: 'line', - })) + const versionTraces = allVersions.map(version => lineTrace( + version || 'unknown', + dates.map(date => data[date].versions[version] || 0), + )) - const layout = { barmode: 'group' } + const layout = { hovermode: 'closest' } Plotly.newPlot('graph-implementations', implementationTraces, { ...layout, title: 'Worker Implementations', From bf28acbcbfe67e0f18823078f86b0d291ba4a731 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Tue, 8 Sep 2026 23:48:31 -0700 Subject: [PATCH 07/74] Clarify worker report scope and graph units --- WorkerVersions/README.md | 2 ++ audit-worker-versions/report.go | 2 ++ audit-worker-versions/report_test.go | 1 + docs/app.js | 24 ++++++++++++++++-------- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/WorkerVersions/README.md b/WorkerVersions/README.md index 39999e79c..cc58044a1 100644 --- a/WorkerVersions/README.md +++ b/WorkerVersions/README.md @@ -2,6 +2,8 @@ # Worker Pool Versions +This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots. Worker implementation and version are inferred from the failure log produced when each pool is given an intentionally malformed probe task; image and capacity metadata come from Worker Manager. Summary counts represent worker pools, not individual workers or tasks. + ## Generic Worker diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index 4dc6db4cd..59b14cf7c 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -79,6 +79,8 @@ _Source: image references in each pool's live Worker Manager launch configuratio # Worker Pool Versions +This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots. Worker implementation and version are inferred from the failure log produced when each pool is given an intentionally malformed probe task; image and capacity metadata come from Worker Manager. Summary counts represent worker pools, not individual workers or tasks. + {{ range . }} {{ template "row" . }} {{ end }} diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index 002897460..92c84c840 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -32,6 +32,7 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { got := renderReadme(workers) for _, want := range []string{ + "This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots.", "Total worker pools: `2`", "### Count by version", "intentionally malformed probe task", diff --git a/docs/app.js b/docs/app.js index 6be6e7308..bf9ac1bd8 100644 --- a/docs/app.js +++ b/docs/app.js @@ -77,15 +77,23 @@ function renderGraphs(data) { dates.map(date => data[date].versions[version] || 0), )) - const layout = { hovermode: 'closest' } - Plotly.newPlot('graph-implementations', implementationTraces, { - ...layout, - title: 'Worker Implementations', - }) - Plotly.newPlot('graph-versions', versionTraces, { - ...layout, - title: 'Worker Versions', + const layout = (title, legendTitle) => ({ + hovermode: 'closest', + title, + legend: { title: { text: legendTitle } }, + xaxis: { title: { text: 'Snapshot date' } }, + yaxis: { title: { text: 'Worker pools' }, rangemode: 'tozero' }, }) + Plotly.newPlot( + 'graph-implementations', + implementationTraces, + layout('Worker pools by implementation', 'Implementation'), + ) + Plotly.newPlot( + 'graph-versions', + versionTraces, + layout('Worker pools by version', 'Version'), + ) } async function loadHistory() { From a52ee4c0a377d3479293ac58248e2b8d50a7c63f Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 00:45:13 -0700 Subject: [PATCH 08/74] Report configured worker pool capacity --- audit-worker-versions/main.go | 143 +++++++++++++++++++-------- audit-worker-versions/main_test.go | 100 +++++++++++++++++++ audit-worker-versions/report.go | 69 +++++++++---- audit-worker-versions/report_test.go | 24 +++-- 4 files changed, 271 insertions(+), 65 deletions(-) create mode 100644 audit-worker-versions/main_test.go diff --git a/audit-worker-versions/main.go b/audit-worker-versions/main.go index aabba5cef..ecdf8bf0e 100644 --- a/audit-worker-versions/main.go +++ b/audit-worker-versions/main.go @@ -27,15 +27,23 @@ type ( ) type WorkerInfo struct { - WorkerPoolID string - Implementation string - Version string - Details map[string]string - hasNoArtifacts bool - isUnknown bool - Imageset string - TotalWorkers int - TotalCapacity int + WorkerPoolID string + Implementation string + Version string + Details map[string]string + hasNoArtifacts bool + isUnknown bool + Imageset string + ConfiguredMinCapacity *int `json:",omitempty"` + ConfiguredMaxCapacity *int `json:",omitempty"` + CapacityPerWorker *int `json:",omitempty"` + ConfiguredMinWorkers *int `json:",omitempty"` + ConfiguredMaxWorkers *int `json:",omitempty"` + WorkerManagerLookupError string `json:",omitempty"` + // Retained only to recognize snapshots created before configured capacity + // fields were collected. New snapshots leave these fields unset. + LegacyTotalWorkers *int `json:"TotalWorkers,omitempty"` + LegacyTotalCapacity *int `json:"TotalCapacity,omitempty"` } func (w *WorkerInfo) String() string { @@ -64,37 +72,50 @@ func (w *WorkerInfo) String() string { return strings.Trim(info, " ") } -func GetImageset(wp *tcworkermanager.WorkerPoolFullDefinition) string { - var p = wp.ProviderID - if p == "test-provisioner" || p == "no-provisioning-nope" || p == "dummy-test-provisioner" || p == "test-dummy-provisioner" { - return "unknown" - } - - type LaunchConfig struct { - // AWS - LaunchConfig struct { - ImageId string - } `json:"launchConfig"` +type workerPoolLaunchConfig struct { + // AWS + LaunchConfig struct { + ImageId string + } `json:"launchConfig"` + + // GCP + Disks []struct { + InitializeParams struct { + SourceImage string `json:"sourceImage"` + } `json:"initializeParams"` + } `json:"disks"` + + // Azure + StorageProfile struct { + ImageReference struct { + Id string `json:"id"` + } `json:"imageReference"` + } `json:"storageProfile"` + + // Older Worker Manager configurations stored this directly on the + // launch config. Newer configurations nest it under workerManager. + CapacityPerInstance *int `json:"capacityPerInstance"` + WorkerManager struct { + CapacityPerInstance *int `json:"capacityPerInstance"` + } `json:"workerManager"` +} - // GCP - Disks []struct { - InitializeParams struct { - SourceImage string `json:"sourceImage"` - } `json:"initializeParams"` - } `json:"disks"` +type workerPoolConfig struct { + MinCapacity *int `json:"minCapacity"` + MaxCapacity *int `json:"maxCapacity"` + LaunchConfigs []workerPoolLaunchConfig `json:"launchConfigs"` +} - // Azure - StorageProfile struct { - ImageReference struct { - Id string `json:"id"` - } `json:"imageReference"` - } `json:"storageProfile"` - } - type Config struct { - LaunchConfigs []LaunchConfig `json:"launchConfigs"` - } - var cfg Config +func parseWorkerPoolConfig(wp *tcworkermanager.WorkerPoolFullDefinition) (workerPoolConfig, error) { + var cfg workerPoolConfig if err := json.Unmarshal(wp.Config, &cfg); err != nil { + return workerPoolConfig{}, err + } + return cfg, nil +} + +func getImageset(providerID string, cfg workerPoolConfig) string { + if providerID == "test-provisioner" || providerID == "no-provisioning-nope" || providerID == "dummy-test-provisioner" || providerID == "test-dummy-provisioner" { return "unknown" } @@ -120,6 +141,46 @@ func GetImageset(wp *tcworkermanager.WorkerPoolFullDefinition) string { return sortedImages } +func ceilDivide(value, divisor int) int { + return (value + divisor - 1) / divisor +} + +func enrichWorkerInfo(workerInfo *WorkerInfo, wp *tcworkermanager.WorkerPoolFullDefinition) { + cfg, err := parseWorkerPoolConfig(wp) + if err != nil { + workerInfo.Imageset = "unknown" + workerInfo.WorkerManagerLookupError = "Could not parse Worker Manager configuration: " + err.Error() + return + } + + workerInfo.Imageset = getImageset(wp.ProviderID, cfg) + workerInfo.ConfiguredMinCapacity = cfg.MinCapacity + workerInfo.ConfiguredMaxCapacity = cfg.MaxCapacity + + capacities := map[int]struct{}{} + for _, launchConfig := range cfg.LaunchConfigs { + capacity := launchConfig.CapacityPerInstance + if launchConfig.WorkerManager.CapacityPerInstance != nil { + capacity = launchConfig.WorkerManager.CapacityPerInstance + } + if capacity != nil && *capacity > 0 { + capacities[*capacity] = struct{}{} + } + } + + if len(capacities) != 1 || cfg.MinCapacity == nil || cfg.MaxCapacity == nil { + return + } + for capacity := range capacities { + capacityPerWorker := capacity + minWorkers := ceilDivide(*cfg.MinCapacity, capacity) + maxWorkers := ceilDivide(*cfg.MaxCapacity, capacity) + workerInfo.CapacityPerWorker = &capacityPerWorker + workerInfo.ConfiguredMinWorkers = &minWorkers + workerInfo.ConfiguredMaxWorkers = &maxWorkers + } +} + var ( // set during build with `-ldflags "-X main.revision=$(git rev-parse HEAD)"` revision string = "" @@ -303,12 +364,10 @@ func inspect(queue *tcqueue.Queue, taskIDs []string) { workerPool, err := workermanager.WorkerPool(workerPoolID) if err != nil { fmt.Println("Could not fetch workerPool " + workerPoolID) + workerInfo.Imageset = "unknown" + workerInfo.WorkerManagerLookupError = err.Error() } else { - workerInfo.Imageset = GetImageset(workerPool) - workerInfo.TotalWorkers = int(workerPool.RunningCount) + int(workerPool.StoppedCount) + - int(workerPool.StoppingCount) + int(workerPool.RequestedCount) - workerInfo.TotalCapacity = int(workerPool.RunningCapacity) + int(workerPool.StoppedCapacity) + - int(workerPool.StoppingCapacity) + int(workerPool.RequestedCapacity) + enrichWorkerInfo(&workerInfo, workerPool) } filename := filepath.Join(outputDir, FilenameEscape(workerPoolID)) WriteFile(filename, append([]byte(workerInfo.String()), '\n')) diff --git a/audit-worker-versions/main_test.go b/audit-worker-versions/main_test.go new file mode 100644 index 000000000..3b4b88f9f --- /dev/null +++ b/audit-worker-versions/main_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/taskcluster/taskcluster/v48/clients/client-go/tcworkermanager" +) + +func intPointer(value int) *int { + return &value +} + +func TestEnrichWorkerInfoWithConfiguredCapacity(t *testing.T) { + tests := []struct { + name string + config string + slots int + }{ + { + name: "current nested capacity", + config: `{"minCapacity":1,"maxCapacity":16,"launchConfigs":[{"workerManager":{"capacityPerInstance":8}}]}`, + slots: 8, + }, + { + name: "legacy direct capacity", + config: `{"minCapacity":1,"maxCapacity":16,"launchConfigs":[{"capacityPerInstance":8}]}`, + slots: 8, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + worker := WorkerInfo{} + pool := tcworkermanager.WorkerPoolFullDefinition{ + Config: json.RawMessage(test.config), + } + + enrichWorkerInfo(&worker, &pool) + + for name, result := range map[string]struct { + got *int + want int + }{ + "minimum capacity": {worker.ConfiguredMinCapacity, 1}, + "maximum capacity": {worker.ConfiguredMaxCapacity, 16}, + "slots per worker": {worker.CapacityPerWorker, test.slots}, + "minimum workers": {worker.ConfiguredMinWorkers, 1}, + "maximum workers": {worker.ConfiguredMaxWorkers, 2}, + } { + if result.got == nil || *result.got != result.want { + t.Errorf("%s = %v, want %d", name, result.got, result.want) + } + } + + encoded, err := json.Marshal(worker) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"ConfiguredMinCapacity":1`, + `"ConfiguredMaxCapacity":16`, + `"CapacityPerWorker":8`, + `"ConfiguredMinWorkers":1`, + `"ConfiguredMaxWorkers":2`, + } { + if !strings.Contains(string(encoded), want) { + t.Errorf("serialized worker does not contain %s", want) + } + } + if strings.Contains(string(encoded), "TotalWorkers") || strings.Contains(string(encoded), "TotalCapacity") { + t.Error("new worker snapshot contains legacy totals") + } + }) + } +} + +func TestEnrichWorkerInfoDoesNotDeriveWorkersForMixedCapacities(t *testing.T) { + worker := WorkerInfo{} + pool := tcworkermanager.WorkerPoolFullDefinition{ + Config: json.RawMessage(`{ + "minCapacity": 0, + "maxCapacity": 16, + "launchConfigs": [ + {"workerManager":{"capacityPerInstance":2}}, + {"workerManager":{"capacityPerInstance":8}} + ] + }`), + } + + enrichWorkerInfo(&worker, &pool) + + if worker.ConfiguredMinCapacity == nil || worker.ConfiguredMaxCapacity == nil { + t.Fatal("configured capacity bounds were not preserved") + } + if worker.CapacityPerWorker != nil || worker.ConfiguredMinWorkers != nil || worker.ConfiguredMaxWorkers != nil { + t.Fatal("worker counts were derived for heterogeneous launch capacities") + } +} diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index 59b14cf7c..af060f7df 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/json" + "fmt" "net/url" "os" "path/filepath" @@ -17,13 +18,14 @@ type count struct { } type reportSection struct { - Title string - Description string - Count int - Versions []count - Images []count - Filtered []WorkerInfo - FullColumns bool + Title string + Description string + Count int + Versions []count + Images []count + Filtered []WorkerInfo + FullColumns bool + HasLegacyTotals bool } func (w WorkerInfo) WorkerPoolURL() string { @@ -36,6 +38,31 @@ func (w WorkerInfo) WorkerPoolURL() string { "?sortBy=Last%20Active&sortDirection=desc" } +func formatConfiguredRange(minimum, maximum *int) string { + if minimum == nil || maximum == nil { + return "—" + } + if *minimum == *maximum { + return fmt.Sprintf("%d", *minimum) + } + return fmt.Sprintf("%d–%d", *minimum, *maximum) +} + +func (w WorkerInfo) ConfiguredWorkerRange() string { + return formatConfiguredRange(w.ConfiguredMinWorkers, w.ConfiguredMaxWorkers) +} + +func (w WorkerInfo) ConfiguredCapacityRange() string { + return formatConfiguredRange(w.ConfiguredMinCapacity, w.ConfiguredMaxCapacity) +} + +func (w WorkerInfo) CapacityPerWorkerValue() string { + if w.CapacityPerWorker == nil { + return "—" + } + return fmt.Sprintf("%d", *w.CapacityPerWorker) +} + const readmeTpl = ` {{- define "row" -}} ## {{ .Title }} @@ -69,10 +96,15 @@ _Source: image references in each pool's live Worker Manager launch configuratio {{if .Count }} ### Worker pools -| Worker Pool | Implementation | Version {{ if .FullColumns }}| Engine | Revision | OS | Arch | GO {{ end }}| Total Workers | Total Capacity | -| --- | --- | --- {{ if .FullColumns }}| --- | --- | --- | --- | --- {{ end }}| ---: | ---: | +_Configured capacity is the pool's autoscaling range in concurrent task slots. The configured worker range is derived from those bounds and the slots per worker, rounding up. An em dash means the configuration is unavailable or a worker count cannot be derived._ +{{ if .HasLegacyTotals }} +_Configured values were not collected in this snapshot. Legacy totals included stopped worker records and are intentionally not displayed._ +{{ end }} + +| Worker Pool | Implementation | Version {{ if .FullColumns }}| Engine | Revision | OS | Arch | GO {{ end }}| Configured Workers | Configured Capacity | Slots per Worker | +| --- | --- | --- {{ if .FullColumns }}| --- | --- | --- | --- | --- {{ end }}| ---: | ---: | ---: | {{ range .Filtered -}} -| [**{{ .WorkerPoolID }}**]({{ .WorkerPoolURL }}) | {{ .Implementation }} | {{ or .Version .Details.error }} {{ if $.FullColumns }}| {{ or .Details.engine "-" }} | {{ or (slice .Details.revision 0 10) "-" }} | {{ or .Details.os "-" }} | {{ or .Details.arch "-" }} | {{ or .Details.go "-" }} {{ end }}| {{ .TotalWorkers }} | {{ .TotalCapacity }} | +| [**{{ .WorkerPoolID }}**]({{ .WorkerPoolURL }}) | {{ .Implementation }} | {{ or .Version .Details.error }} {{ if $.FullColumns }}| {{ or .Details.engine "-" }} | {{ or (slice .Details.revision 0 10) "-" }} | {{ or .Details.os "-" }} | {{ or .Details.arch "-" }} | {{ or .Details.go "-" }} {{ end }}| {{ .ConfiguredWorkerRange }} | {{ .ConfiguredCapacityRange }} | {{ .CapacityPerWorkerValue }} | {{end}} {{- end -}} {{end}} @@ -110,12 +142,14 @@ func generateReadmeSection(title, description string, workers []WorkerInfo, filt filtered := make([]WorkerInfo, 0) versions := make(map[string]int) imagesets := make(map[string]int) + hasLegacyTotals := false for _, worker := range workers { if filter(worker) { filtered = append(filtered, worker) versions[worker.Version]++ imagesets[worker.Imageset]++ + hasLegacyTotals = hasLegacyTotals || worker.LegacyTotalWorkers != nil || worker.LegacyTotalCapacity != nil } } @@ -124,13 +158,14 @@ func generateReadmeSection(title, description string, workers []WorkerInfo, filt }) return reportSection{ - Title: title, - Description: description, - Count: len(filtered), - Versions: sortedCounts(versions), - Images: sortedCounts(imagesets), - Filtered: filtered, - FullColumns: title == "Generic Worker", + Title: title, + Description: description, + Count: len(filtered), + Versions: sortedCounts(versions), + Images: sortedCounts(imagesets), + Filtered: filtered, + FullColumns: title == "Generic Worker", + HasLegacyTotals: hasLegacyTotals, } } diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index 92c84c840..fc61cb05b 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -17,11 +17,16 @@ func TestWorkerPoolURL(t *testing.T) { func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { workers := []WorkerInfo{{ - WorkerPoolID: "example/pool", - Implementation: "generic-worker", - Version: "1.2.3", - Imageset: "image-one", - Details: map[string]string{"revision": "1234567890"}, + WorkerPoolID: "example/pool", + Implementation: "generic-worker", + Version: "1.2.3", + Imageset: "image-one", + Details: map[string]string{"revision": "1234567890"}, + ConfiguredMinCapacity: intPointer(1), + ConfiguredMaxCapacity: intPointer(16), + CapacityPerWorker: intPointer(8), + ConfiguredMinWorkers: intPointer(1), + ConfiguredMaxWorkers: intPointer(2), }, { WorkerPoolID: "example/other-pool", Implementation: "generic-worker", @@ -39,6 +44,9 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { "expected to fail with a malformed-payload exception", "live Worker Manager launch configuration", "### Worker pools", + "Configured Workers | Configured Capacity | Slots per Worker", + "pool's autoscaling range in concurrent task slots", + "| 1–2 | 1–16 | 8 |", "[**example/pool**](https://firefox-ci-tc.services.mozilla.com/provisioners/example/worker-types/pool?sortBy=Last%20Active&sortDirection=desc)", } { if !strings.Contains(got, want) { @@ -72,7 +80,7 @@ func TestRenderReadmeExplainsIncompleteProbesInline(t *testing.T) { func TestReadSnapshotRestoresRenderingState(t *testing.T) { filename := filepath.Join(t.TempDir(), "workers.json") data := `[ - {"WorkerPoolID":"one/pool","Details":{"error":"No artifacts found"}}, + {"WorkerPoolID":"one/pool","Details":{"error":"No artifacts found"},"TotalWorkers":42,"TotalCapacity":84}, {"WorkerPoolID":"two/pool","Details":{"error":"Version not determined; task not (yet) claimed"}} ]` if err := os.WriteFile(filename, []byte(data), 0644); err != nil { @@ -86,6 +94,10 @@ func TestReadSnapshotRestoresRenderingState(t *testing.T) { if !workers[0].hasNoArtifacts { t.Error("no-artifacts state was not restored") } + if workers[0].LegacyTotalWorkers == nil || *workers[0].LegacyTotalWorkers != 42 || + workers[0].LegacyTotalCapacity == nil || *workers[0].LegacyTotalCapacity != 84 { + t.Error("legacy totals were not recognized") + } if !workers[1].isUnknown { t.Error("unknown-version state was not restored") } From 9eb97d1f002142eee5206c1003cfe8b68a56e10e Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 09:23:11 -0700 Subject: [PATCH 09/74] Naturally sort worker version counts --- audit-worker-versions/report.go | 57 +++++++++++++++++++++++++++- audit-worker-versions/report_test.go | 18 +++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index af060f7df..ee4662164 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -138,6 +138,61 @@ func sortedCounts(values map[string]int) []count { return counts } +func naturalLess(left, right string) bool { + for leftIndex, rightIndex := 0, 0; leftIndex < len(left) && rightIndex < len(right); { + leftDigit := left[leftIndex] >= '0' && left[leftIndex] <= '9' + rightDigit := right[rightIndex] >= '0' && right[rightIndex] <= '9' + if leftDigit && rightDigit { + leftEnd, rightEnd := leftIndex, rightIndex + for leftEnd < len(left) && left[leftEnd] >= '0' && left[leftEnd] <= '9' { + leftEnd++ + } + for rightEnd < len(right) && right[rightEnd] >= '0' && right[rightEnd] <= '9' { + rightEnd++ + } + + leftSignificant, rightSignificant := leftIndex, rightIndex + for leftSignificant < leftEnd-1 && left[leftSignificant] == '0' { + leftSignificant++ + } + for rightSignificant < rightEnd-1 && right[rightSignificant] == '0' { + rightSignificant++ + } + + leftLength := leftEnd - leftSignificant + rightLength := rightEnd - rightSignificant + if leftLength != rightLength { + return leftLength < rightLength + } + if leftNumber, rightNumber := left[leftSignificant:leftEnd], right[rightSignificant:rightEnd]; leftNumber != rightNumber { + return leftNumber < rightNumber + } + if leftRunLength, rightRunLength := leftEnd-leftIndex, rightEnd-rightIndex; leftRunLength != rightRunLength { + return leftRunLength < rightRunLength + } + + leftIndex, rightIndex = leftEnd, rightEnd + continue + } + + if left[leftIndex] != right[rightIndex] { + return left[leftIndex] < right[rightIndex] + } + leftIndex++ + rightIndex++ + } + + return len(left) < len(right) +} + +func sortedVersionCounts(values map[string]int) []count { + counts := sortedCounts(values) + sort.Slice(counts, func(i, j int) bool { + return naturalLess(counts[i].Key, counts[j].Key) + }) + return counts +} + func generateReadmeSection(title, description string, workers []WorkerInfo, filter func(WorkerInfo) bool) reportSection { filtered := make([]WorkerInfo, 0) versions := make(map[string]int) @@ -161,7 +216,7 @@ func generateReadmeSection(title, description string, workers []WorkerInfo, filt Title: title, Description: description, Count: len(filtered), - Versions: sortedCounts(versions), + Versions: sortedVersionCounts(versions), Images: sortedCounts(imagesets), Filtered: filtered, FullColumns: title == "Generic Worker", diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index fc61cb05b..9f570c013 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -15,6 +15,24 @@ func TestWorkerPoolURL(t *testing.T) { } } +func TestSortedVersionCountsUsesNaturalOrder(t *testing.T) { + versions := map[string]int{ + "100.0.1": 1, + "108.0.0": 1, + "9.10.0": 1, + "9.2.0": 1, + "99.2.0": 1, + } + want := []string{"9.2.0", "9.10.0", "99.2.0", "100.0.1", "108.0.0"} + + got := sortedVersionCounts(versions) + for i := range want { + if got[i].Key != want[i] { + t.Fatalf("version %d = %q, want %q", i, got[i].Key, want[i]) + } + } +} + func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { workers := []WorkerInfo{{ WorkerPoolID: "example/pool", From b5ab5294f16015131be409bca1508769d0a537ed Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 11:17:03 -0700 Subject: [PATCH 10/74] Record worker snapshot collection times --- audit-worker-versions/main.go | 36 +++++++++++++------ audit-worker-versions/report.go | 54 ++++++++++++++++++++-------- audit-worker-versions/report_test.go | 43 ++++++++++++++++++++-- build-docs-history.sh | 3 +- 4 files changed, 107 insertions(+), 29 deletions(-) diff --git a/audit-worker-versions/main.go b/audit-worker-versions/main.go index ecdf8bf0e..c5cc9cf16 100644 --- a/audit-worker-versions/main.go +++ b/audit-worker-versions/main.go @@ -46,6 +46,13 @@ type WorkerInfo struct { LegacyTotalCapacity *int `json:"TotalCapacity,omitempty"` } +type WorkerSnapshot struct { + GeneratedAt time.Time `json:"generatedAt"` + ProbeStartedAt time.Time `json:"probeStartedAt"` + TaskGroupID string `json:"taskGroupId"` + Workers []WorkerInfo `json:"workers"` +} + func (w *WorkerInfo) String() string { revision := "" engine := "" @@ -226,9 +233,9 @@ func main() { if len(os.Args) < 3 || len(os.Args) > 4 { log.Fatal("Usage: audit-worker-versions render INPUT_JSON [OUTPUT_MARKDOWN]") } - workers, err := readSnapshot(os.Args[2]) + snapshot, err := readSnapshot(os.Args[2]) fatalOnError(err) - contents := renderReadme(workers) + contents := renderReadme(snapshot) if len(os.Args) == 4 { WriteFile(os.Args[3], []byte(contents)) } else { @@ -252,7 +259,7 @@ func main() { if len(taskIDs) == 0 { log.Fatalf("No tasks with taskGroupId %q", taskGroupID) } - inspect(queue, taskIDs) + inspect(queue, taskGroupID, taskIDs) default: log.Fatalf("Expected zero or one program arguments, but have %v: %q", len(os.Args)-1, os.Args[1:]) } @@ -345,8 +352,10 @@ func createTasks(queue *tcqueue.Queue, taskGroupID string) { log.Printf("Task group sealed at: %v", tg.Sealed) } -func inspect(queue *tcqueue.Queue, taskIDs []string) { +func inspect(queue *tcqueue.Queue, taskGroupID string, taskIDs []string) { EmptyDirectory(outputDir) + probeTask, err := queue.Task(taskIDs[0]) + fatalOnError(err) workermanager := tcworkermanager.NewFromEnv() wp := workerpool.New(50) workers := make([]WorkerInfo, 0) @@ -382,20 +391,27 @@ func inspect(queue *tcqueue.Queue, taskIDs []string) { workers = append(workers, result.(WorkerInfo)) }) + snapshot := WorkerSnapshot{ + GeneratedAt: time.Now().UTC(), + ProbeStartedAt: time.Time(probeTask.Created).UTC(), + TaskGroupID: taskGroupID, + Workers: workers, + } + fmt.Printf("\nWriting README.md\n") - writeReadme(workers) + writeReadme(snapshot) fmt.Println("Writing workers.json") - writeSnapshot(workers) + writeSnapshot(snapshot) } -func writeSnapshot(workers []WorkerInfo) { +func writeSnapshot(snapshot WorkerSnapshot) { filename := filepath.Join(outputDir, "workers.json") - sort.Slice(workers, func(i, j int) bool { - return strings.Compare(workers[i].WorkerPoolID, workers[j].WorkerPoolID) <= 0 + sort.Slice(snapshot.Workers, func(i, j int) bool { + return strings.Compare(snapshot.Workers[i].WorkerPoolID, snapshot.Workers[j].WorkerPoolID) <= 0 }) - contents, err := json.MarshalIndent(workers, "", " ") + contents, err := json.MarshalIndent(snapshot, "", " ") if err != nil { log.Fatalf("Error:\n%v", err) } diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index ee4662164..86fa15b2c 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -28,6 +28,12 @@ type reportSection struct { HasLegacyTotals bool } +type reportData struct { + GeneratedAt string + ProbeStartedAt string + Sections [5]reportSection +} + func (w WorkerInfo) WorkerPoolURL() string { parts := strings.SplitN(w.WorkerPoolID, "/", 2) if len(parts) != 2 { @@ -113,7 +119,11 @@ _Configured values were not collected in this snapshot. Legacy totals included s This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots. Worker implementation and version are inferred from the failure log produced when each pool is given an intentionally malformed probe task; image and capacity metadata come from Worker Manager. Summary counts represent worker pools, not individual workers or tasks. -{{ range . }} +{{ if .ProbeStartedAt }}Probe run started: **{{ .ProbeStartedAt }}**{{ if .GeneratedAt }} · Results collected: **{{ .GeneratedAt }}**{{ end }} +{{ else if .GeneratedAt }}Results collected: **{{ .GeneratedAt }}** +{{ end }} + +{{ range .Sections }} {{ template "row" . }} {{ end }} ` @@ -224,12 +234,13 @@ func generateReadmeSection(title, description string, workers []WorkerInfo, filt } } -func writeReadme(workers []WorkerInfo) { +func writeReadme(snapshot WorkerSnapshot) { filename := filepath.Join(outputDir, "README.md") - WriteFile(filename, []byte(renderReadme(workers))) + WriteFile(filename, []byte(renderReadme(snapshot))) } -func renderReadme(workers []WorkerInfo) string { +func renderReadme(snapshot WorkerSnapshot) string { + workers := snapshot.Workers sections := [5]reportSection{ generateReadmeSection("Generic Worker", "", workers, func(w WorkerInfo) bool { return w.Implementation == "generic-worker" }), generateReadmeSection("Docker Worker", "", workers, func(w WorkerInfo) bool { return w.Implementation == "docker-worker" }), @@ -238,30 +249,43 @@ func renderReadme(workers []WorkerInfo) string { generateReadmeSection("Version not determined", "These pools did not claim the probe task within two hours, so their worker implementation and version could not be determined.", workers, func(w WorkerInfo) bool { return w.isUnknown }), } - return renderTemplate(sections) + const timestampFormat = "2006-01-02 15:04 UTC" + data := reportData{Sections: sections} + if !snapshot.GeneratedAt.IsZero() { + data.GeneratedAt = snapshot.GeneratedAt.UTC().Format(timestampFormat) + } + if !snapshot.ProbeStartedAt.IsZero() { + data.ProbeStartedAt = snapshot.ProbeStartedAt.UTC().Format(timestampFormat) + } + + return renderTemplate(data) } -func readSnapshot(filename string) ([]WorkerInfo, error) { +func readSnapshot(filename string) (WorkerSnapshot, error) { contents, err := os.ReadFile(filename) if err != nil { - return nil, err + return WorkerSnapshot{}, err } - workers := []WorkerInfo{} - if err := json.Unmarshal(contents, &workers); err != nil { - return nil, err + var snapshot WorkerSnapshot + if strings.HasPrefix(strings.TrimSpace(string(contents)), "[") { + if err := json.Unmarshal(contents, &snapshot.Workers); err != nil { + return WorkerSnapshot{}, err + } + } else if err := json.Unmarshal(contents, &snapshot); err != nil { + return WorkerSnapshot{}, err } // These flags are internal rendering state and are not serialized in the // snapshot. Restore them from the persisted error value for offline renders. - for i := range workers { - switch workers[i].Details["error"] { + for i := range snapshot.Workers { + switch snapshot.Workers[i].Details["error"] { case "No artifacts found": - workers[i].hasNoArtifacts = true + snapshot.Workers[i].hasNoArtifacts = true case "Version not determined; task not (yet) claimed": - workers[i].isUnknown = true + snapshot.Workers[i].isUnknown = true } } - return workers, nil + return snapshot, nil } diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index 9f570c013..9c50b002e 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) func TestWorkerPoolURL(t *testing.T) { @@ -53,9 +54,14 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { Details: map[string]string{"revision": "1234567890"}, }} - got := renderReadme(workers) + got := renderReadme(WorkerSnapshot{ + GeneratedAt: time.Date(2026, time.September, 9, 15, 29, 53, 0, time.UTC), + ProbeStartedAt: time.Date(2026, time.September, 9, 7, 58, 29, 0, time.UTC), + Workers: workers, + }) for _, want := range []string{ "This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots.", + "Probe run started: **2026-09-09 07:58 UTC** · Results collected: **2026-09-09 15:29 UTC**", "Total worker pools: `2`", "### Count by version", "intentionally malformed probe task", @@ -79,7 +85,7 @@ func TestRenderReadmeExplainsIncompleteProbesInline(t *testing.T) { {WorkerPoolID: "example/pending", Details: map[string]string{"error": "Version not determined; task not (yet) claimed"}, isUnknown: true}, } - got := renderReadme(workers) + got := renderReadme(WorkerSnapshot{Workers: workers}) for _, want := range []string{ "## No artifacts found\n", "did not publish `public/logs/live_backing.log`", @@ -105,10 +111,11 @@ func TestReadSnapshotRestoresRenderingState(t *testing.T) { t.Fatal(err) } - workers, err := readSnapshot(filename) + snapshot, err := readSnapshot(filename) if err != nil { t.Fatal(err) } + workers := snapshot.Workers if !workers[0].hasNoArtifacts { t.Error("no-artifacts state was not restored") } @@ -120,3 +127,33 @@ func TestReadSnapshotRestoresRenderingState(t *testing.T) { t.Error("unknown-version state was not restored") } } + +func TestReadSnapshotReadsMetadata(t *testing.T) { + filename := filepath.Join(t.TempDir(), "workers.json") + data := `{ + "generatedAt":"2026-09-09T15:29:53Z", + "probeStartedAt":"2026-09-09T07:58:29.422Z", + "taskGroupId":"AnhEjBL2SYuUedNBvjgsWA", + "workers":[{"WorkerPoolID":"one/pool","Details":{}}] + }` + if err := os.WriteFile(filename, []byte(data), 0644); err != nil { + t.Fatal(err) + } + + snapshot, err := readSnapshot(filename) + if err != nil { + t.Fatal(err) + } + if got := snapshot.GeneratedAt.Format(time.RFC3339); got != "2026-09-09T15:29:53Z" { + t.Errorf("generated time = %q", got) + } + if got := snapshot.ProbeStartedAt.Format(time.RFC3339Nano); got != "2026-09-09T07:58:29.422Z" { + t.Errorf("probe start time = %q", got) + } + if snapshot.TaskGroupID != "AnhEjBL2SYuUedNBvjgsWA" { + t.Errorf("task group ID = %q", snapshot.TaskGroupID) + } + if len(snapshot.Workers) != 1 || snapshot.Workers[0].WorkerPoolID != "one/pool" { + t.Errorf("workers = %#v", snapshot.Workers) + } +} diff --git a/build-docs-history.sh b/build-docs-history.sh index 6b20ee867..892cdf337 100755 --- a/build-docs-history.sh +++ b/build-docs-history.sh @@ -12,6 +12,7 @@ process.exit(1) } const runStats = (data) => { + const workers = Array.isArray(data) ? data : data.workers const implementations = {} const versions = {} const imagesets = {} @@ -20,7 +21,7 @@ // indistinguishable once merged into a single bucket const versionsByImplementation = {} const inc = (dict, key) => dict[key] = (dict[key] || 0) + 1 - data.forEach(worker => { + workers.forEach(worker => { inc(implementations, worker.Implementation) inc(versions, worker.Version) inc(imagesets, worker.Imageset) From 21f2b122799c285dcba8a3bf1d7141f216055e65 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 11:21:24 -0700 Subject: [PATCH 11/74] feat: support Quick deployment for worker report --- docs/app.js | 10 ++++++++-- docs/index-local.html | 4 ++-- index.html | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 index.html diff --git a/docs/app.js b/docs/app.js index bf9ac1bd8..c35482ae3 100644 --- a/docs/app.js +++ b/docs/app.js @@ -2,8 +2,14 @@ let basePath = String(window.location.pathname) if (!basePath.includes('-history')) basePath = '/mozilla-history/' -const localPreview = new URLSearchParams(window.location.search).has('local') -const readmeUrl = localPreview +const query = new URLSearchParams(window.location.search) +const bundledReport = query.has('local') || ( + !query.has('remote') && ( + window.location.hostname.endsWith('.quick.mozilla.cloud') || + window.location.pathname.includes('/docs/') + ) +) +const readmeUrl = bundledReport ? new URL('../WorkerVersions/README.md', window.location.href) : `https://raw.githubusercontent.com/taskcluster${basePath}master/WorkerVersions/README.md` const historyUrl = new URL('history.json', window.location.href) diff --git a/docs/index-local.html b/docs/index-local.html index 93947ef89..24891023a 100644 --- a/docs/index-local.html +++ b/docs/index-local.html @@ -8,10 +8,10 @@

Opening the worker version report with local data…

diff --git a/index.html b/index.html new file mode 100644 index 000000000..99643eec5 --- /dev/null +++ b/index.html @@ -0,0 +1,19 @@ + + + + + + + + Firefox CI Worker Pool Versions + + + +

Opening the Firefox CI worker pool version report…

+ + + From f3928aae73db83036d59def892bb3ba2d0363b9b Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 11:34:09 -0700 Subject: [PATCH 12/74] fix: sort worker report values by type --- docs/app.js | 4 ++- docs/table-sort.mjs | 56 ++++++++++++++++++++++++++++++++++++++++ docs/table-sort.test.mjs | 39 ++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 docs/table-sort.mjs create mode 100644 docs/table-sort.test.mjs diff --git a/docs/app.js b/docs/app.js index c35482ae3..79cb87230 100644 --- a/docs/app.js +++ b/docs/app.js @@ -1,3 +1,5 @@ +import { decorateSortableTables } from './table-sort.mjs' + // This works for both the Mozilla and community history GitHub Pages sites. let basePath = String(window.location.pathname) if (!basePath.includes('-history')) basePath = '/mozilla-history/' @@ -171,7 +173,7 @@ function buildTableOfContents() { async function init() { await Promise.all([loadReadme(), loadHistory()]) buildTableOfContents() - document.querySelectorAll('table').forEach(table => table.classList.add('sortable')) + decorateSortableTables() } init() diff --git a/docs/table-sort.mjs b/docs/table-sort.mjs new file mode 100644 index 000000000..badc0881f --- /dev/null +++ b/docs/table-sort.mjs @@ -0,0 +1,56 @@ +const SORT_KEY_WIDTH = 16 + +function paddedInteger(value) { + return value.replace(/^0+(?=\d)/, '').padStart(SORT_KEY_WIDTH, '0') +} + +export function versionSortKey(value) { + const version = value.trim() + if (!/^\d+(?:\.\d+)*$/.test(version)) return '' + + // The leading letter makes sortable compare this as text rather than + // parseFloat-ing only the first dotted component. + return `v${version.split('.').map(paddedInteger).join('.')}` +} + +export function rangeSortKey(value) { + const match = value.trim().match(/^(\d+)(?:\s*[–-]\s*(\d+))?$/) + if (!match) return '' + + const minimum = paddedInteger(match[1]) + const maximum = paddedInteger(match[2] || match[1]) + return `r${maximum}.${minimum}` +} + +export function numericSortKey(value) { + const number = value.trim() + if (!/^\d+$/.test(number)) return '' + return `n${paddedInteger(number)}` +} + +function setColumnSortKeys(table, columnIndex, keyForValue) { + table.querySelectorAll('tbody tr').forEach(row => { + const cell = row.cells[columnIndex] + if (cell) cell.dataset.sort = keyForValue(cell.textContent) + }) +} + +export function decorateSortableTable(table) { + table.classList.add('sortable', 'n-last') + + table.querySelectorAll('thead th').forEach((header, columnIndex) => { + const title = header.textContent.trim() + if (title === 'Version') { + setColumnSortKeys(table, columnIndex, versionSortKey) + } else if (title === 'Configured Workers' || title === 'Configured Capacity') { + setColumnSortKeys(table, columnIndex, rangeSortKey) + header.title = 'Sorted by maximum, then minimum' + } else if (title === 'Slots per Worker') { + setColumnSortKeys(table, columnIndex, numericSortKey) + } + }) +} + +export function decorateSortableTables(root = document) { + root.querySelectorAll('table').forEach(decorateSortableTable) +} diff --git a/docs/table-sort.test.mjs b/docs/table-sort.test.mjs new file mode 100644 index 000000000..cc54b37af --- /dev/null +++ b/docs/table-sort.test.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { numericSortKey, rangeSortKey, versionSortKey } from './table-sort.mjs' + +test('version sort keys order dotted numeric components naturally', () => { + const versions = ['108.0.0', '9.10.0', '100.0.1', '9.2.0', '99.2.0'] + assert.deepEqual( + versions.sort((left, right) => versionSortKey(left).localeCompare(versionSortKey(right))), + ['9.2.0', '9.10.0', '99.2.0', '100.0.1', '108.0.0'], + ) +}) + +test('non-version values receive an empty nulls-last sort key', () => { + assert.equal(versionSortKey('No artifacts found'), '') + assert.equal(versionSortKey('—'), '') +}) + +test('range sort keys order by maximum and then minimum', () => { + const ranges = ['0–100', '1–4', '0–10', '4', '0–4'] + assert.deepEqual( + ranges.sort((left, right) => rangeSortKey(left).localeCompare(rangeSortKey(right))), + ['0–4', '1–4', '4', '0–10', '0–100'], + ) +}) + +test('missing ranges receive an empty nulls-last sort key', () => { + assert.equal(rangeSortKey('—'), '') + assert.equal(rangeSortKey('unknown'), '') +}) + +test('numeric sort keys order slots numerically and identify missing values', () => { + const slots = ['16', '2', '128', '1'] + assert.deepEqual( + slots.sort((left, right) => numericSortKey(left).localeCompare(numericSortKey(right))), + ['1', '2', '16', '128'], + ) + assert.equal(numericSortKey('—'), '') +}) From 09416467f8387e96a03ef9deb49573117e1ff5b4 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 11:40:26 -0700 Subject: [PATCH 13/74] fix: serve report sorter as JavaScript --- docs/app.js | 2 +- docs/package.json | 3 +++ docs/{table-sort.mjs => table-sort.js} | 0 docs/table-sort.test.mjs | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 docs/package.json rename docs/{table-sort.mjs => table-sort.js} (100%) diff --git a/docs/app.js b/docs/app.js index 79cb87230..fa87d866e 100644 --- a/docs/app.js +++ b/docs/app.js @@ -1,4 +1,4 @@ -import { decorateSortableTables } from './table-sort.mjs' +import { decorateSortableTables } from './table-sort.js' // This works for both the Mozilla and community history GitHub Pages sites. let basePath = String(window.location.pathname) diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 000000000..3dbc1ca59 --- /dev/null +++ b/docs/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/docs/table-sort.mjs b/docs/table-sort.js similarity index 100% rename from docs/table-sort.mjs rename to docs/table-sort.js diff --git a/docs/table-sort.test.mjs b/docs/table-sort.test.mjs index cc54b37af..d30403f4f 100644 --- a/docs/table-sort.test.mjs +++ b/docs/table-sort.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { numericSortKey, rangeSortKey, versionSortKey } from './table-sort.mjs' +import { numericSortKey, rangeSortKey, versionSortKey } from './table-sort.js' test('version sort keys order dotted numeric components naturally', () => { const versions = ['108.0.0', '9.10.0', '100.0.1', '9.2.0', '99.2.0'] From 004d9d70624abdda64bfa718152045f8035e6d00 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 13:02:34 -0700 Subject: [PATCH 14/74] chore: add Quick staging deploy script --- deploy-prod.sh | 6 ++++++ deploy-staging | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100755 deploy-prod.sh create mode 100755 deploy-staging diff --git a/deploy-prod.sh b/deploy-prod.sh new file mode 100755 index 000000000..16cfb2708 --- /dev/null +++ b/deploy-prod.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd -- "$(dirname -- "$0")" +quick deploy . diff --git a/deploy-staging b/deploy-staging new file mode 100755 index 000000000..e1cf111a9 --- /dev/null +++ b/deploy-staging @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd -- "$(dirname -- "$0")" + +production_url=$(quick url) +production_host=${production_url#*://} +production_host=${production_host%%/*} + +case "$production_host" in + *.quick.mozilla.cloud) + production_name=${production_host%.quick.mozilla.cloud} + ;; + *) + echo "Could not derive a site name from Quick URL: $production_url" >&2 + exit 1 + ;; +esac + +staging_name="${production_name}-staging" +if [[ ${#staging_name} -gt 63 ]]; then + echo "Derived staging site name is longer than Quick's 63-character limit: $staging_name" >&2 + exit 1 +fi + +echo "Deploying staging site: https://${staging_name}.quick.mozilla.cloud/" +quick deploy . "$staging_name" From 70a4dc70320391ad6613e96cd22bdf7ea2b9d4f8 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 13:09:30 -0700 Subject: [PATCH 15/74] chore: standardize deploy script names --- deploy-prod.sh => deploy-prod | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename deploy-prod.sh => deploy-prod (100%) diff --git a/deploy-prod.sh b/deploy-prod similarity index 100% rename from deploy-prod.sh rename to deploy-prod From 50158d4f1646560ae6c59db50b8b304c2d8b26f9 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 13:12:04 -0700 Subject: [PATCH 16/74] chore: clarify Quick deploy script names --- deploy-prod => deploy-quick-prod | 0 deploy-staging => deploy-quick-staging | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename deploy-prod => deploy-quick-prod (100%) rename deploy-staging => deploy-quick-staging (100%) diff --git a/deploy-prod b/deploy-quick-prod similarity index 100% rename from deploy-prod rename to deploy-quick-prod diff --git a/deploy-staging b/deploy-quick-staging similarity index 100% rename from deploy-staging rename to deploy-quick-staging From cf30373ce810d4d2f096671a40cf1ecd50a58a12 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 13:22:50 -0700 Subject: [PATCH 17/74] feat: add local data refresh workflow --- README.md | 40 ++++++- audit-worker-versions/main.go | 86 ++++++++++++++ audit-worker-versions/main_test.go | 35 ++++++ fetch_and_generate.py | 177 +++++++++++++++++++++++++++++ 4 files changed, 333 insertions(+), 5 deletions(-) create mode 100755 fetch_and_generate.py diff --git a/README.md b/README.md index 5fbcf3121..0d2c4ec21 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,34 @@ mozilla-history This will populate subdirectories `Clients`, `Hooks`, `Roles` and `WorkerPools` of the current directory. +## Refreshing All Local Data + +For a complete local refresh, set Taskcluster credentials and run: + +```sh +export TASKCLUSTER_CLIENT_ID='...' +export TASKCLUSTER_ACCESS_TOKEN='...' +./fetch_and_generate.py +``` + +The script builds temporary copies of the Go tools, refreshes `Clients`, +`Hooks`, `Roles`, and `WorkerPools`, schedules worker-version probes, polls their +sealed task group until every task reaches a terminal state, writes +`WorkerVersions`, and rebuilds `docs/history.json`. It does not pull, commit, +push, or deploy anything. Review the generated changes before committing them. + +Use `--poll-interval SECONDS` to change the 60-second polling interval. If the +script is interrupted after scheduling probes, resume without creating another +group using `--task-group-id TASK_GROUP_ID`. `TASKCLUSTER_ROOT_URL`, +`REPORT_SCHEDULER_ID`, and `REPORT_PREFIX` can be overridden in the environment. + +Worker versions require two phases because Worker Manager configuration does +not expose the implementation and version actually running in each pool. The +first phase schedules an intentionally malformed task on every pool so the +worker identifies itself in its task log. Once all tasks are resolved, the +second phase reads those logs and generates the snapshot. Polling replaces the +older fixed-delay assumption while preserving that probe-and-collect design. + ## Rendering an Existing Worker Snapshot The worker-version report can be regenerated from a saved snapshot without @@ -69,9 +97,11 @@ Then open . The local preview uses `WorkerVersions/README.md` and `docs/history.json` from the checkout. The published page continues to load the current report from GitHub. -## Automating the Process +## Production Automation -You can automate this reporting process by setting up a cron job to execute `run-report.sh` at regular intervals. +The existing `run-reports.sh` and `audit.sh` scripts implement the repository's +production publishing workflow. They include git and production-site behavior; +use `fetch_and_generate.py` for local refreshes. ### Prerequisites - Valid Taskcluster credentials must be set in the environment variables: @@ -79,13 +109,13 @@ You can automate this reporting process by setting up a cron job to execute `run - `TASKCLUSTER_ACCESS_TOKEN` ### How it works -1. `run-report.sh` executes `audit.sh` +1. `run-reports.sh` executes `audit.sh` 2. `audit.sh` schedules tasks for each worker pool to extract worker implementation details from logs -3. Results are stored in the `WorkerPools` directory +3. Results are stored in the `WorkerVersions` directory 4. `mozilla-history` stores Taskcluster configurations in their respective directories: - Hooks definitions in `Hooks/` - Roles definitions in `Roles/` - - WorkerPool definitions in `WorkerPool/` + - Worker Pool definitions in `WorkerPools/` 5. `build-docs-history.sh` collects every past revision of `WorkerVersions/workers.json` into `docs/history.json`, which feeds the two GitHub Pages views: - `docs/index.html` — the current report plus the full history tables and graphs diff --git a/audit-worker-versions/main.go b/audit-worker-versions/main.go index c5cc9cf16..a46b79c45 100644 --- a/audit-worker-versions/main.go +++ b/audit-worker-versions/main.go @@ -53,6 +53,38 @@ type WorkerSnapshot struct { Workers []WorkerInfo `json:"workers"` } +type taskGroupProgress struct { + Total int + Terminal int + States map[string]int +} + +func (progress taskGroupProgress) complete() bool { + return progress.Total > 0 && progress.Terminal == progress.Total +} + +func isTerminalTaskState(state string) bool { + switch state { + case "completed", "failed", "exception": + return true + default: + return false + } +} + +func summarizeTaskGroup(tasks []tcqueue.TaskDefinitionAndStatus) taskGroupProgress { + progress := taskGroupProgress{States: map[string]int{}} + for _, task := range tasks { + state := task.Status.State + progress.Total++ + progress.States[state]++ + if isTerminalTaskState(state) { + progress.Terminal++ + } + } + return progress +} + func (w *WorkerInfo) String() string { revision := "" engine := "" @@ -222,6 +254,7 @@ func WriteFile(path string, content []byte) { // Call with no arguments -> New task group generated // Call with one argument (taskGroupID) -> Report generated for previously created task group +// Call with "status taskGroupID" -> Task group progress reported // // Expected workflow for this tool is to: // 1. Run without arguments to generate probing tasks and get taskGroupId @@ -245,6 +278,18 @@ func main() { } queue := tcqueue.NewFromEnv() + if len(os.Args) >= 2 && os.Args[1] == "status" { + if len(os.Args) != 3 { + log.Fatal("Usage: audit-worker-versions status TASK_GROUP_ID") + } + progress, err := taskGroupStatus(queue, os.Args[2]) + fatalOnError(err) + printTaskGroupProgress(os.Args[2], progress) + if !progress.complete() { + os.Exit(3) + } + return + } switch len(os.Args) { case 1: @@ -265,6 +310,47 @@ func main() { } } +func taskGroupStatus(queue *tcqueue.Queue, taskGroupID string) (taskGroupProgress, error) { + tasks := []tcqueue.TaskDefinitionAndStatus{} + continuationToken := "" + for { + response, err := queue.ListTaskGroup(taskGroupID, continuationToken, "") + if err != nil { + return taskGroupProgress{}, err + } + tasks = append(tasks, response.Tasks...) + continuationToken = response.ContinuationToken + if continuationToken == "" { + break + } + } + if len(tasks) == 0 { + return taskGroupProgress{}, fmt.Errorf("no tasks with taskGroupId %q", taskGroupID) + } + return summarizeTaskGroup(tasks), nil +} + +func printTaskGroupProgress(taskGroupID string, progress taskGroupProgress) { + stateOrder := []string{"completed", "failed", "exception", "running", "pending", "unscheduled"} + states := make([]string, 0, len(progress.States)) + knownStates := map[string]bool{} + for _, state := range stateOrder { + knownStates[state] = true + if count := progress.States[state]; count > 0 { + states = append(states, fmt.Sprintf("%s=%d", state, count)) + } + } + extraStates := []string{} + for state, count := range progress.States { + if !knownStates[state] { + extraStates = append(extraStates, fmt.Sprintf("%s=%d", state, count)) + } + } + sort.Strings(extraStates) + states = append(states, extraStates...) + fmt.Printf("Task group %s: %d/%d terminal (%s)\n", taskGroupID, progress.Terminal, progress.Total, strings.Join(states, ", ")) +} + func createTasks(queue *tcqueue.Queue, taskGroupID string) { if revision != "" { log.Printf("%v built from revision %v", os.Args[0], revision) diff --git a/audit-worker-versions/main_test.go b/audit-worker-versions/main_test.go index 3b4b88f9f..d80e8d2e0 100644 --- a/audit-worker-versions/main_test.go +++ b/audit-worker-versions/main_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/taskcluster/taskcluster/v48/clients/client-go/tcqueue" "github.com/taskcluster/taskcluster/v48/clients/client-go/tcworkermanager" ) @@ -98,3 +99,37 @@ func TestEnrichWorkerInfoDoesNotDeriveWorkersForMixedCapacities(t *testing.T) { t.Fatal("worker counts were derived for heterogeneous launch capacities") } } + +func TestSummarizeTaskGroup(t *testing.T) { + tasks := []tcqueue.TaskDefinitionAndStatus{ + {Status: tcqueue.TaskStatusStructure{State: "completed"}}, + {Status: tcqueue.TaskStatusStructure{State: "failed"}}, + {Status: tcqueue.TaskStatusStructure{State: "exception"}}, + {Status: tcqueue.TaskStatusStructure{State: "running"}}, + {Status: tcqueue.TaskStatusStructure{State: "pending"}}, + } + + progress := summarizeTaskGroup(tasks) + + if progress.Total != 5 || progress.Terminal != 3 { + t.Fatalf("progress = %#v, want 3 of 5 terminal", progress) + } + if progress.complete() { + t.Fatal("incomplete task group was reported complete") + } +} + +func TestTaskGroupComplete(t *testing.T) { + progress := summarizeTaskGroup([]tcqueue.TaskDefinitionAndStatus{ + {Status: tcqueue.TaskStatusStructure{State: "completed"}}, + {Status: tcqueue.TaskStatusStructure{State: "failed"}}, + {Status: tcqueue.TaskStatusStructure{State: "exception"}}, + }) + + if !progress.complete() { + t.Fatalf("progress = %#v, want complete", progress) + } + if (taskGroupProgress{}).complete() { + t.Fatal("empty task group was reported complete") + } +} diff --git a/fetch_and_generate.py b/fetch_and_generate.py new file mode 100755 index 000000000..0ae23925b --- /dev/null +++ b/fetch_and_generate.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Refresh all local Taskcluster snapshots and generated report data.""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +REPOSITORY = Path(__file__).resolve().parent +TASK_GROUP_PATTERN = re.compile(r"^Task Group ID: (\S+)$") +WAITING_EXIT_CODE = 3 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Refresh local Taskcluster configuration snapshots, probe worker versions, " + "wait for every probe to resolve, and generate report data." + ) + ) + parser.add_argument( + "--poll-interval", + type=positive_number, + default=60.0, + metavar="SECONDS", + help="seconds between task-group status checks (default: 60)", + ) + parser.add_argument( + "--task-group-id", + help="resume polling and collecting an existing probe group instead of creating one", + ) + return parser.parse_args() + + +def positive_number(raw: str) -> float: + value = float(raw) + if value <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return value + + +def run(command: list[str], *, env: dict[str, str]) -> None: + print(f"\n+ {' '.join(command)}", flush=True) + subprocess.run(command, cwd=REPOSITORY, env=env, check=True) + + +def create_probe_group(auditor: str, *, env: dict[str, str]) -> str: + print(f"\n+ {auditor}", flush=True) + process = subprocess.Popen( + [auditor], + cwd=REPOSITORY, + env=env, + stdout=subprocess.PIPE, + text=True, + bufsize=1, + ) + task_group_id = None + assert process.stdout is not None + for line in process.stdout: + print(line, end="", flush=True) + match = TASK_GROUP_PATTERN.match(line.rstrip("\n")) + if match: + task_group_id = match.group(1) + + return_code = process.wait() + if return_code != 0: + raise subprocess.CalledProcessError(return_code, [auditor]) + if task_group_id is None: + raise RuntimeError("auditor did not report a task group ID") + return task_group_id + + +def wait_for_probe_group( + auditor: str, + task_group_id: str, + poll_interval: float, + *, + env: dict[str, str], +) -> None: + while True: + result = subprocess.run( + [auditor, "status", task_group_id], + cwd=REPOSITORY, + env=env, + check=False, + ) + if result.returncode == 0: + return + if result.returncode != WAITING_EXIT_CODE: + raise subprocess.CalledProcessError( + result.returncode, [auditor, "status", task_group_id] + ) + time.sleep(poll_interval) + + +def taskcluster_environment() -> dict[str, str]: + env = os.environ.copy() + env.setdefault( + "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com/" + ) + env.setdefault("REPORT_SCHEDULER_ID", "smoketest") + env.setdefault( + "REPORT_PREFIX", + "https://github.com/taskcluster/mozilla-history/blob/master/WorkerVersions/", + ) + return env + + +def require_probe_credentials(env: dict[str, str]) -> None: + missing = [ + name + for name in ("TASKCLUSTER_CLIENT_ID", "TASKCLUSTER_ACCESS_TOKEN") + if not env.get(name) + ] + if missing: + names = ", ".join(missing) + raise RuntimeError(f"missing required environment variable(s): {names}") + + +def main() -> int: + args = parse_args() + env = taskcluster_environment() + if args.task_group_id is None: + require_probe_credentials(env) + + with tempfile.TemporaryDirectory(prefix="mozilla-history-") as temp_dir: + snapshot_tool = str(Path(temp_dir) / "mozilla-history") + auditor = str(Path(temp_dir) / "audit-worker-versions") + + run( + ["go", "build", "-buildvcs=false", "-o", snapshot_tool, "."], + env=env, + ) + run( + [ + "go", + "build", + "-buildvcs=false", + "-o", + auditor, + "./audit-worker-versions", + ], + env=env, + ) + + run([snapshot_tool], env=env) + + task_group_id = args.task_group_id + if task_group_id is None: + task_group_id = create_probe_group(auditor, env=env) + print(f"\nPolling Taskcluster task group {task_group_id}...", flush=True) + wait_for_probe_group(auditor, task_group_id, args.poll_interval, env=env) + run([auditor, task_group_id], env=env) + + run([str(REPOSITORY / "build-docs-history.sh")], env=env) + print( + "\nLocal refresh complete. Review the generated files before committing them.", + flush=True, + ) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print("\nInterrupted.", file=sys.stderr) + sys.exit(130) + except (OSError, RuntimeError, subprocess.CalledProcessError) as error: + print(f"\nError: {error}", file=sys.stderr) + sys.exit(1) From b2d4b75fe1bdfaf9b3066ae4a94976abc63c1f38 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 14:19:25 -0700 Subject: [PATCH 18/74] feat: load Taskcluster credentials from token file --- README.md | 9 +++-- fetch_and_generate.py | 41 ++++++++++++++++++++++- test_fetch_and_generate.py | 67 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 test_fetch_and_generate.py diff --git a/README.md b/README.md index 0d2c4ec21..2983d9429 100644 --- a/README.md +++ b/README.md @@ -48,14 +48,17 @@ of the current directory. ## Refreshing All Local Data -For a complete local refresh, set Taskcluster credentials and run: +For a complete local refresh, run: ```sh -export TASKCLUSTER_CLIENT_ID='...' -export TASKCLUSTER_ACCESS_TOKEN='...' ./fetch_and_generate.py ``` +When `~/.tc_token` exists, the script loads its JSON `clientId` and +`accessToken` fields. Otherwise, it uses `TASKCLUSTER_CLIENT_ID` and +`TASKCLUSTER_ACCESS_TOKEN` from the environment. The token file takes +precedence so stale exported credentials cannot silently override it. + The script builds temporary copies of the Go tools, refreshes `Clients`, `Hooks`, `Roles`, and `WorkerPools`, schedules worker-version probes, polls their sealed task group until every task reaches a terminal state, writes diff --git a/fetch_and_generate.py b/fetch_and_generate.py index 0ae23925b..4985a53ec 100755 --- a/fetch_and_generate.py +++ b/fetch_and_generate.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import json import os import re import subprocess @@ -15,6 +16,7 @@ REPOSITORY = Path(__file__).resolve().parent TASK_GROUP_PATTERN = re.compile(r"^Task Group ID: (\S+)$") WAITING_EXIT_CODE = 3 +TOKEN_FILE = Path.home() / ".tc_token" def parse_args() -> argparse.Namespace: @@ -99,8 +101,45 @@ def wait_for_probe_group( time.sleep(poll_interval) +def load_taskcluster_token(env: dict[str, str], token_file: Path = TOKEN_FILE) -> bool: + credential_keys = { + "TASKCLUSTER_CLIENT_ID": "clientId", + "TASKCLUSTER_ACCESS_TOKEN": "accessToken", + } + if not token_file.exists(): + return False + + try: + token = json.loads(token_file.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise RuntimeError( + f"could not read Taskcluster token file {token_file}: {error}" + ) from error + if not isinstance(token, dict): + raise TypeError( + f"Taskcluster token file {token_file} must contain a JSON object" + ) + + invalid = [ + token_name + for token_name in credential_keys.values() + if not isinstance(token.get(token_name), str) or not token[token_name] + ] + if invalid: + names = ", ".join(invalid) + raise RuntimeError( + f"Taskcluster token file {token_file} has missing or invalid field(s): {names}" + ) + + for environment_name, token_name in credential_keys.items(): + env[environment_name] = token[token_name] + return True + + def taskcluster_environment() -> dict[str, str]: env = os.environ.copy() + if load_taskcluster_token(env): + print(f"Using Taskcluster credentials from {TOKEN_FILE}", flush=True) env.setdefault( "TASKCLUSTER_ROOT_URL", "https://firefox-ci-tc.services.mozilla.com/" ) @@ -172,6 +211,6 @@ def main() -> int: except KeyboardInterrupt: print("\nInterrupted.", file=sys.stderr) sys.exit(130) - except (OSError, RuntimeError, subprocess.CalledProcessError) as error: + except (OSError, RuntimeError, TypeError, subprocess.CalledProcessError) as error: print(f"\nError: {error}", file=sys.stderr) sys.exit(1) diff --git a/test_fetch_and_generate.py b/test_fetch_and_generate.py new file mode 100644 index 000000000..60bcfc76d --- /dev/null +++ b/test_fetch_and_generate.py @@ -0,0 +1,67 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from fetch_and_generate import load_taskcluster_token + + +class LoadTaskclusterTokenTests(unittest.TestCase): + def test_loads_missing_credentials_from_json(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + token_file = Path(temp_dir) / ".tc_token" + token_file.write_text( + json.dumps({"clientId": "file-client", "accessToken": "file-token"}), + encoding="utf-8", + ) + env: dict[str, str] = {} + + loaded = load_taskcluster_token(env, token_file) + + self.assertTrue(loaded) + self.assertEqual(env["TASKCLUSTER_CLIENT_ID"], "file-client") + self.assertEqual(env["TASKCLUSTER_ACCESS_TOKEN"], "file-token") + + def test_token_file_takes_precedence_over_environment(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + token_file = Path(temp_dir) / ".tc_token" + token_file.write_text( + json.dumps({"clientId": "file-client", "accessToken": "file-token"}), + encoding="utf-8", + ) + env = { + "TASKCLUSTER_CLIENT_ID": "environment-client", + "TASKCLUSTER_ACCESS_TOKEN": "environment-token", + } + + loaded = load_taskcluster_token(env, token_file) + + self.assertTrue(loaded) + self.assertEqual(env["TASKCLUSTER_CLIENT_ID"], "file-client") + self.assertEqual(env["TASKCLUSTER_ACCESS_TOKEN"], "file-token") + + def test_environment_is_unchanged_when_token_file_is_absent(self) -> None: + env = { + "TASKCLUSTER_CLIENT_ID": "environment-client", + "TASKCLUSTER_ACCESS_TOKEN": "environment-token", + } + + loaded = load_taskcluster_token(env, Path("/does/not/exist/.tc_token")) + + self.assertFalse(loaded) + self.assertEqual(env["TASKCLUSTER_CLIENT_ID"], "environment-client") + self.assertEqual(env["TASKCLUSTER_ACCESS_TOKEN"], "environment-token") + + def test_rejects_malformed_json_when_credentials_are_needed(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + token_file = Path(temp_dir) / ".tc_token" + token_file.write_text("not json", encoding="utf-8") + + with self.assertRaisesRegex( + RuntimeError, "could not read Taskcluster token file" + ): + load_taskcluster_token({}, token_file) + + +if __name__ == "__main__": + unittest.main() From fbdd132481dc42a7d192248003b66a1832dbc03b Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 14:37:12 -0700 Subject: [PATCH 19/74] fix: skip committed history in local refresh --- README.md | 9 +++++++-- fetch_and_generate.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2983d9429..c1d1a1d72 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,13 @@ precedence so stale exported credentials cannot silently override it. The script builds temporary copies of the Go tools, refreshes `Clients`, `Hooks`, `Roles`, and `WorkerPools`, schedules worker-version probes, polls their sealed task group until every task reaches a terminal state, writes -`WorkerVersions`, and rebuilds `docs/history.json`. It does not pull, commit, -push, or deploy anything. Review the generated changes before committing them. +`WorkerVersions`, and stops. It does not pull, commit, push, or deploy anything. +Review the generated changes before committing them. + +The local refresh intentionally does not rebuild `docs/history.json`. History is +derived exclusively from worker snapshots already committed to Git, so it cannot +include the snapshot generated by the current run. The production publishing +workflow rebuilds it after the snapshot is committed. Use `--poll-interval SECONDS` to change the 60-second polling interval. If the script is interrupted after scheduling probes, resume without creating another diff --git a/fetch_and_generate.py b/fetch_and_generate.py index 4985a53ec..b3cfb4026 100755 --- a/fetch_and_generate.py +++ b/fetch_and_generate.py @@ -197,7 +197,6 @@ def main() -> int: wait_for_probe_group(auditor, task_group_id, args.poll_interval, env=env) run([auditor, task_group_id], env=env) - run([str(REPOSITORY / "build-docs-history.sh")], env=env) print( "\nLocal refresh complete. Review the generated files before committing them.", flush=True, From 70ff0e0d08781f9e8cb8def8ec6051987ccf0115 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 14:38:07 -0700 Subject: [PATCH 20/74] chore: add local preview helper --- README.md | 5 +++-- run_local.sh | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100755 run_local.sh diff --git a/README.md b/README.md index c1d1a1d72..92343b76d 100644 --- a/README.md +++ b/README.md @@ -98,12 +98,13 @@ Omit the output path to print the generated Markdown to standard output. To preview the generated report in the website, serve the repository root: ```sh -python3 -m http.server 8000 +./run_local.sh ``` Then open . The local preview uses `WorkerVersions/README.md` and `docs/history.json` from the checkout. The -published page continues to load the current report from GitHub. +published page continues to load the current report from GitHub. Pass a port as +the first argument to override the default, for example `./run_local.sh 8080`. ## Production Automation diff --git a/run_local.sh b/run_local.sh new file mode 100755 index 000000000..ba23b73d8 --- /dev/null +++ b/run_local.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -eu + +port="${1:-8000}" +repo_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +case "$port" in + *[!0-9]*|'') + echo "Usage: $0 [port]" >&2 + exit 2 + ;; +esac + +echo "Serving the local report at http://localhost:${port}/docs/index-local.html" +exec python3 -m http.server "$port" --directory "$repo_dir" From 3180c725c8c49d26eacbe190ae0ff12f2a0ada24 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 15:19:41 -0700 Subject: [PATCH 21/74] fix: distinguish standalone and Azure worker images --- audit-worker-versions/main.go | 88 ++++++++++++++++++++-- audit-worker-versions/main_test.go | 105 +++++++++++++++++++++++++++ audit-worker-versions/report.go | 71 +++++++++++++++++- audit-worker-versions/report_test.go | 55 ++++++++++++++ 4 files changed, 308 insertions(+), 11 deletions(-) diff --git a/audit-worker-versions/main.go b/audit-worker-versions/main.go index a46b79c45..e82aa2f19 100644 --- a/audit-worker-versions/main.go +++ b/audit-worker-versions/main.go @@ -34,6 +34,8 @@ type WorkerInfo struct { hasNoArtifacts bool isUnknown bool Imageset string + ImageStatus string `json:",omitempty"` + ProviderID string `json:",omitempty"` ConfiguredMinCapacity *int `json:",omitempty"` ConfiguredMaxCapacity *int `json:",omitempty"` CapacityPerWorker *int `json:",omitempty"` @@ -46,6 +48,20 @@ type WorkerInfo struct { LegacyTotalCapacity *int `json:"TotalCapacity,omitempty"` } +const ( + imageStatusKnown = "known" + imageStatusNotApplicable = "not-applicable" + imageStatusNotDetermined = "not-determined" + imageStatusUnavailable = "unavailable" + standaloneProviderID = "standalone" + standaloneWorkerState = "standalone" +) + +type workerManagerClient interface { + WorkerPool(workerPoolID string) (*tcworkermanager.WorkerPoolFullDefinition, error) + ListWorkers(provisionerID, workerType, continuationToken, limit, quarantined, workerState string) (*tcworkermanager.ListWorkersResponse, error) +} + type WorkerSnapshot struct { GeneratedAt time.Time `json:"generatedAt"` ProbeStartedAt time.Time `json:"probeStartedAt"` @@ -130,6 +146,13 @@ type workerPoolLaunchConfig struct { Id string `json:"id"` } `json:"imageReference"` } `json:"storageProfile"` + ArmDeployment struct { + Parameters struct { + ImageID struct { + Value string `json:"value"` + } `json:"imageId"` + } `json:"parameters"` + } `json:"armDeployment"` // Older Worker Manager configurations stored this directly on the // launch config. Newer configurations nest it under workerManager. @@ -165,6 +188,7 @@ func getImageset(providerID string, cfg workerPoolConfig) string { imagesMap[disk.InitializeParams.SourceImage] = struct{}{} } imagesMap[launchCfg.StorageProfile.ImageReference.Id] = struct{}{} + imagesMap[launchCfg.ArmDeployment.Parameters.ImageID.Value] = struct{}{} } // remove empty image name "" delete(imagesMap, "") @@ -185,14 +209,20 @@ func ceilDivide(value, divisor int) int { } func enrichWorkerInfo(workerInfo *WorkerInfo, wp *tcworkermanager.WorkerPoolFullDefinition) { + workerInfo.ProviderID = wp.ProviderID cfg, err := parseWorkerPoolConfig(wp) if err != nil { workerInfo.Imageset = "unknown" + workerInfo.ImageStatus = imageStatusUnavailable workerInfo.WorkerManagerLookupError = "Could not parse Worker Manager configuration: " + err.Error() return } workerInfo.Imageset = getImageset(wp.ProviderID, cfg) + workerInfo.ImageStatus = imageStatusKnown + if workerInfo.Imageset == "unknown" { + workerInfo.ImageStatus = imageStatusNotDetermined + } workerInfo.ConfiguredMinCapacity = cfg.MinCapacity workerInfo.ConfiguredMaxCapacity = cfg.MaxCapacity @@ -220,6 +250,55 @@ func enrichWorkerInfo(workerInfo *WorkerInfo, wp *tcworkermanager.WorkerPoolFull } } +func workerPoolIsStandalone(workermanager workerManagerClient, workerPoolID string) (bool, error) { + parts := strings.SplitN(workerPoolID, "/", 2) + if len(parts) != 2 { + return false, fmt.Errorf("invalid worker pool ID %q", workerPoolID) + } + + foundStandaloneWorker := false + continuationToken := "" + for { + workers, err := workermanager.ListWorkers(parts[0], parts[1], continuationToken, "", "", "") + if err != nil { + return false, err + } + for _, worker := range workers.Workers { + if worker.State == standaloneWorkerState { + foundStandaloneWorker = true + } + } + continuationToken = workers.ContinuationToken + if continuationToken == "" { + break + } + } + return foundStandaloneWorker, nil +} + +func lookupAndEnrichWorkerInfo(workerInfo *WorkerInfo, workermanager workerManagerClient) { + workerPool, lookupErr := workermanager.WorkerPool(workerInfo.WorkerPoolID) + if lookupErr == nil { + enrichWorkerInfo(workerInfo, workerPool) + return + } + + standalone, standaloneErr := workerPoolIsStandalone(workermanager, workerInfo.WorkerPoolID) + if standaloneErr == nil && standalone { + workerInfo.ProviderID = standaloneProviderID + workerInfo.Imageset = "unknown" + workerInfo.ImageStatus = imageStatusNotApplicable + return + } + + workerInfo.Imageset = "unknown" + workerInfo.ImageStatus = imageStatusUnavailable + workerInfo.WorkerManagerLookupError = lookupErr.Error() + if standaloneErr != nil { + workerInfo.WorkerManagerLookupError += "; could not check for standalone workers: " + standaloneErr.Error() + } +} + var ( // set during build with `-ldflags "-X main.revision=$(git rev-parse HEAD)"` revision string = "" @@ -456,14 +535,7 @@ func inspect(queue *tcqueue.Queue, taskGroupID string, taskIDs []string) { panic(err) } workerPoolID, workerInfo := show(queue, statusResponse) - workerPool, err := workermanager.WorkerPool(workerPoolID) - if err != nil { - fmt.Println("Could not fetch workerPool " + workerPoolID) - workerInfo.Imageset = "unknown" - workerInfo.WorkerManagerLookupError = err.Error() - } else { - enrichWorkerInfo(&workerInfo, workerPool) - } + lookupAndEnrichWorkerInfo(&workerInfo, workermanager) filename := filepath.Join(outputDir, FilenameEscape(workerPoolID)) WriteFile(filename, append([]byte(workerInfo.String()), '\n')) fmt.Printf("%-70s %s\n", workerPoolID+":", &workerInfo) diff --git a/audit-worker-versions/main_test.go b/audit-worker-versions/main_test.go index d80e8d2e0..ab8d8c0ab 100644 --- a/audit-worker-versions/main_test.go +++ b/audit-worker-versions/main_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "strings" "testing" @@ -9,6 +10,27 @@ import ( "github.com/taskcluster/taskcluster/v48/clients/client-go/tcworkermanager" ) +type fakeWorkerManager struct { + workerPool *tcworkermanager.WorkerPoolFullDefinition + workerPoolError error + workerPages []*tcworkermanager.ListWorkersResponse + workerError error + listCalls int +} + +func (manager *fakeWorkerManager) WorkerPool(string) (*tcworkermanager.WorkerPoolFullDefinition, error) { + return manager.workerPool, manager.workerPoolError +} + +func (manager *fakeWorkerManager) ListWorkers(string, string, string, string, string, string) (*tcworkermanager.ListWorkersResponse, error) { + if manager.workerError != nil { + return nil, manager.workerError + } + page := manager.workerPages[manager.listCalls] + manager.listCalls++ + return page, nil +} + func intPointer(value int) *int { return &value } @@ -133,3 +155,86 @@ func TestTaskGroupComplete(t *testing.T) { t.Fatal("empty task group was reported complete") } } + +func TestGetImagesetReadsAzureArmDeployment(t *testing.T) { + pool := tcworkermanager.WorkerPoolFullDefinition{ + ProviderID: "azure2", + Config: json.RawMessage(`{ + "launchConfigs": [ + {"armDeployment":{"parameters":{"imageId":{"value":"azure/image/10"}}}}, + {"armDeployment":{"parameters":{"imageId":{"value":"azure/image/2"}}}} + ] + }`), + } + worker := WorkerInfo{} + + enrichWorkerInfo(&worker, &pool) + + if worker.Imageset != "azure/image/10,azure/image/2" { + t.Fatalf("Imageset = %q, want Azure image IDs", worker.Imageset) + } + if worker.ImageStatus != imageStatusKnown || worker.ProviderID != "azure2" { + t.Fatalf("worker metadata = %#v, want known azure2 image", worker) + } +} + +func TestLookupRecognizesPaginatedStandaloneWorkers(t *testing.T) { + manager := &fakeWorkerManager{ + workerPoolError: errors.New("worker pool not found"), + workerPages: []*tcworkermanager.ListWorkersResponse{ + {ContinuationToken: "next", Workers: []tcworkermanager.Worker{{ProviderID: "none", State: standaloneWorkerState}}}, + {Workers: []tcworkermanager.Worker{{ProviderID: "none", State: "running"}}}, + }, + } + worker := WorkerInfo{WorkerPoolID: "releng-hardware/example"} + + lookupAndEnrichWorkerInfo(&worker, manager) + + if worker.ProviderID != standaloneProviderID || worker.ImageStatus != imageStatusNotApplicable { + t.Fatalf("worker metadata = %#v, want standalone/not-applicable", worker) + } + if worker.WorkerManagerLookupError != "" { + t.Fatalf("standalone worker retained lookup error %q", worker.WorkerManagerLookupError) + } + if manager.listCalls != 2 { + t.Fatalf("ListWorkers calls = %d, want 2", manager.listCalls) + } +} + +func TestLookupRetainsUnexpectedFailure(t *testing.T) { + manager := &fakeWorkerManager{ + workerPoolError: errors.New("worker pool unavailable"), + workerError: errors.New("worker list unavailable"), + } + worker := WorkerInfo{WorkerPoolID: "example/pool"} + + lookupAndEnrichWorkerInfo(&worker, manager) + + if worker.ImageStatus != imageStatusUnavailable { + t.Fatalf("ImageStatus = %q, want unavailable", worker.ImageStatus) + } + for _, want := range []string{"worker pool unavailable", "worker list unavailable"} { + if !strings.Contains(worker.WorkerManagerLookupError, want) { + t.Errorf("lookup error %q does not contain %q", worker.WorkerManagerLookupError, want) + } + } +} + +func TestLookupWithoutStandaloneSignalRemainsUnavailable(t *testing.T) { + manager := &fakeWorkerManager{ + workerPoolError: errors.New("worker pool not found"), + workerPages: []*tcworkermanager.ListWorkersResponse{ + {Workers: []tcworkermanager.Worker{{ProviderID: "none", State: "running"}}}, + }, + } + worker := WorkerInfo{WorkerPoolID: "example/pool"} + + lookupAndEnrichWorkerInfo(&worker, manager) + + if worker.ProviderID == standaloneProviderID { + t.Fatal("worker without standalone state was classified as standalone") + } + if worker.ImageStatus != imageStatusUnavailable || worker.WorkerManagerLookupError == "" { + t.Fatalf("worker metadata = %#v, want unavailable with lookup error", worker) + } +} diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index 86fa15b2c..4fa4b6dd6 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -91,9 +91,9 @@ _Source: version information parsed from the log artifact produced when each wor {{ if gt (len .Images) 1 }} ### Count by image -_Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images; unknown means no supported image reference was found._ +_Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images. Standalone pools do not have a Worker Manager-managed image._ -| Version | Count | +| Image | Count | | :--- | ---: | {{ range .Images -}} | {{ .Key }} | {{ .Value }} | @@ -203,6 +203,71 @@ func sortedVersionCounts(values map[string]int) []count { return counts } +func compactAzureImageReference(reference string) string { + parts := strings.Split(strings.Trim(reference, "/"), "/") + if len(parts) < 6 || !strings.EqualFold(parts[0], "subscriptions") { + return reference + } + + providerIsAzureCompute := false + for i := 0; i+1 < len(parts); i++ { + if strings.EqualFold(parts[i], "providers") && strings.EqualFold(parts[i+1], "Microsoft.Compute") { + providerIsAzureCompute = true + break + } + } + if !providerIsAzureCompute { + return reference + } + + for i := 0; i+5 < len(parts); i++ { + if strings.EqualFold(parts[i], "galleries") && + strings.EqualFold(parts[i+2], "images") && + strings.EqualFold(parts[i+4], "versions") { + gallery := parts[i+1] + image := parts[i+3] + version := parts[i+5] + if gallery == image { + return fmt.Sprintf("Azure gallery %s@%s", image, version) + } + return fmt.Sprintf("Azure gallery %s/%s@%s", gallery, image, version) + } + } + + for i := 0; i+1 < len(parts); i++ { + if strings.EqualFold(parts[i], "images") { + return "Azure image " + parts[i+1] + } + } + return reference +} + +func compactImageReferences(imageset string) string { + references := strings.Split(imageset, ",") + for i, reference := range references { + references[i] = compactAzureImageReference(strings.TrimSpace(reference)) + } + return strings.Join(references, ", ") +} + +func imageCountLabel(worker WorkerInfo) string { + switch worker.ImageStatus { + case imageStatusNotApplicable: + return "Not applicable (standalone)" + case imageStatusUnavailable: + return "Configuration unavailable" + case imageStatusNotDetermined: + return "Image not determined" + } + if worker.WorkerManagerLookupError != "" { + return "Configuration unavailable" + } + if worker.Imageset == "" || worker.Imageset == "unknown" { + return "Image not determined" + } + return compactImageReferences(worker.Imageset) +} + func generateReadmeSection(title, description string, workers []WorkerInfo, filter func(WorkerInfo) bool) reportSection { filtered := make([]WorkerInfo, 0) versions := make(map[string]int) @@ -213,7 +278,7 @@ func generateReadmeSection(title, description string, workers []WorkerInfo, filt if filter(worker) { filtered = append(filtered, worker) versions[worker.Version]++ - imagesets[worker.Imageset]++ + imagesets[imageCountLabel(worker)]++ hasLegacyTotals = hasLegacyTotals || worker.LegacyTotalWorkers != nil || worker.LegacyTotalCapacity != nil } } diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index 9c50b002e..8217c179f 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -79,6 +79,61 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { } } +func TestRenderReadmeDistinguishesImageStatuses(t *testing.T) { + details := map[string]string{"revision": "1234567890"} + workers := []WorkerInfo{ + {Implementation: "generic-worker", Imageset: "unknown", ImageStatus: imageStatusNotApplicable, Details: details}, + {Implementation: "generic-worker", Imageset: "unknown", ImageStatus: imageStatusUnavailable, Details: details}, + {Implementation: "generic-worker", Imageset: "unknown", ImageStatus: imageStatusNotDetermined, Details: details}, + {Implementation: "generic-worker", Imageset: "azure/image", ImageStatus: imageStatusKnown, Details: details}, + } + + got := renderReadme(WorkerSnapshot{Workers: workers}) + + for _, want := range []string{ + "| Image | Count |", + "| Not applicable (standalone) | 1 |", + "| Configuration unavailable | 1 |", + "| Image not determined | 1 |", + "| azure/image | 1 |", + } { + if !strings.Contains(got, want) { + t.Errorf("rendered README does not contain %q", want) + } + } +} + +func TestCompactAzureImageReferences(t *testing.T) { + tests := map[string]string{ + "gallery with redundant image name": "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/galleries/win2022/images/win2022/versions/1.0.0", + "gallery with distinct image name": "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/galleries/gallery/images/image/versions/2.0", + "managed image": "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/imageset-worker-eastus", + "non-Azure image": "projects/example/global/images/worker-image", + } + want := map[string]string{ + "gallery with redundant image name": "Azure gallery win2022@1.0.0", + "gallery with distinct image name": "Azure gallery gallery/image@2.0", + "managed image": "Azure image imageset-worker-eastus", + "non-Azure image": "projects/example/global/images/worker-image", + } + + for name, input := range tests { + t.Run(name, func(t *testing.T) { + if got := compactAzureImageReference(input); got != want[name] { + t.Fatalf("compactAzureImageReference() = %q, want %q", got, want[name]) + } + }) + } +} + +func TestCompactImageReferencesPreservesConfiguredSet(t *testing.T) { + input := "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/one,/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/two" + want := "Azure image one, Azure image two" + if got := compactImageReferences(input); got != want { + t.Fatalf("compactImageReferences() = %q, want %q", got, want) + } +} + func TestRenderReadmeExplainsIncompleteProbesInline(t *testing.T) { workers := []WorkerInfo{ {WorkerPoolID: "example/no-artifact", Details: map[string]string{"error": "No artifacts found"}, hasNoArtifacts: true}, From bd0f80e26522a11b3ccbc7bd3547fa930a4c3dc2 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 15:33:26 -0700 Subject: [PATCH 22/74] fix: improve Azure image report labels --- audit-worker-versions/report.go | 64 ++++++++++++++++++++++++++-- audit-worker-versions/report_test.go | 35 +++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index 4fa4b6dd6..5d277e3e2 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "html" "net/url" "os" "path/filepath" @@ -15,6 +16,7 @@ import ( type count struct { Key string Value int + Hover string } type reportSection struct { @@ -96,7 +98,7 @@ _Source: image references in each pool's live Worker Manager launch configuratio | Image | Count | | :--- | ---: | {{ range .Images -}} -| {{ .Key }} | {{ .Value }} | +| {{ if .Hover }}{{ .Key }}{{ else }}{{ .Key }}{{ end }} | {{ .Value }} | {{ end }} {{- end }} {{if .Count }} @@ -242,14 +244,55 @@ func compactAzureImageReference(reference string) string { return reference } +func compactAzureImageSet(references []string) (string, bool) { + regions := make([]string, 0, len(references)) + family := "" + for _, reference := range references { + parts := strings.Split(strings.Trim(reference, "/"), "/") + if len(parts) < 2 || !strings.EqualFold(parts[0], "subscriptions") || + !strings.EqualFold(parts[len(parts)-2], "images") { + return "", false + } + nameParts := strings.Split(parts[len(parts)-1], "-") + if len(nameParts) < 4 || nameParts[0] != "imageset" { + return "", false + } + currentFamily := strings.Join(nameParts[3:], "-") + if family != "" && currentFamily != family { + return "", false + } + family = currentFamily + regions = append(regions, nameParts[2]) + } + if len(regions) < 2 { + return "", false + } + sort.Strings(regions) + return fmt.Sprintf("Azure image set %s (%s)", family, strings.Join(regions, ", ")), true +} + func compactImageReferences(imageset string) string { references := strings.Split(imageset, ",") + for i := range references { + references[i] = strings.TrimSpace(references[i]) + } + if compact, ok := compactAzureImageSet(references); ok { + return compact + } for i, reference := range references { - references[i] = compactAzureImageReference(strings.TrimSpace(reference)) + references[i] = compactAzureImageReference(reference) } return strings.Join(references, ", ") } +func imageHoverTitle(imageset string) string { + references := strings.Split(imageset, ",") + for i, reference := range references { + references[i] = html.EscapeString(strings.TrimSpace(reference)) + } + return strings.Join(references, " ") +} + func imageCountLabel(worker WorkerInfo) string { switch worker.ImageStatus { case imageStatusNotApplicable: @@ -272,13 +315,18 @@ func generateReadmeSection(title, description string, workers []WorkerInfo, filt filtered := make([]WorkerInfo, 0) versions := make(map[string]int) imagesets := make(map[string]int) + imageHovers := make(map[string]string) hasLegacyTotals := false for _, worker := range workers { if filter(worker) { filtered = append(filtered, worker) versions[worker.Version]++ - imagesets[imageCountLabel(worker)]++ + imageLabel := imageCountLabel(worker) + imagesets[imageLabel]++ + if worker.ImageStatus == imageStatusKnown && imageLabel != worker.Imageset { + imageHovers[imageLabel] = worker.Imageset + } hasLegacyTotals = hasLegacyTotals || worker.LegacyTotalWorkers != nil || worker.LegacyTotalCapacity != nil } } @@ -287,12 +335,20 @@ func generateReadmeSection(title, description string, workers []WorkerInfo, filt return strings.Compare(filtered[i].WorkerPoolID, filtered[j].WorkerPoolID) < 0 }) + images := sortedCounts(imagesets) + for i := range images { + if hover := imageHovers[images[i].Key]; hover != "" { + images[i].Key = html.EscapeString(images[i].Key) + images[i].Hover = imageHoverTitle(hover) + } + } + return reportSection{ Title: title, Description: description, Count: len(filtered), Versions: sortedVersionCounts(versions), - Images: sortedCounts(imagesets), + Images: images, Filtered: filtered, FullColumns: title == "Generic Worker", HasLegacyTotals: hasLegacyTotals, diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index 8217c179f..5a2aae4ef 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -134,6 +134,41 @@ func TestCompactImageReferencesPreservesConfiguredSet(t *testing.T) { } } +func TestCompactImageReferencesSummarizesRegionalAzureSet(t *testing.T) { + input := strings.Join([]string{ + "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/imageset-abcdefghijklmnopqrst-westus2-fuzzing", + "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/imageset-zyxwvutsrqponmlkjihg-eastus-fuzzing", + }, ",") + want := "Azure image set fuzzing (eastus, westus2)" + if got := compactImageReferences(input); got != want { + t.Fatalf("compactImageReferences() = %q, want %q", got, want) + } +} + +func TestImageHoverTitleSeparatesImageSetWithLineBreaks(t *testing.T) { + if got := imageHoverTitle("/azure/one,/azure/two"); got != "/azure/one /azure/two" { + t.Fatalf("imageHoverTitle() = %q, want encoded line break", got) + } +} + +func TestRenderReadmeShowsFullAzureImageOnHover(t *testing.T) { + fullImage := "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/galleries/win2022/images/win2022/versions/1.0.0" + workers := []WorkerInfo{ + {Implementation: "generic-worker", Imageset: fullImage, ImageStatus: imageStatusKnown, Details: map[string]string{"revision": "1234567890"}}, + {Implementation: "generic-worker", Imageset: "projects/example/global/images/linux", ImageStatus: imageStatusKnown, Details: map[string]string{"revision": "1234567890"}}, + } + + got := renderReadme(WorkerSnapshot{Workers: workers}) + + want := `Azure gallery win2022@1.0.0` + if !strings.Contains(got, want) { + t.Fatalf("rendered README does not contain Azure hover label %q", want) + } + if strings.Contains(got, ` Date: Wed, 9 Sep 2026 15:54:22 -0700 Subject: [PATCH 23/74] feat: show report generation timestamp --- audit-worker-versions/report.go | 20 +++++++++++++++----- audit-worker-versions/report_test.go | 6 +++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index 5d277e3e2..0f1f7f700 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -11,6 +11,7 @@ import ( "sort" "strings" "text/template" + "time" ) type count struct { @@ -31,9 +32,10 @@ type reportSection struct { } type reportData struct { - GeneratedAt string - ProbeStartedAt string - Sections [5]reportSection + GeneratedAt string + ProbeStartedAt string + ReportGeneratedAt string + Sections [5]reportSection } func (w WorkerInfo) WorkerPoolURL() string { @@ -121,8 +123,9 @@ _Configured values were not collected in this snapshot. Legacy totals included s This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots. Worker implementation and version are inferred from the failure log produced when each pool is given an intentionally malformed probe task; image and capacity metadata come from Worker Manager. Summary counts represent worker pools, not individual workers or tasks. -{{ if .ProbeStartedAt }}Probe run started: **{{ .ProbeStartedAt }}**{{ if .GeneratedAt }} · Results collected: **{{ .GeneratedAt }}**{{ end }} -{{ else if .GeneratedAt }}Results collected: **{{ .GeneratedAt }}** +{{ if .ProbeStartedAt }}Probe run started: **{{ .ProbeStartedAt }}**{{ if .GeneratedAt }} · Results collected: **{{ .GeneratedAt }}**{{ end }}{{ if .ReportGeneratedAt }} · Report generated: **{{ .ReportGeneratedAt }}**{{ end }} +{{ else if .GeneratedAt }}Results collected: **{{ .GeneratedAt }}**{{ if .ReportGeneratedAt }} · Report generated: **{{ .ReportGeneratedAt }}**{{ end }} +{{ else if .ReportGeneratedAt }}Report generated: **{{ .ReportGeneratedAt }}** {{ end }} {{ range .Sections }} @@ -361,6 +364,10 @@ func writeReadme(snapshot WorkerSnapshot) { } func renderReadme(snapshot WorkerSnapshot) string { + return renderReadmeAt(snapshot, time.Now()) +} + +func renderReadmeAt(snapshot WorkerSnapshot, reportGeneratedAt time.Time) string { workers := snapshot.Workers sections := [5]reportSection{ generateReadmeSection("Generic Worker", "", workers, func(w WorkerInfo) bool { return w.Implementation == "generic-worker" }), @@ -378,6 +385,9 @@ func renderReadme(snapshot WorkerSnapshot) string { if !snapshot.ProbeStartedAt.IsZero() { data.ProbeStartedAt = snapshot.ProbeStartedAt.UTC().Format(timestampFormat) } + if !reportGeneratedAt.IsZero() { + data.ReportGeneratedAt = reportGeneratedAt.UTC().Format(timestampFormat) + } return renderTemplate(data) } diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index 5a2aae4ef..1d80b358a 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -54,14 +54,14 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { Details: map[string]string{"revision": "1234567890"}, }} - got := renderReadme(WorkerSnapshot{ + got := renderReadmeAt(WorkerSnapshot{ GeneratedAt: time.Date(2026, time.September, 9, 15, 29, 53, 0, time.UTC), ProbeStartedAt: time.Date(2026, time.September, 9, 7, 58, 29, 0, time.UTC), Workers: workers, - }) + }, time.Date(2026, time.September, 10, 1, 2, 3, 0, time.UTC)) for _, want := range []string{ "This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots.", - "Probe run started: **2026-09-09 07:58 UTC** · Results collected: **2026-09-09 15:29 UTC**", + "Probe run started: **2026-09-09 07:58 UTC** · Results collected: **2026-09-09 15:29 UTC** · Report generated: **2026-09-10 01:02 UTC**", "Total worker pools: `2`", "### Count by version", "intentionally malformed probe task", From 4153249c4faedf6eb95ca48065b1e54b8a47a464 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 16:15:48 -0700 Subject: [PATCH 24/74] fix: customize Quick navigation pill --- docs/index-local.html | 14 ++++++++++++++ docs/index.html | 13 +++++++++++++ docs/worker-metrics.html | 13 +++++++++++++ index.html | 14 ++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/docs/index-local.html b/docs/index-local.html index 24891023a..ffcaf57f1 100644 --- a/docs/index-local.html +++ b/docs/index-local.html @@ -4,6 +4,20 @@ Local Worker Version Report +

Opening the worker version report with local data…

diff --git a/docs/index.html b/docs/index.html index a60e5aa1d..ad36f4884 100644 --- a/docs/index.html +++ b/docs/index.html @@ -12,6 +12,19 @@ crossorigin="anonymous" referrerpolicy="no-referrer" />

Opening the Firefox CI worker pool version report…

From 856b427eabf71b01f0e91d8eb5207e6562f8a5f9 Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 17:01:06 -0700 Subject: [PATCH 25/74] fix: clarify unknown worker implementations --- audit-worker-versions/report.go | 2 +- audit-worker-versions/report_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index 0f1f7f700..db23c3087 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -373,7 +373,7 @@ func renderReadmeAt(snapshot WorkerSnapshot, reportGeneratedAt time.Time) string generateReadmeSection("Generic Worker", "", workers, func(w WorkerInfo) bool { return w.Implementation == "generic-worker" }), generateReadmeSection("Docker Worker", "", workers, func(w WorkerInfo) bool { return w.Implementation == "docker-worker" }), generateReadmeSection("Script Worker", "", workers, func(w WorkerInfo) bool { return strings.Contains(w.Implementation, "Scriptworker") }), - generateReadmeSection("No artifacts found", "These pools claimed and resolved the probe task, but did not publish `public/logs/live_backing.log` or `public/logs/chain_of_trust.log`, which are used to identify the worker implementation.", workers, func(w WorkerInfo) bool { return w.hasNoArtifacts }), + generateReadmeSection("Worker implementation unknown", "These pools claimed and resolved the probe task, but did not publish a recognized worker log artifact. Their worker implementation and version could therefore not be identified.", workers, func(w WorkerInfo) bool { return w.hasNoArtifacts }), generateReadmeSection("Version not determined", "These pools did not claim the probe task within two hours, so their worker implementation and version could not be determined.", workers, func(w WorkerInfo) bool { return w.isUnknown }), } diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index 1d80b358a..eb90ee39f 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -177,8 +177,8 @@ func TestRenderReadmeExplainsIncompleteProbesInline(t *testing.T) { got := renderReadme(WorkerSnapshot{Workers: workers}) for _, want := range []string{ - "## No artifacts found\n", - "did not publish `public/logs/live_backing.log`", + "## Worker implementation unknown\n", + "did not publish a recognized worker log artifact", "## Version not determined\n", "did not claim the probe task within two hours", } { From 0ad0d635d1f19c5096ecddb944f522d79e8a95bf Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 17:15:51 -0700 Subject: [PATCH 26/74] fix: clarify worker pool count headings --- audit-worker-versions/report.go | 8 ++++---- audit-worker-versions/report_test.go | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go index db23c3087..b982680a1 100644 --- a/audit-worker-versions/report.go +++ b/audit-worker-versions/report.go @@ -82,22 +82,22 @@ const readmeTpl = ` Total worker pools: ` + "`" + `{{ .Count }}` + "`" + ` {{ if gt (len .Versions) 1 }} -### Count by version +### Worker pools by version _Source: version information parsed from the log artifact produced when each worker claims an intentionally malformed probe task. The task is expected to fail with a malformed-payload exception; known worker implementations and versions are identified from their distinct log output._ -| Version | Count | +| Version | Worker pools | | :--- | ---: | {{ range .Versions -}} | {{ .Key }} | {{ .Value }} | {{ end }} {{- end }} {{ if gt (len .Images) 1 }} -### Count by image +### Worker pools by image _Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images. Standalone pools do not have a Worker Manager-managed image._ -| Image | Count | +| Image | Worker pools | | :--- | ---: | {{ range .Images -}} | {{ if .Hover }}{{ .Key }}{{ else }}{{ .Key }}{{ end }} | {{ .Value }} | diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go index eb90ee39f..0c62230bc 100644 --- a/audit-worker-versions/report_test.go +++ b/audit-worker-versions/report_test.go @@ -63,7 +63,8 @@ func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { "This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots.", "Probe run started: **2026-09-09 07:58 UTC** · Results collected: **2026-09-09 15:29 UTC** · Report generated: **2026-09-10 01:02 UTC**", "Total worker pools: `2`", - "### Count by version", + "### Worker pools by version", + "### Worker pools by image", "intentionally malformed probe task", "expected to fail with a malformed-payload exception", "live Worker Manager launch configuration", @@ -91,7 +92,7 @@ func TestRenderReadmeDistinguishesImageStatuses(t *testing.T) { got := renderReadme(WorkerSnapshot{Workers: workers}) for _, want := range []string{ - "| Image | Count |", + "| Image | Worker pools |", "| Not applicable (standalone) | 1 |", "| Configuration unavailable | 1 |", "| Image not determined | 1 |", From 1c92bf3cd965e50bb150f8a1d5ba08c863a2543e Mon Sep 17 00:00:00 2001 From: Andrew Erickson Date: Wed, 9 Sep 2026 18:39:27 -0700 Subject: [PATCH 27/74] fix: reconcile report UI with upstream --- docs/app.js | 179 ------------------------------ docs/index-local.html | 5 +- docs/index.html | 250 ++++++++++++++++++++++++++---------------- docs/index.test.mjs | 29 +++++ 4 files changed, 191 insertions(+), 272 deletions(-) delete mode 100644 docs/app.js create mode 100644 docs/index.test.mjs diff --git a/docs/app.js b/docs/app.js deleted file mode 100644 index fa87d866e..000000000 --- a/docs/app.js +++ /dev/null @@ -1,179 +0,0 @@ -import { decorateSortableTables } from './table-sort.js' - -// This works for both the Mozilla and community history GitHub Pages sites. -let basePath = String(window.location.pathname) -if (!basePath.includes('-history')) basePath = '/mozilla-history/' - -const query = new URLSearchParams(window.location.search) -const bundledReport = query.has('local') || ( - !query.has('remote') && ( - window.location.hostname.endsWith('.quick.mozilla.cloud') || - window.location.pathname.includes('/docs/') - ) -) -const readmeUrl = bundledReport - ? new URL('../WorkerVersions/README.md', window.location.href) - : `https://raw.githubusercontent.com/taskcluster${basePath}master/WorkerVersions/README.md` -const historyUrl = new URL('history.json', window.location.href) - -async function loadReadme() { - const response = await fetch(readmeUrl) - const markdown = await response.text() - showdown.setFlavor('github') - document.getElementById('content').innerHTML = new showdown.Converter({ - ghCompatibleHeaderId: true, - tables: true, - }).makeHtml(markdown) -} - -function renderHistoryTables(data) { - const dates = Object.keys(data).sort((a, b) => b.localeCompare(a)) - - const tableByDate = (title, property, description = '') => { - let columns = [] - dates.forEach(date => { - const values = data[date][property] - Object.keys(values).forEach(key => { - if (!columns.includes(key)) columns.push(key) - }) - }) - columns = columns.sort((a, b) => a.localeCompare(b)) - - let table = `

${title}

${description}` - table += `
Date${key || 'unknown'}
${date}
${columns.map(key => ``).join('')}` - dates.forEach(date => { - table += ` - ${columns.map(column => data[date][property][column] || '-').map(value => ``).join('')} - ` - }) - table += '
Date${key || 'unknown'}
${date}${value}
' - return table - } - - let html = '

History

' - html += tableByDate( - 'Worker implementations', - 'implementations', - '

Worker implementation is inferred from distinctive content in the log artifact produced by the intentionally malformed probe task. Docker Worker identifies the legacy docker-worker implementation; it does not indicate that a generic-worker pool is configured to accept docker-worker-style payloads.

', - ) - html += tableByDate('Worker versions', 'versions') - document.getElementById('history-content').innerHTML = html -} - -function renderGraphs(data) { - const dates = Object.keys(data).sort() - const lineTrace = (name, y) => ({ - x: dates, - y, - name, - type: 'scatter', - mode: 'lines', - hovertemplate: '%{fullData.name}
Date: %{x}
Worker pools: %{y}', - }) - const implementationTraces = [ - ['Generic worker', 'generic-worker'], - ['Docker worker', 'docker-worker'], - ['Unknown', ''], - ].map(([name, implementation]) => lineTrace( - name, - dates.map(date => data[date].implementations[implementation] || 0), - )) - - const allVersions = [...new Set(dates.map(date => Object.keys(data[date].versions)).flat())].sort() - const versionTraces = allVersions.map(version => lineTrace( - version || 'unknown', - dates.map(date => data[date].versions[version] || 0), - )) - - const layout = (title, legendTitle) => ({ - hovermode: 'closest', - title, - legend: { title: { text: legendTitle } }, - xaxis: { title: { text: 'Snapshot date' } }, - yaxis: { title: { text: 'Worker pools' }, rangemode: 'tozero' }, - }) - Plotly.newPlot( - 'graph-implementations', - implementationTraces, - layout('Worker pools by implementation', 'Implementation'), - ) - Plotly.newPlot( - 'graph-versions', - versionTraces, - layout('Worker pools by version', 'Version'), - ) -} - -async function loadHistory() { - const response = await fetch(historyUrl) - const data = await response.json() - renderHistoryTables(data) - renderGraphs(data) -} - -function slugify(value) { - return value - .toLowerCase() - .trim() - .replace(/[^a-z0-9]+/g, '-') - .replace(/^-|-$/g, '') -} - -function buildTableOfContents() { - const headings = document.querySelectorAll([ - '#content h2', - '#content h3', - '#history-content h2', - '#history-content h3', - '#graphs-content h2', - '#graphs-content h3', - ].join(', ')) - const toc = document.getElementById('toc-list') - const usedIds = new Set() - let sectionId = '' - let sectionItem = null - let subsectionList = null - - headings.forEach(heading => { - const baseId = slugify(heading.textContent) - const candidateId = heading.tagName === 'H3' && sectionId - ? `${sectionId}-${baseId}` - : baseId - let id = candidateId - let suffix = 2 - while (usedIds.has(id)) { - id = `${candidateId}-${suffix}` - suffix += 1 - } - usedIds.add(id) - heading.id = id - - const item = document.createElement('li') - const link = document.createElement('a') - link.href = `#${id}` - link.textContent = heading.textContent - item.appendChild(link) - - if (heading.tagName === 'H2') { - sectionId = id - sectionItem = item - subsectionList = null - toc.appendChild(item) - return - } - - if (!subsectionList) { - subsectionList = document.createElement('ul') - sectionItem.appendChild(subsectionList) - } - subsectionList.appendChild(item) - }) -} - -async function init() { - await Promise.all([loadReadme(), loadHistory()]) - buildTableOfContents() - decorateSortableTables() -} - -init() diff --git a/docs/index-local.html b/docs/index-local.html index ffcaf57f1..c0b3c0372 100644 --- a/docs/index-local.html +++ b/docs/index-local.html @@ -22,7 +22,10 @@

Opening the worker version report with local data…