diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..762c51cb5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ +name: CI + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + name: Go, Python, and JavaScript tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: stable + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.13' + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: '24' + - name: Go tests + run: go test ./... + - name: Python tests + run: python3 -m unittest test_fetch_and_generate.py + - name: JavaScript tests + working-directory: docs + run: npm test + + browser-tests: + name: Firefox layout tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: stable + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.13' + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: '24' + cache: npm + cache-dependency-path: docs/package-lock.json + - name: Install test dependencies + working-directory: docs + run: npm ci + - name: Install Firefox and system dependencies + working-directory: docs + run: npx playwright install --with-deps firefox + - name: Run layout tests + working-directory: docs + run: npm run test:browser + - name: Upload failure screenshots and traces + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: firefox-layout-results + path: docs/test-results/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index 09a76685e..cf6fbf15b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ -mozilla-history -audit-worker-versions -!audit-worker-versions/main.go +# Locally built executables. +/mozilla-history +/audit-worker-versions/audit-worker-versions + +# Browser-test dependencies and artifacts. +/docs/node_modules/ +/docs/playwright-report/ +/docs/test-results/ +__pycache__/ diff --git a/README.md b/README.md index 3d55383c6..4c4d5e873 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,82 @@ mozilla-history This will populate subdirectories `Clients`, `Hooks`, `Roles` and `WorkerPools` of the current directory. -## Automating the Process +## Refreshing All Local Data -You can automate this reporting process by setting up a cron job to execute `run-report.sh` at regular intervals. +For a complete local refresh, run: + +```sh +./fetch_and_generate.py +``` + +When `~/.tc_token` exists, the script loads its JSON `clientId` and +`accessToken` fields. Otherwise, it uses `TASKCLUSTER_CLIENT_ID` and +`TASKCLUSTER_ACCESS_TOKEN` from the environment. The token file takes +precedence so stale exported credentials cannot silently override it. + +The script builds temporary copies of the Go tools, refreshes `Clients`, +`Hooks`, `Roles`, and `WorkerPools`, schedules worker-version probes, polls their +sealed task group until every task reaches a terminal state, writes +`WorkerVersions`, and stops. It does not pull, commit, push, or deploy anything. +Review the generated changes before committing them. + +The local refresh intentionally does not rebuild `docs/history.json`. History is +derived exclusively from worker snapshots already committed to Git, so it cannot +include the snapshot generated by the current run. The production publishing +workflow rebuilds it after the snapshot is committed. + +Use `--poll-interval SECONDS` to change the 60-second polling interval. If the +script is interrupted after scheduling probes, resume without creating another +group using `--task-group-id TASK_GROUP_ID`. `TASKCLUSTER_ROOT_URL`, +`REPORT_SCHEDULER_ID`, and `REPORT_PREFIX` can be overridden in the environment. + +Worker versions require two phases because Worker Manager configuration does +not expose the implementation and version actually running in each pool. The +first phase schedules an intentionally malformed task on every pool so the +worker identifies itself in its task log. Once all tasks are resolved, the +second phase reads those logs and generates the snapshot. Polling replaces the +older fixed-delay assumption while preserving that probe-and-collect design. + +## Rendering an Existing Worker Snapshot + +The worker-version report can be regenerated from a saved snapshot without +Taskcluster credentials or probe tasks: + +```sh +go run ./audit-worker-versions render \ + WorkerVersions/workers.json \ + WorkerVersions/README.md +``` + +Omit the output path to print the generated Markdown to standard output. + +To preview the generated report in the website, serve the repository root: + +```sh +./run_local.sh +``` + +Then open . The local preview uses +`WorkerVersions/README.md` and `docs/history.json` from the checkout. The +published page continues to load the current report from GitHub. Pass a port as +the first argument to override the default, for example `./run_local.sh 8080`. + +## Testing + +Pull requests and pushes to `master` run Go, Python, JavaScript, and Firefox +layout tests through `.github/workflows/ci.yml`. These checks do not create +Taskcluster probes or publish reports. + +## Optional Quick previews + +See [scripts/quick/README.md](scripts/quick/README.md) for personal Quick previews +of the checkout's report. These helpers are separate from production publishing. + +## Production Automation + +The existing `run-reports.sh` and `audit.sh` scripts implement the repository's +production publishing workflow. They include git and production-site behavior; +use `fetch_and_generate.py` for local refreshes. ### Prerequisites - Valid Taskcluster credentials must be set in the environment variables: @@ -56,13 +129,13 @@ You can automate this reporting process by setting up a cron job to execute `run - `TASKCLUSTER_ACCESS_TOKEN` ### How it works -1. `run-report.sh` executes `audit.sh` +1. `run-reports.sh` executes `audit.sh` 2. `audit.sh` schedules tasks for each worker pool to extract worker implementation details from logs -3. Results are stored in the `WorkerPools` directory +3. Results are stored in the `WorkerVersions` directory 4. `mozilla-history` stores Taskcluster configurations in their respective directories: - Hooks definitions in `Hooks/` - Roles definitions in `Roles/` - - WorkerPool definitions in `WorkerPool/` + - Worker Pool definitions in `WorkerPools/` 5. `build-docs-history.sh` collects every past revision of `WorkerVersions/workers.json` into `docs/history.json`, which feeds the two GitHub Pages views: - `docs/index.html` — the current report plus the full history tables and graphs diff --git a/audit-worker-versions/main.go b/audit-worker-versions/main.go index f7a4b8312..8af3ec5f3 100644 --- a/audit-worker-versions/main.go +++ b/audit-worker-versions/main.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "encoding/json" "fmt" "io" @@ -13,7 +12,6 @@ import ( "sort" "strings" "sync" - "text/template" "time" "github.com/taskcluster/httpbackoff/v3" @@ -29,15 +27,78 @@ type ( ) type WorkerInfo struct { - WorkerPoolID string - Implementation string - Version string - Details map[string]string - hasNoArtifacts bool - isUnknown bool - Imageset string - TotalWorkers int - TotalCapacity int + WorkerPoolID string + Implementation string + Version string + Details map[string]string + hasNoArtifacts bool + isUnknown bool + Imageset string + ImageStatus string `json:",omitempty"` + ProviderID string `json:",omitempty"` + ConfiguredMinCapacity *int `json:",omitempty"` + ConfiguredMaxCapacity *int `json:",omitempty"` + CapacityPerWorker *int `json:",omitempty"` + ConfiguredMinWorkers *int `json:",omitempty"` + ConfiguredMaxWorkers *int `json:",omitempty"` + WorkerManagerLookupError string `json:",omitempty"` + // Retained only to recognize snapshots created before configured capacity + // fields were collected. New snapshots leave these fields unset. + LegacyTotalWorkers *int `json:"TotalWorkers,omitempty"` + LegacyTotalCapacity *int `json:"TotalCapacity,omitempty"` +} + +const ( + imageStatusKnown = "known" + imageStatusNotApplicable = "not-applicable" + imageStatusNotDetermined = "not-determined" + imageStatusUnavailable = "unavailable" + standaloneProviderID = "standalone" + standaloneWorkerState = "standalone" +) + +type workerManagerClient interface { + WorkerPool(workerPoolID string) (*tcworkermanager.WorkerPoolFullDefinition, error) + ListWorkers(provisionerID, workerType, continuationToken, limit, quarantined, workerState string) (*tcworkermanager.ListWorkersResponse, error) +} + +type WorkerSnapshot struct { + GeneratedAt time.Time `json:"generatedAt"` + ProbeStartedAt time.Time `json:"probeStartedAt"` + TaskGroupID string `json:"taskGroupId"` + Workers []WorkerInfo `json:"workers"` +} + +type taskGroupProgress struct { + Total int + Terminal int + States map[string]int +} + +func (progress taskGroupProgress) complete() bool { + return progress.Total > 0 && progress.Terminal == progress.Total +} + +func isTerminalTaskState(state string) bool { + switch state { + case "completed", "failed", "exception": + return true + default: + return false + } +} + +func summarizeTaskGroup(tasks []tcqueue.TaskDefinitionAndStatus) taskGroupProgress { + progress := taskGroupProgress{States: map[string]int{}} + for _, task := range tasks { + state := task.Status.State + progress.Total++ + progress.States[state]++ + if isTerminalTaskState(state) { + progress.Terminal++ + } + } + return progress } func (w *WorkerInfo) String() string { @@ -66,37 +127,57 @@ func (w *WorkerInfo) String() string { return strings.Trim(info, " ") } -func GetImageset(wp *tcworkermanager.WorkerPoolFullDefinition) string { - var p = wp.ProviderID - if p == "test-provisioner" || p == "no-provisioning-nope" || p == "dummy-test-provisioner" || p == "test-dummy-provisioner" { - return "unknown" - } - - type LaunchConfig struct { - // AWS - LaunchConfig struct { - ImageId string - } `json:"launchConfig"` +type workerPoolLaunchConfig struct { + // AWS + LaunchConfig struct { + ImageId string + } `json:"launchConfig"` + + // GCP + Disks []struct { + InitializeParams struct { + SourceImage string `json:"sourceImage"` + } `json:"initializeParams"` + } `json:"disks"` + + // Azure + StorageProfile struct { + ImageReference struct { + Id string `json:"id"` + } `json:"imageReference"` + } `json:"storageProfile"` + ArmDeployment struct { + Parameters struct { + ImageID struct { + Value string `json:"value"` + } `json:"imageId"` + } `json:"parameters"` + } `json:"armDeployment"` + + // Older Worker Manager configurations stored this directly on the + // launch config. Newer configurations nest it under workerManager. + CapacityPerInstance *int `json:"capacityPerInstance"` + WorkerManager struct { + CapacityPerInstance *int `json:"capacityPerInstance"` + } `json:"workerManager"` +} - // GCP - Disks []struct { - InitializeParams struct { - SourceImage string `json:"sourceImage"` - } `json:"initializeParams"` - } `json:"disks"` +type workerPoolConfig struct { + MinCapacity *int `json:"minCapacity"` + MaxCapacity *int `json:"maxCapacity"` + LaunchConfigs []workerPoolLaunchConfig `json:"launchConfigs"` +} - // Azure - StorageProfile struct { - ImageReference struct { - Id string `json:"id"` - } `json:"imageReference"` - } `json:"storageProfile"` - } - type Config struct { - LaunchConfigs []LaunchConfig `json:"launchConfigs"` - } - var cfg Config +func parseWorkerPoolConfig(wp *tcworkermanager.WorkerPoolFullDefinition) (workerPoolConfig, error) { + var cfg workerPoolConfig if err := json.Unmarshal(wp.Config, &cfg); err != nil { + return workerPoolConfig{}, err + } + return cfg, nil +} + +func getImageset(providerID string, cfg workerPoolConfig) string { + if providerID == "test-provisioner" || providerID == "no-provisioning-nope" || providerID == "dummy-test-provisioner" || providerID == "test-dummy-provisioner" { return "unknown" } @@ -107,6 +188,7 @@ func GetImageset(wp *tcworkermanager.WorkerPoolFullDefinition) string { imagesMap[disk.InitializeParams.SourceImage] = struct{}{} } imagesMap[launchCfg.StorageProfile.ImageReference.Id] = struct{}{} + imagesMap[launchCfg.ArmDeployment.Parameters.ImageID.Value] = struct{}{} } // remove empty image name "" delete(imagesMap, "") @@ -122,57 +204,99 @@ func GetImageset(wp *tcworkermanager.WorkerPoolFullDefinition) string { return sortedImages } -var ( - // Templates for README file - readmeTpl string = ` -{{- define "row" -}} -## {{ .Title }} - -Total: ` + "`" + `{{ .Count }}` + "`" + ` -{{ if gt (len .Versions) 1 }} -Count by version: - -| Version | Count | -| :--- | ---: | -{{ range .Versions -}} -| {{ .Key }} | {{ .Value }} | -{{ end }} -{{- end }} -{{ if gt (len .Images) 1 }} -Count by image: - -| Version | Count | -| :--- | ---: | -{{ range .Images -}} -| {{ .Key }} | {{ .Value }} | -{{ end }} -{{- end }} -{{if .Count }} -| Worker Pool | Implementation | Version {{ if .FullColumns }}| Engine | Revision | OS | Arch | GO {{ end }}| Total Workers | Total Capacity | -| --- | --- | --- {{ if .FullColumns }}| --- | --- | --- | --- | --- {{ end }}| ---: | ---: | -{{ range .Filtered -}} -| **{{ .WorkerPoolID }}** | {{ .Implementation }} | {{ or .Version .Details.error }} {{ if $.FullColumns }}| {{ or .Details.engine "-" }} | {{ or (slice .Details.revision 0 10) "-" }} | {{ or .Details.os "-" }} | {{ or .Details.arch "-" }} | {{ or .Details.go "-" }} {{ end }}| {{ .TotalWorkers }} | {{ .TotalCapacity }} | -{{end}} -{{- end -}} -{{end}} - -# Worker Pool Versions - -{{ range . }} -{{ template "row" . }} -{{ end }} - -[^1]: Those are the pools whose tasks were claimed and resolved by a worker as expected, but the worker did not publish either artifact ` + "`public/logs/live_backing.log` nor `public/logs/chain_of_trust.log`" + `, which is the source used to identify the worker implementation. - -[^2]: Probing task remains pending after two hours. Those are the pools that were not able to start any worker to claim the task within two hours. -` -) +func ceilDivide(value, divisor int) int { + return (value + divisor - 1) / divisor +} + +func enrichWorkerInfo(workerInfo *WorkerInfo, wp *tcworkermanager.WorkerPoolFullDefinition) { + workerInfo.ProviderID = wp.ProviderID + cfg, err := parseWorkerPoolConfig(wp) + if err != nil { + workerInfo.Imageset = "unknown" + workerInfo.ImageStatus = imageStatusUnavailable + workerInfo.WorkerManagerLookupError = "Could not parse Worker Manager configuration: " + err.Error() + return + } + + workerInfo.Imageset = getImageset(wp.ProviderID, cfg) + workerInfo.ImageStatus = imageStatusKnown + if workerInfo.Imageset == "unknown" { + workerInfo.ImageStatus = imageStatusNotDetermined + } + workerInfo.ConfiguredMinCapacity = cfg.MinCapacity + workerInfo.ConfiguredMaxCapacity = cfg.MaxCapacity + + capacities := map[int]struct{}{} + for _, launchConfig := range cfg.LaunchConfigs { + capacity := launchConfig.CapacityPerInstance + if launchConfig.WorkerManager.CapacityPerInstance != nil { + capacity = launchConfig.WorkerManager.CapacityPerInstance + } + if capacity != nil && *capacity > 0 { + capacities[*capacity] = struct{}{} + } + } + + if len(capacities) != 1 || cfg.MinCapacity == nil || cfg.MaxCapacity == nil { + return + } + for capacity := range capacities { + capacityPerWorker := capacity + minWorkers := ceilDivide(*cfg.MinCapacity, capacity) + maxWorkers := ceilDivide(*cfg.MaxCapacity, capacity) + workerInfo.CapacityPerWorker = &capacityPerWorker + workerInfo.ConfiguredMinWorkers = &minWorkers + workerInfo.ConfiguredMaxWorkers = &maxWorkers + } +} + +func workerPoolIsStandalone(workermanager workerManagerClient, workerPoolID string) (bool, error) { + parts := strings.SplitN(workerPoolID, "/", 2) + if len(parts) != 2 { + return false, fmt.Errorf("invalid worker pool ID %q", workerPoolID) + } + + foundStandaloneWorker := false + continuationToken := "" + for { + workers, err := workermanager.ListWorkers(parts[0], parts[1], continuationToken, "", "", "") + if err != nil { + return false, err + } + for _, worker := range workers.Workers { + if worker.State == standaloneWorkerState { + foundStandaloneWorker = true + } + } + continuationToken = workers.ContinuationToken + if continuationToken == "" { + break + } + } + return foundStandaloneWorker, nil +} + +func lookupAndEnrichWorkerInfo(workerInfo *WorkerInfo, workermanager workerManagerClient) { + workerPool, lookupErr := workermanager.WorkerPool(workerInfo.WorkerPoolID) + if lookupErr == nil { + enrichWorkerInfo(workerInfo, workerPool) + return + } -func renderTemplate(data interface{}) string { - t := template.Must(template.New("").Parse(readmeTpl)) - var content bytes.Buffer - t.Execute(&content, data) - return content.String() + standalone, standaloneErr := workerPoolIsStandalone(workermanager, workerInfo.WorkerPoolID) + if standaloneErr == nil && standalone { + workerInfo.ProviderID = standaloneProviderID + workerInfo.Imageset = "unknown" + workerInfo.ImageStatus = imageStatusNotApplicable + return + } + + workerInfo.Imageset = "unknown" + workerInfo.ImageStatus = imageStatusUnavailable + workerInfo.WorkerManagerLookupError = lookupErr.Error() + if standaloneErr != nil { + workerInfo.WorkerManagerLookupError += "; could not check for standalone workers: " + standaloneErr.Error() + } } var ( @@ -181,7 +305,7 @@ var ( ) var ( - outputDir = "WorkerVersions" + outputDir = "WorkerVersions" ) func FilenameEscape(raw string) (escaped string) { @@ -195,7 +319,6 @@ func EmptyDirectory(dir string) { } } - func WriteFile(path string, content []byte) { err := os.MkdirAll(filepath.Dir(path), 0755) if err != nil { @@ -204,12 +327,13 @@ func WriteFile(path string, content []byte) { err = os.WriteFile(path, content, 0644) if err != nil { - log.Fatalf("Error:\n%v", err) + log.Fatalf("Error:\n%v", err) } } // Call with no arguments -> New task group generated // Call with one argument (taskGroupID) -> Report generated for previously created task group +// Call with "status taskGroupID" -> Task group progress reported // // Expected workflow for this tool is to: // 1. Run without arguments to generate probing tasks and get taskGroupId @@ -217,8 +341,34 @@ func WriteFile(path string, content []byte) { // // Files are written to the WorkerVersions directory func main() { + if len(os.Args) >= 2 && os.Args[1] == "render" { + if len(os.Args) < 3 || len(os.Args) > 4 { + log.Fatal("Usage: audit-worker-versions render INPUT_JSON [OUTPUT_MARKDOWN]") + } + snapshot, err := readSnapshot(os.Args[2]) + fatalOnError(err) + contents := renderReadme(snapshot) + if len(os.Args) == 4 { + WriteFile(os.Args[3], []byte(contents)) + } else { + fmt.Print(contents) + } + return + } queue := tcqueue.NewFromEnv() + if len(os.Args) >= 2 && os.Args[1] == "status" { + if len(os.Args) != 3 { + log.Fatal("Usage: audit-worker-versions status TASK_GROUP_ID") + } + progress, err := taskGroupStatus(queue, os.Args[2]) + fatalOnError(err) + printTaskGroupProgress(os.Args[2], progress) + if !progress.complete() { + os.Exit(3) + } + return + } switch len(os.Args) { case 1: @@ -233,12 +383,53 @@ func main() { if len(taskIDs) == 0 { log.Fatalf("No tasks with taskGroupId %q", taskGroupID) } - inspect(queue, taskIDs) + inspect(queue, taskGroupID, taskIDs) default: log.Fatalf("Expected zero or one program arguments, but have %v: %q", len(os.Args)-1, os.Args[1:]) } } +func taskGroupStatus(queue *tcqueue.Queue, taskGroupID string) (taskGroupProgress, error) { + tasks := []tcqueue.TaskDefinitionAndStatus{} + continuationToken := "" + for { + response, err := queue.ListTaskGroup(taskGroupID, continuationToken, "") + if err != nil { + return taskGroupProgress{}, err + } + tasks = append(tasks, response.Tasks...) + continuationToken = response.ContinuationToken + if continuationToken == "" { + break + } + } + if len(tasks) == 0 { + return taskGroupProgress{}, fmt.Errorf("no tasks with taskGroupId %q", taskGroupID) + } + return summarizeTaskGroup(tasks), nil +} + +func printTaskGroupProgress(taskGroupID string, progress taskGroupProgress) { + stateOrder := []string{"completed", "failed", "exception", "running", "pending", "unscheduled"} + states := make([]string, 0, len(progress.States)) + knownStates := map[string]bool{} + for _, state := range stateOrder { + knownStates[state] = true + if count := progress.States[state]; count > 0 { + states = append(states, fmt.Sprintf("%s=%d", state, count)) + } + } + extraStates := []string{} + for state, count := range progress.States { + if !knownStates[state] { + extraStates = append(extraStates, fmt.Sprintf("%s=%d", state, count)) + } + } + sort.Strings(extraStates) + states = append(states, extraStates...) + fmt.Printf("Task group %s: %d/%d terminal (%s)\n", taskGroupID, progress.Terminal, progress.Total, strings.Join(states, ", ")) +} + func createTasks(queue *tcqueue.Queue, taskGroupID string) { if revision != "" { log.Printf("%v built from revision %v", os.Args[0], revision) @@ -326,8 +517,10 @@ func createTasks(queue *tcqueue.Queue, taskGroupID string) { log.Printf("Task group sealed at: %v", tg.Sealed) } -func inspect(queue *tcqueue.Queue, taskIDs []string) { +func inspect(queue *tcqueue.Queue, taskGroupID string, taskIDs []string) { EmptyDirectory(outputDir) + probeTask, err := queue.Task(taskIDs[0]) + fatalOnError(err) workermanager := tcworkermanager.NewFromEnv() wp := workerpool.New(50) workers := make([]WorkerInfo, 0) @@ -342,16 +535,7 @@ func inspect(queue *tcqueue.Queue, taskIDs []string) { panic(err) } workerPoolID, workerInfo := show(queue, statusResponse) - workerPool, err := workermanager.WorkerPool(workerPoolID) - if err != nil { - fmt.Println("Could not fetch workerPool " + workerPoolID) - } else { - workerInfo.Imageset = GetImageset(workerPool) - workerInfo.TotalWorkers = int(workerPool.RunningCount) + int(workerPool.StoppedCount) + - int(workerPool.StoppingCount) + int(workerPool.RequestedCount) - workerInfo.TotalCapacity = int(workerPool.RunningCapacity) + int(workerPool.StoppedCapacity) + - int(workerPool.StoppingCapacity) + int(workerPool.RequestedCapacity) - } + lookupAndEnrichWorkerInfo(&workerInfo, workermanager) filename := filepath.Join(outputDir, FilenameEscape(workerPoolID)) WriteFile(filename, append([]byte(workerInfo.String()), '\n')) fmt.Printf("%-70s %s\n", workerPoolID+":", &workerInfo) @@ -365,79 +549,27 @@ func inspect(queue *tcqueue.Queue, taskIDs []string) { workers = append(workers, result.(WorkerInfo)) }) - fmt.Printf("\nWriting README.md\n") - writeReadme(workers) - fmt.Println("Writing workers.json") - writeSnapshot(workers) -} - -func generateReadmeSection(title string, workers []WorkerInfo, filter func(WorkerInfo) bool) map[string]interface{} { - filtered := make([]WorkerInfo, 0) - versions := make(map[string]int) - imagesets := make(map[string]int) - - for _, w := range workers { - if filter(w) { - filtered = append(filtered, w) - versions[w.Version]++ - imagesets[w.Imageset]++ - } - } - - sort.Slice(filtered, func(i, j int) bool { - return strings.Compare(filtered[i].WorkerPoolID, filtered[j].WorkerPoolID) < 0 - }) - - type kv struct { - Key string - Value int + snapshot := WorkerSnapshot{ + GeneratedAt: time.Now().UTC(), + ProbeStartedAt: time.Time(probeTask.Created).UTC(), + TaskGroupID: taskGroupID, + Workers: workers, } - var sortedVersions []kv - for k, v := range versions { - sortedVersions = append(sortedVersions, kv{k, v}) - } - sort.Slice(sortedVersions, func(i, j int) bool { - return strings.Compare(sortedVersions[i].Key, sortedVersions[j].Key) < 0 - }) - var images []kv - for k, v := range imagesets { - images = append(images, kv{k, v}) - } - - return map[string]interface{}{ - "FullColumns": title == "Generic Worker", - "Filtered": filtered, - "Count": len(filtered), - "Versions": sortedVersions, - "Images": images, - "Title": title, - } -} - -func writeReadme(workers []WorkerInfo) { - filename := filepath.Join(outputDir, "README.md") - - sections := [5]map[string]interface{}{ - generateReadmeSection("Generic Worker", workers, func(w WorkerInfo) bool { return w.Implementation == "generic-worker" }), - generateReadmeSection("Docker Worker", workers, func(w WorkerInfo) bool { return w.Implementation == "docker-worker" }), - generateReadmeSection("Script Worker", workers, func(w WorkerInfo) bool { return strings.Contains(w.Implementation, "Scriptworker") }), - generateReadmeSection("No artifacts found [^1]", workers, func(w WorkerInfo) bool { return w.hasNoArtifacts }), - generateReadmeSection("Version not determined [^2]", workers, func(w WorkerInfo) bool { return w.isUnknown }), - } - - contents := renderTemplate(sections) - WriteFile(filename, []byte(contents)) + fmt.Printf("\nWriting README.md\n") + writeReadme(snapshot) + fmt.Println("Writing workers.json") + writeSnapshot(snapshot) } -func writeSnapshot(workers []WorkerInfo) { +func writeSnapshot(snapshot WorkerSnapshot) { filename := filepath.Join(outputDir, "workers.json") - sort.Slice(workers, func(i, j int) bool { - return strings.Compare(workers[i].WorkerPoolID, workers[j].WorkerPoolID) <= 0 + sort.Slice(snapshot.Workers, func(i, j int) bool { + return strings.Compare(snapshot.Workers[i].WorkerPoolID, snapshot.Workers[j].WorkerPoolID) <= 0 }) - contents, err := json.MarshalIndent(workers, "", " ") + contents, err := json.MarshalIndent(snapshot, "", " ") if err != nil { log.Fatalf("Error:\n%v", err) } @@ -564,11 +696,8 @@ func show(queue *tcqueue.Queue, t *tcqueue.TaskStatusResponse) (workerPoolID str if workerInfo.Details == nil { workerInfo.Details = map[string]string{} } - if t.Status.State == "pending" { - // We schedule task with deadline for 3h and running report generation anywhere before that time - // In case some pools were not able to claim task within 3h we would consider this pool to have unknown workers - // which could probably tell that something is wrong with configuration of this pool - workerInfo.Details["error"] = "Version not determined; task not (yet) claimed" + if !taskWasClaimed(t.Status.Runs) { + workerInfo.Details["error"] = "Version not determined; task was not claimed" workerInfo.isUnknown = true return } @@ -670,6 +799,15 @@ func show(queue *tcqueue.Queue, t *tcqueue.TaskStatusResponse) (workerPoolID str return } +func taskWasClaimed(runs []tcqueue.RunInformation) bool { + for _, run := range runs { + if run.WorkerID != "" || run.WorkerGroup != "" || !time.Time(run.Started).IsZero() { + return true + } + } + return false +} + func taskIDsForTaskGroup(queue *tcqueue.Queue, taskGroupID string) []string { taskIDs := []string{} continuationToken := "" diff --git a/audit-worker-versions/main_test.go b/audit-worker-versions/main_test.go new file mode 100644 index 000000000..09cb8713d --- /dev/null +++ b/audit-worker-versions/main_test.go @@ -0,0 +1,296 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/taskcluster/taskcluster/v48/clients/client-go/tcqueue" + "github.com/taskcluster/taskcluster/v48/clients/client-go/tcworkermanager" +) + +type fakeWorkerManager struct { + workerPool *tcworkermanager.WorkerPoolFullDefinition + workerPoolError error + workerPages []*tcworkermanager.ListWorkersResponse + workerError error + listCalls int +} + +func (manager *fakeWorkerManager) WorkerPool(string) (*tcworkermanager.WorkerPoolFullDefinition, error) { + return manager.workerPool, manager.workerPoolError +} + +func (manager *fakeWorkerManager) ListWorkers(string, string, string, string, string, string) (*tcworkermanager.ListWorkersResponse, error) { + if manager.workerError != nil { + return nil, manager.workerError + } + page := manager.workerPages[manager.listCalls] + manager.listCalls++ + return page, nil +} + +func intPointer(value int) *int { + return &value +} + +func TestEnrichWorkerInfoWithConfiguredCapacity(t *testing.T) { + tests := []struct { + name string + config string + slots int + }{ + { + name: "current nested capacity", + config: `{"minCapacity":1,"maxCapacity":16,"launchConfigs":[{"workerManager":{"capacityPerInstance":8}}]}`, + slots: 8, + }, + { + name: "legacy direct capacity", + config: `{"minCapacity":1,"maxCapacity":16,"launchConfigs":[{"capacityPerInstance":8}]}`, + slots: 8, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + worker := WorkerInfo{} + pool := tcworkermanager.WorkerPoolFullDefinition{ + Config: json.RawMessage(test.config), + } + + enrichWorkerInfo(&worker, &pool) + + for name, result := range map[string]struct { + got *int + want int + }{ + "minimum capacity": {worker.ConfiguredMinCapacity, 1}, + "maximum capacity": {worker.ConfiguredMaxCapacity, 16}, + "slots per worker": {worker.CapacityPerWorker, test.slots}, + "minimum workers": {worker.ConfiguredMinWorkers, 1}, + "maximum workers": {worker.ConfiguredMaxWorkers, 2}, + } { + if result.got == nil || *result.got != result.want { + t.Errorf("%s = %v, want %d", name, result.got, result.want) + } + } + + encoded, err := json.Marshal(worker) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"ConfiguredMinCapacity":1`, + `"ConfiguredMaxCapacity":16`, + `"CapacityPerWorker":8`, + `"ConfiguredMinWorkers":1`, + `"ConfiguredMaxWorkers":2`, + } { + if !strings.Contains(string(encoded), want) { + t.Errorf("serialized worker does not contain %s", want) + } + } + if strings.Contains(string(encoded), "TotalWorkers") || strings.Contains(string(encoded), "TotalCapacity") { + t.Error("new worker snapshot contains legacy totals") + } + }) + } +} + +func TestEnrichWorkerInfoDoesNotDeriveWorkersForMixedCapacities(t *testing.T) { + worker := WorkerInfo{} + pool := tcworkermanager.WorkerPoolFullDefinition{ + Config: json.RawMessage(`{ + "minCapacity": 0, + "maxCapacity": 16, + "launchConfigs": [ + {"workerManager":{"capacityPerInstance":2}}, + {"workerManager":{"capacityPerInstance":8}} + ] + }`), + } + + enrichWorkerInfo(&worker, &pool) + + if worker.ConfiguredMinCapacity == nil || worker.ConfiguredMaxCapacity == nil { + t.Fatal("configured capacity bounds were not preserved") + } + if worker.CapacityPerWorker != nil || worker.ConfiguredMinWorkers != nil || worker.ConfiguredMaxWorkers != nil { + t.Fatal("worker counts were derived for heterogeneous launch capacities") + } +} + +func TestSummarizeTaskGroup(t *testing.T) { + tasks := []tcqueue.TaskDefinitionAndStatus{ + {Status: tcqueue.TaskStatusStructure{State: "completed"}}, + {Status: tcqueue.TaskStatusStructure{State: "failed"}}, + {Status: tcqueue.TaskStatusStructure{State: "exception"}}, + {Status: tcqueue.TaskStatusStructure{State: "running"}}, + {Status: tcqueue.TaskStatusStructure{State: "pending"}}, + } + + progress := summarizeTaskGroup(tasks) + + if progress.Total != 5 || progress.Terminal != 3 { + t.Fatalf("progress = %#v, want 3 of 5 terminal", progress) + } + if progress.complete() { + t.Fatal("incomplete task group was reported complete") + } +} + +func TestTaskGroupComplete(t *testing.T) { + progress := summarizeTaskGroup([]tcqueue.TaskDefinitionAndStatus{ + {Status: tcqueue.TaskStatusStructure{State: "completed"}}, + {Status: tcqueue.TaskStatusStructure{State: "failed"}}, + {Status: tcqueue.TaskStatusStructure{State: "exception"}}, + }) + + if !progress.complete() { + t.Fatalf("progress = %#v, want complete", progress) + } + if (taskGroupProgress{}).complete() { + t.Fatal("empty task group was reported complete") + } +} + +func TestTaskWasClaimed(t *testing.T) { + tests := []struct { + name string + runs []tcqueue.RunInformation + want bool + }{ + {name: "no runs"}, + { + name: "pending run", + runs: []tcqueue.RunInformation{{State: "pending"}}, + }, + { + name: "expired without claim", + runs: []tcqueue.RunInformation{{State: "exception", ReasonResolved: "deadline-exceeded"}}, + }, + { + name: "claimed malformed payload", + runs: []tcqueue.RunInformation{{State: "exception", ReasonResolved: "malformed-payload", WorkerGroup: "us-west1-b", WorkerID: "worker-1"}}, + want: true, + }, + { + name: "deadline exceeded after claim", + runs: []tcqueue.RunInformation{{State: "exception", ReasonResolved: "deadline-exceeded", WorkerID: "worker-1"}}, + want: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := taskWasClaimed(test.runs); got != test.want { + t.Fatalf("taskWasClaimed() = %v, want %v", got, test.want) + } + }) + } +} + +func TestShowClassifiesDeadlineExceededWithoutClaimAsUnresponsive(t *testing.T) { + status := &tcqueue.TaskStatusResponse{Status: tcqueue.TaskStatusStructure{ + ProvisionerID: "example", + WorkerType: "pool", + State: "exception", + Runs: []tcqueue.RunInformation{{ + State: "exception", + ReasonResolved: "deadline-exceeded", + }}, + }} + + _, worker := show(nil, status) + if !worker.isUnknown || worker.hasNoArtifacts { + t.Fatalf("worker classification = %#v, want unresponsive", worker) + } + if got := worker.Details["error"]; got != "Version not determined; task was not claimed" { + t.Fatalf("error = %q, want task-was-not-claimed result", got) + } +} + +func TestGetImagesetReadsAzureArmDeployment(t *testing.T) { + pool := tcworkermanager.WorkerPoolFullDefinition{ + ProviderID: "azure2", + Config: json.RawMessage(`{ + "launchConfigs": [ + {"armDeployment":{"parameters":{"imageId":{"value":"azure/image/10"}}}}, + {"armDeployment":{"parameters":{"imageId":{"value":"azure/image/2"}}}} + ] + }`), + } + worker := WorkerInfo{} + + enrichWorkerInfo(&worker, &pool) + + if worker.Imageset != "azure/image/10,azure/image/2" { + t.Fatalf("Imageset = %q, want Azure image IDs", worker.Imageset) + } + if worker.ImageStatus != imageStatusKnown || worker.ProviderID != "azure2" { + t.Fatalf("worker metadata = %#v, want known azure2 image", worker) + } +} + +func TestLookupRecognizesPaginatedStandaloneWorkers(t *testing.T) { + manager := &fakeWorkerManager{ + workerPoolError: errors.New("worker pool not found"), + workerPages: []*tcworkermanager.ListWorkersResponse{ + {ContinuationToken: "next", Workers: []tcworkermanager.Worker{{ProviderID: "none", State: standaloneWorkerState}}}, + {Workers: []tcworkermanager.Worker{{ProviderID: "none", State: "running"}}}, + }, + } + worker := WorkerInfo{WorkerPoolID: "releng-hardware/example"} + + lookupAndEnrichWorkerInfo(&worker, manager) + + if worker.ProviderID != standaloneProviderID || worker.ImageStatus != imageStatusNotApplicable { + t.Fatalf("worker metadata = %#v, want standalone/not-applicable", worker) + } + if worker.WorkerManagerLookupError != "" { + t.Fatalf("standalone worker retained lookup error %q", worker.WorkerManagerLookupError) + } + if manager.listCalls != 2 { + t.Fatalf("ListWorkers calls = %d, want 2", manager.listCalls) + } +} + +func TestLookupRetainsUnexpectedFailure(t *testing.T) { + manager := &fakeWorkerManager{ + workerPoolError: errors.New("worker pool unavailable"), + workerError: errors.New("worker list unavailable"), + } + worker := WorkerInfo{WorkerPoolID: "example/pool"} + + lookupAndEnrichWorkerInfo(&worker, manager) + + if worker.ImageStatus != imageStatusUnavailable { + t.Fatalf("ImageStatus = %q, want unavailable", worker.ImageStatus) + } + for _, want := range []string{"worker pool unavailable", "worker list unavailable"} { + if !strings.Contains(worker.WorkerManagerLookupError, want) { + t.Errorf("lookup error %q does not contain %q", worker.WorkerManagerLookupError, want) + } + } +} + +func TestLookupWithoutStandaloneSignalRemainsUnavailable(t *testing.T) { + manager := &fakeWorkerManager{ + workerPoolError: errors.New("worker pool not found"), + workerPages: []*tcworkermanager.ListWorkersResponse{ + {Workers: []tcworkermanager.Worker{{ProviderID: "none", State: "running"}}}, + }, + } + worker := WorkerInfo{WorkerPoolID: "example/pool"} + + lookupAndEnrichWorkerInfo(&worker, manager) + + if worker.ProviderID == standaloneProviderID { + t.Fatal("worker without standalone state was classified as standalone") + } + if worker.ImageStatus != imageStatusUnavailable || worker.WorkerManagerLookupError == "" { + t.Fatalf("worker metadata = %#v, want unavailable with lookup error", worker) + } +} diff --git a/audit-worker-versions/report.go b/audit-worker-versions/report.go new file mode 100644 index 000000000..9ddd9d534 --- /dev/null +++ b/audit-worker-versions/report.go @@ -0,0 +1,496 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "html" + "net/url" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "text/template" + "time" +) + +type count struct { + Key string + Value int + Hover string +} + +type reportSection struct { + Title string + TotalLabel string + Description string + Count int + Versions []count + Images []count + Filtered []WorkerInfo + FullColumns bool + HasLegacyTotals bool +} + +type reportData struct { + GeneratedAt string + ProbeStartedAt string + TaskGroupURL string + ReportGeneratedAt string + Revision string + RevisionURL string + Sections [5]reportSection +} + +// Keep offline report rendering usable without Taskcluster configuration. +func reportTaskclusterRootURL() string { + root := os.Getenv("TASKCLUSTER_ROOT_URL") + if root == "" { + root = "https://firefox-ci-tc.services.mozilla.com" + } + return strings.TrimRight(root, "/") +} + +func (w WorkerInfo) WorkerPoolURL() string { + parts := strings.SplitN(w.WorkerPoolID, "/", 2) + if len(parts) != 2 { + return "" + } + return reportTaskclusterRootURL() + "/provisioners/" + + url.PathEscape(parts[0]) + "/worker-types/" + url.PathEscape(parts[1]) + + "?sortBy=Last%20Active&sortDirection=desc" +} + +func formatConfiguredRange(minimum, maximum *int) string { + if minimum == nil || maximum == nil { + return "—" + } + if *minimum == *maximum { + return fmt.Sprintf("%d", *minimum) + } + return fmt.Sprintf("%d–%d", *minimum, *maximum) +} + +func (w WorkerInfo) ConfiguredWorkerRange() string { + return formatConfiguredRange(w.ConfiguredMinWorkers, w.ConfiguredMaxWorkers) +} + +func (w WorkerInfo) ConfiguredCapacityRange() string { + return formatConfiguredRange(w.ConfiguredMinCapacity, w.ConfiguredMaxCapacity) +} + +func (w WorkerInfo) CapacityPerWorkerValue() string { + if w.CapacityPerWorker == nil { + return "—" + } + return fmt.Sprintf("%d", *w.CapacityPerWorker) +} + +const readmeTpl = ` +{{- define "row" -}} +## {{ .Title }} +{{ if .Description }} +{{ .Description }} +{{ end }} + +Total {{ .TotalLabel }}: ` + "`" + `{{ .Count }}` + "`" + ` +{{ if gt (len .Versions) 1 }} +### Worker pools by version + +_Source: version information parsed from the log artifact produced when each worker claims an intentionally malformed probe task. The task is expected to fail with a malformed-payload exception; known worker implementations and versions are identified from their distinct log output._ + +| Version | Worker pools | +| :--- | ---: | +{{ range .Versions -}} +| {{ .Key }} | {{ .Value }} | +{{ end }} +{{- end }} +{{ if gt (len .Images) 1 }} +### Worker pools by image + +_Source: image references in each pool's live Worker Manager launch configuration at report time. A value may represent multiple configured images. Standalone pools do not have a Worker Manager-managed image._ + +| Image | Worker pools | +| :--- | ---: | +{{ range .Images -}} +| {{ if .Hover }}{{ .Key }}{{ else }}{{ .Key }}{{ end }} | {{ .Value }} | +{{ end }} +{{- end }} +{{if .Count }} +### Worker pools + +_Configured capacity is the pool's autoscaling range in concurrent task slots. The configured worker range is derived from those bounds and the slots per worker, rounding up. An em dash means the configuration is unavailable or a worker count cannot be derived._ +{{ if .HasLegacyTotals }} +_Configured values were not collected in this snapshot. Legacy totals included stopped worker records and are intentionally not displayed._ +{{ end }} + +| Worker Pool | Implementation | Version {{ if .FullColumns }}| Engine | Revision | OS | Arch | GO {{ end }}| Configured Workers | Configured Capacity | Slots per Worker | +| --- | --- | --- {{ if .FullColumns }}| --- | --- | --- | --- | --- {{ end }}| ---: | ---: | ---: | +{{ range .Filtered -}} +| [**{{ .WorkerPoolID }}**]({{ .WorkerPoolURL }}) | {{ .Implementation }} | {{ or .Version .Details.error }} {{ if $.FullColumns }}| {{ or .Details.engine "-" }} | {{ or (slice .Details.revision 0 10) "-" }} | {{ or .Details.os "-" }} | {{ or .Details.arch "-" }} | {{ or .Details.go "-" }} {{ end }}| {{ .ConfiguredWorkerRange }} | {{ .ConfiguredCapacityRange }} | {{ .CapacityPerWorkerValue }} | +{{end}} +{{- end -}} +{{end}} + +# Worker Pool Versions + +This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots. + +- **Implementation and version** are inferred from the failure log produced when each pool receives an intentionally malformed probe task. +- **Image and capacity metadata** come from Worker Manager. +- **Summary values** are counts of worker pools, not individual workers or tasks. + +{{ define "generation" }}Report generated: **{{ .ReportGeneratedAt }}**{{ if .Revision }} ({{ if .RevisionURL }}[{{ .Revision }}]({{ .RevisionURL }}){{ else }}{{ .Revision }}{{ end }}){{ end }}{{ end }} +{{ if .ProbeStartedAt }}Probe run started: **{{ .ProbeStartedAt }}**{{ if .TaskGroupURL }} ([Taskcluster task group]({{ .TaskGroupURL }})){{ end }}{{ if .GeneratedAt }} · Results collected: **{{ .GeneratedAt }}**{{ end }}{{ if .ReportGeneratedAt }} · {{ template "generation" . }}{{ end }} +{{ else if .GeneratedAt }}Results collected: **{{ .GeneratedAt }}**{{ if .ReportGeneratedAt }} · {{ template "generation" . }}{{ end }} +{{ else if .ReportGeneratedAt }}{{ template "generation" . }} +{{ end }} + +{{ range .Sections }} +{{ template "row" . }} +{{ end }} +` + +func renderTemplate(data interface{}) string { + t := template.Must(template.New("").Parse(readmeTpl)) + var content bytes.Buffer + if err := t.Execute(&content, data); err != nil { + panic(err) + } + return strings.TrimSpace(content.String()) + "\n" +} + +func sortedCounts(values map[string]int) []count { + counts := make([]count, 0, len(values)) + for key, value := range values { + counts = append(counts, count{Key: key, Value: value}) + } + sort.Slice(counts, func(i, j int) bool { + return strings.Compare(counts[i].Key, counts[j].Key) < 0 + }) + return counts +} + +func naturalLess(left, right string) bool { + for leftIndex, rightIndex := 0, 0; leftIndex < len(left) && rightIndex < len(right); { + leftDigit := left[leftIndex] >= '0' && left[leftIndex] <= '9' + rightDigit := right[rightIndex] >= '0' && right[rightIndex] <= '9' + if leftDigit && rightDigit { + leftEnd, rightEnd := leftIndex, rightIndex + for leftEnd < len(left) && left[leftEnd] >= '0' && left[leftEnd] <= '9' { + leftEnd++ + } + for rightEnd < len(right) && right[rightEnd] >= '0' && right[rightEnd] <= '9' { + rightEnd++ + } + + leftSignificant, rightSignificant := leftIndex, rightIndex + for leftSignificant < leftEnd-1 && left[leftSignificant] == '0' { + leftSignificant++ + } + for rightSignificant < rightEnd-1 && right[rightSignificant] == '0' { + rightSignificant++ + } + + leftLength := leftEnd - leftSignificant + rightLength := rightEnd - rightSignificant + if leftLength != rightLength { + return leftLength < rightLength + } + if leftNumber, rightNumber := left[leftSignificant:leftEnd], right[rightSignificant:rightEnd]; leftNumber != rightNumber { + return leftNumber < rightNumber + } + if leftRunLength, rightRunLength := leftEnd-leftIndex, rightEnd-rightIndex; leftRunLength != rightRunLength { + return leftRunLength < rightRunLength + } + + leftIndex, rightIndex = leftEnd, rightEnd + continue + } + + if left[leftIndex] != right[rightIndex] { + return left[leftIndex] < right[rightIndex] + } + leftIndex++ + rightIndex++ + } + + return len(left) < len(right) +} + +func sortedVersionCounts(values map[string]int) []count { + counts := sortedCounts(values) + sort.Slice(counts, func(i, j int) bool { + return naturalLess(counts[j].Key, counts[i].Key) + }) + return counts +} + +func compactAzureImageReference(reference string) string { + parts := strings.Split(strings.Trim(reference, "/"), "/") + if len(parts) < 6 || !strings.EqualFold(parts[0], "subscriptions") { + return reference + } + + providerIsAzureCompute := false + for i := 0; i+1 < len(parts); i++ { + if strings.EqualFold(parts[i], "providers") && strings.EqualFold(parts[i+1], "Microsoft.Compute") { + providerIsAzureCompute = true + break + } + } + if !providerIsAzureCompute { + return reference + } + + for i := 0; i+5 < len(parts); i++ { + if strings.EqualFold(parts[i], "galleries") && + strings.EqualFold(parts[i+2], "images") && + strings.EqualFold(parts[i+4], "versions") { + gallery := parts[i+1] + image := parts[i+3] + version := parts[i+5] + if gallery == image { + return fmt.Sprintf("Azure gallery %s@%s", image, version) + } + return fmt.Sprintf("Azure gallery %s/%s@%s", gallery, image, version) + } + } + + for i := 0; i+1 < len(parts); i++ { + if strings.EqualFold(parts[i], "images") { + return "Azure image " + parts[i+1] + } + } + return reference +} + +func compactAzureImageSet(references []string) (string, bool) { + regions := make([]string, 0, len(references)) + family := "" + for _, reference := range references { + parts := strings.Split(strings.Trim(reference, "/"), "/") + if len(parts) < 2 || !strings.EqualFold(parts[0], "subscriptions") || + !strings.EqualFold(parts[len(parts)-2], "images") { + return "", false + } + nameParts := strings.Split(parts[len(parts)-1], "-") + if len(nameParts) < 4 || nameParts[0] != "imageset" { + return "", false + } + currentFamily := strings.Join(nameParts[3:], "-") + if family != "" && currentFamily != family { + return "", false + } + family = currentFamily + regions = append(regions, nameParts[2]) + } + if len(regions) < 2 { + return "", false + } + sort.Strings(regions) + return fmt.Sprintf("Azure image set %s (%s)", family, strings.Join(regions, ", ")), true +} + +func compactImageReferences(imageset string) string { + references := strings.Split(imageset, ",") + for i := range references { + references[i] = strings.TrimSpace(references[i]) + } + if compact, ok := compactAzureImageSet(references); ok { + return compact + } + for i, reference := range references { + references[i] = compactAzureImageReference(reference) + } + return strings.Join(references, ", ") +} + +func imageHoverTitle(imageset string) string { + references := strings.Split(imageset, ",") + for i, reference := range references { + references[i] = html.EscapeString(strings.TrimSpace(reference)) + } + return strings.Join(references, " ") +} + +func imageCountLabel(worker WorkerInfo) string { + switch worker.ImageStatus { + case imageStatusNotApplicable: + return "Not applicable (standalone)" + case imageStatusUnavailable: + return "Configuration unavailable" + case imageStatusNotDetermined: + return "Image not determined" + } + if worker.WorkerManagerLookupError != "" { + return "Configuration unavailable" + } + if worker.Imageset == "" || worker.Imageset == "unknown" { + return "Image not determined" + } + return compactImageReferences(worker.Imageset) +} + +func generateReadmeSection(title, description string, workers []WorkerInfo, filter func(WorkerInfo) bool) reportSection { + filtered := make([]WorkerInfo, 0) + versions := make(map[string]int) + imagesets := make(map[string]int) + imageHovers := make(map[string]string) + hasLegacyTotals := false + + for _, worker := range workers { + if filter(worker) { + filtered = append(filtered, worker) + versions[worker.Version]++ + imageLabel := imageCountLabel(worker) + imagesets[imageLabel]++ + if worker.ImageStatus == imageStatusKnown && imageLabel != worker.Imageset { + imageHovers[imageLabel] = worker.Imageset + } + hasLegacyTotals = hasLegacyTotals || worker.LegacyTotalWorkers != nil || worker.LegacyTotalCapacity != nil + } + } + + sort.Slice(filtered, func(i, j int) bool { + return strings.Compare(filtered[i].WorkerPoolID, filtered[j].WorkerPoolID) < 0 + }) + + images := sortedCounts(imagesets) + for i := range images { + if hover := imageHovers[images[i].Key]; hover != "" { + images[i].Key = html.EscapeString(images[i].Key) + images[i].Hover = imageHoverTitle(hover) + } + } + + return reportSection{ + Title: title, + TotalLabel: sectionTotalLabel(title), + Description: description, + Count: len(filtered), + Versions: sortedVersionCounts(versions), + Images: images, + Filtered: filtered, + FullColumns: title == "Generic Worker", + HasLegacyTotals: hasLegacyTotals, + } +} + +func sectionTotalLabel(title string) string { + switch title { + case "Generic Worker": + return "generic-worker pools" + case "Docker Worker": + return "docker-worker pools" + case "Script Worker": + return "scriptworker pools" + case "Unknown implementation": + return "pools with unknown implementation" + case "Unresponsive": + return "unresponsive worker pools" + default: + return "worker pools" + } +} + +func writeReadme(snapshot WorkerSnapshot) { + filename := filepath.Join(outputDir, "README.md") + WriteFile(filename, []byte(renderReadme(snapshot))) +} + +func renderReadme(snapshot WorkerSnapshot) string { + return renderReadmeAt(snapshot, time.Now()) +} + +// Read provenance before writing generated artifacts. Use origin so fork-only +// commits link to the repository that actually contains them. +func reportRevision(dir string) (revision, revisionURL string) { + git := func(args ...string) (string, error) { + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.Output() + return strings.TrimSpace(string(out)), err + } + sha, err := git("rev-parse", "--verify", "HEAD") + if err != nil || len(sha) < 9 { + return "", "" + } + status, err := git("status", "--porcelain", "--untracked-files=all", "--", ":/", ":(top,exclude)WorkerVersions/**", ":(top,exclude).beads/**", ":(top,exclude)docs/history.json") + if err != nil { + return "", "" // Do not claim a clean revision if status is unavailable. + } + revision = sha[:9] + if status != "" { + revision += "-dirty" + } + remote, err := git("remote", "get-url", "origin") + if err == nil { + remote = strings.TrimSuffix(remote, ".git") + remote = strings.Replace(remote, "git@github.com:", "https://github.com/", 1) + remote = strings.Replace(remote, "ssh://git@github.com/", "https://github.com/", 1) + if strings.HasPrefix(remote, "https://github.com/") { + revisionURL = strings.TrimRight(remote, "/") + "/commit/" + sha + } + } + return revision, revisionURL +} + +func renderReadmeAt(snapshot WorkerSnapshot, reportGeneratedAt time.Time) string { + workers := snapshot.Workers + sections := [5]reportSection{ + generateReadmeSection("Generic Worker", "These pools use generic-worker to execute tasks. Recent versions also support docker-worker payloads and Docker container execution through d2g. [Documentation](https://docs.taskcluster.net/docs/reference/workers/generic-worker) · [Source](https://github.com/taskcluster/taskcluster/tree/main/workers/generic-worker).", workers, func(w WorkerInfo) bool { return w.Implementation == "generic-worker" }), + generateReadmeSection("Docker Worker", "These pools use docker-worker to run tasks inside Docker containers. Docker-worker is fully deprecated, and its source repository has been removed. See the [worker migration timeline](https://taskcluster.github.io/mozilla-history/migration.html) for the transition to generic-worker.", workers, func(w WorkerInfo) bool { return w.Implementation == "docker-worker" }), + generateReadmeSection("Script Worker", "", workers, func(w WorkerInfo) bool { return strings.Contains(w.Implementation, "Scriptworker") }), + generateReadmeSection("Unknown implementation", "These pools claimed and resolved the probe task, but did not publish a worker log artifact. Their worker implementation and version could therefore not be identified.", workers, func(w WorkerInfo) bool { return w.hasNoArtifacts }), + generateReadmeSection("Unresponsive", "These pools did not claim the probe task, so their worker implementation and version could not be determined.", workers, func(w WorkerInfo) bool { return w.isUnknown }), + } + + const timestampFormat = "2006-01-02 15:04 UTC" + data := reportData{Sections: sections} + data.Revision, data.RevisionURL = reportRevision(".") + if snapshot.TaskGroupID != "" { + data.TaskGroupURL = reportTaskclusterRootURL() + "/tasks/groups/" + url.PathEscape(snapshot.TaskGroupID) + } + if !snapshot.GeneratedAt.IsZero() { + data.GeneratedAt = snapshot.GeneratedAt.UTC().Format(timestampFormat) + } + if !snapshot.ProbeStartedAt.IsZero() { + data.ProbeStartedAt = snapshot.ProbeStartedAt.UTC().Format(timestampFormat) + } + if !reportGeneratedAt.IsZero() { + data.ReportGeneratedAt = reportGeneratedAt.UTC().Format(timestampFormat) + } + + return renderTemplate(data) +} + +func readSnapshot(filename string) (WorkerSnapshot, error) { + contents, err := os.ReadFile(filename) + if err != nil { + return WorkerSnapshot{}, err + } + + var snapshot WorkerSnapshot + if strings.HasPrefix(strings.TrimSpace(string(contents)), "[") { + if err := json.Unmarshal(contents, &snapshot.Workers); err != nil { + return WorkerSnapshot{}, err + } + } else if err := json.Unmarshal(contents, &snapshot); err != nil { + return WorkerSnapshot{}, err + } + + // These flags are internal rendering state and are not serialized in the + // snapshot. Restore them from the persisted error value for offline renders. + for i := range snapshot.Workers { + switch snapshot.Workers[i].Details["error"] { + case "No artifacts found": + snapshot.Workers[i].hasNoArtifacts = true + case "Version not determined; task not (yet) claimed", "Version not determined; task was not claimed": + snapshot.Workers[i].isUnknown = true + } + } + + return snapshot, nil +} diff --git a/audit-worker-versions/report_test.go b/audit-worker-versions/report_test.go new file mode 100644 index 000000000..f51bdfc2e --- /dev/null +++ b/audit-worker-versions/report_test.go @@ -0,0 +1,372 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestReportRevision(t *testing.T) { + dir := t.TempDir() + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + write := func(name, content string) { + t.Helper() + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + } + if revision, link := reportRevision(dir); revision != "" || link != "" { + t.Fatal("expected no revision outside a Git repository") + } + git("init") + git("config", "user.name", "Report test") + git("config", "user.email", "report@example.com") + git("config", "commit.gpgsign", "false") + git("remote", "add", "origin", "git@github.com:example/reports.git") + write("source.go", "original") + write("WorkerVersions/README.md", "original report") + git("add", ".") + git("commit", "-m", "initial") + sha := git("rev-parse", "HEAD") + check := func(suffix string) { + t.Helper() + revision, link := reportRevision(dir) + if revision != sha[:9]+suffix || link != "https://github.com/example/reports/commit/"+sha { + t.Fatalf("reportRevision() = (%q, %q)", revision, link) + } + } + check("") + write("WorkerVersions/README.md", "regenerated") + write("WorkerVersions/workers.json", "{}") + write(".beads/issues.jsonl", "{}") + write("docs/history.json", "{}") + check("") + write("source.go", "modified") + check("-dirty") + git("add", "source.go") + check("-dirty") + git("restore", "--staged", "source.go") + git("restore", "source.go") + write("new-source.go", "new") + check("-dirty") +} + +func TestRenderRevisionBesideGenerationTime(t *testing.T) { + for _, revision := range []string{"697b57411", "697b57411-dirty"} { + for _, timings := range [][2]string{{"probe", "collected"}, {"", "collected"}, {"", ""}} { + got := renderTemplate(reportData{ + ProbeStartedAt: timings[0], GeneratedAt: timings[1], + ReportGeneratedAt: "2026-09-10 22:34 UTC", + Revision: revision, RevisionURL: "https://github.com/example/reports/commit/697b57411", + }) + want := "Report generated: **2026-09-10 22:34 UTC** ([" + revision + "](https://github.com/example/reports/commit/697b57411))" + if !strings.Contains(got, want) { + t.Fatalf("missing generation provenance %q", want) + } + } + } +} + +func TestWorkerPoolURL(t *testing.T) { + t.Setenv("TASKCLUSTER_ROOT_URL", "") + worker := WorkerInfo{WorkerPoolID: "releng-hardware/gecko-t-win7-32-hw"} + want := "https://firefox-ci-tc.services.mozilla.com/provisioners/releng-hardware/worker-types/gecko-t-win7-32-hw?sortBy=Last%20Active&sortDirection=desc" + if got := worker.WorkerPoolURL(); got != want { + t.Fatalf("WorkerPoolURL() = %q, want %q", got, want) + } +} + +func TestRenderReadmeTaskclusterRootURL(t *testing.T) { + for _, root := range []string{"", "https://stage.taskcluster.nonprod.cloudops.mozgcp.net", "https://stage.taskcluster.nonprod.cloudops.mozgcp.net/"} { + t.Run(root, func(t *testing.T) { + t.Setenv("TASKCLUSTER_ROOT_URL", root) + wantRoot := strings.TrimRight(root, "/") + if wantRoot == "" { + wantRoot = "https://firefox-ci-tc.services.mozilla.com" + } + got := renderReadmeAt(WorkerSnapshot{ + TaskGroupID: "probe/group", + ProbeStartedAt: time.Date(2026, time.September, 9, 7, 58, 29, 0, time.UTC), + Workers: []WorkerInfo{{WorkerPoolID: "example/pool name", Implementation: "generic-worker", Details: map[string]string{"revision": "1234567890"}}}, + }, time.Now()) + for _, path := range []string{ + "/provisioners/example/worker-types/pool%20name?sortBy=Last%20Active&sortDirection=desc", + "/tasks/groups/probe%2Fgroup", + } { + if !strings.Contains(got, "("+wantRoot+path+")") { + t.Errorf("report missing link %q", wantRoot+path) + } + } + }) + } +} + +func TestSortedVersionCountsUsesDescendingNaturalOrder(t *testing.T) { + versions := map[string]int{ + "100.0.1": 1, + "108.0.0": 1, + "9.10.0": 1, + "9.2.0": 1, + "99.2.0": 1, + } + want := []string{"108.0.0", "100.0.1", "99.2.0", "9.10.0", "9.2.0"} + + got := sortedVersionCounts(versions) + for i := range want { + if got[i].Key != want[i] { + t.Fatalf("version %d = %q, want %q", i, got[i].Key, want[i]) + } + } +} + +func TestRenderReadmeIncludesLinksAndSubheadings(t *testing.T) { + t.Setenv("TASKCLUSTER_ROOT_URL", "") + workers := []WorkerInfo{{ + WorkerPoolID: "example/pool", + Implementation: "generic-worker", + Version: "1.2.3", + Imageset: "image-one", + Details: map[string]string{"revision": "1234567890"}, + ConfiguredMinCapacity: intPointer(1), + ConfiguredMaxCapacity: intPointer(16), + CapacityPerWorker: intPointer(8), + ConfiguredMinWorkers: intPointer(1), + ConfiguredMaxWorkers: intPointer(2), + }, { + WorkerPoolID: "example/other-pool", + Implementation: "generic-worker", + Version: "2.0.0", + Imageset: "image-two", + Details: map[string]string{"revision": "1234567890"}, + }} + + got := renderReadmeAt(WorkerSnapshot{ + TaskGroupID: "AnhEjBL2SYuUedNBvjgsWA", + GeneratedAt: time.Date(2026, time.September, 9, 15, 29, 53, 0, time.UTC), + ProbeStartedAt: time.Date(2026, time.September, 9, 7, 58, 29, 0, time.UTC), + Workers: workers, + }, time.Date(2026, time.September, 10, 1, 2, 3, 0, time.UTC)) + for _, want := range []string{ + "This report shows the latest detailed inventory of Firefox CI worker pools alongside historical trends from earlier snapshots.", + "- **Implementation and version** are inferred from the failure log produced when each pool receives an intentionally malformed probe task.", + "- **Image and capacity metadata** come from Worker Manager.", + "- **Summary values** are counts of worker pools, not individual workers or tasks.", + "Probe run started: **2026-09-09 07:58 UTC** ([Taskcluster task group](https://firefox-ci-tc.services.mozilla.com/tasks/groups/AnhEjBL2SYuUedNBvjgsWA)) · Results collected: **2026-09-09 15:29 UTC** · Report generated: **2026-09-10 01:02 UTC**", + "Total generic-worker pools: `2`", + "Total docker-worker pools: `0`", + "Total scriptworker pools: `0`", + "Total pools with unknown implementation: `0`", + "Total unresponsive worker pools: `0`", + "### Worker pools by version", + "### Worker pools by image", + "intentionally malformed probe task", + "expected to fail with a malformed-payload exception", + "live Worker Manager launch configuration", + "### Worker pools", + "Configured Workers | Configured Capacity | Slots per Worker", + "pool's autoscaling range in concurrent task slots", + "| 1–2 | 1–16 | 8 |", + "[**example/pool**](https://firefox-ci-tc.services.mozilla.com/provisioners/example/worker-types/pool?sortBy=Last%20Active&sortDirection=desc)", + } { + if !strings.Contains(got, want) { + t.Errorf("rendered README does not contain %q", want) + } + } +} + +func TestRenderReadmeWithoutTaskGroup(t *testing.T) { + got := renderReadmeAt(WorkerSnapshot{ + ProbeStartedAt: time.Date(2026, time.September, 9, 7, 58, 29, 0, time.UTC), + }, time.Date(2026, time.September, 10, 1, 2, 3, 0, time.UTC)) + want := "Probe run started: **2026-09-09 07:58 UTC** · Report generated: **2026-09-10 01:02 UTC**" + if !strings.Contains(got, want) || strings.Contains(got, "Taskcluster task group") { + t.Fatal("report without a task group should retain its timing line without a link") + } +} + +func TestRenderReadmeDistinguishesImageStatuses(t *testing.T) { + details := map[string]string{"revision": "1234567890"} + workers := []WorkerInfo{ + {Implementation: "generic-worker", Imageset: "unknown", ImageStatus: imageStatusNotApplicable, Details: details}, + {Implementation: "generic-worker", Imageset: "unknown", ImageStatus: imageStatusUnavailable, Details: details}, + {Implementation: "generic-worker", Imageset: "unknown", ImageStatus: imageStatusNotDetermined, Details: details}, + {Implementation: "generic-worker", Imageset: "azure/image", ImageStatus: imageStatusKnown, Details: details}, + } + + got := renderReadme(WorkerSnapshot{Workers: workers}) + + for _, want := range []string{ + "| Image | Worker pools |", + "| Not applicable (standalone) | 1 |", + "| Configuration unavailable | 1 |", + "| Image not determined | 1 |", + "| azure/image | 1 |", + } { + if !strings.Contains(got, want) { + t.Errorf("rendered README does not contain %q", want) + } + } +} + +func TestCompactAzureImageReferences(t *testing.T) { + tests := map[string]string{ + "gallery with redundant image name": "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/galleries/win2022/images/win2022/versions/1.0.0", + "gallery with distinct image name": "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/galleries/gallery/images/image/versions/2.0", + "managed image": "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/imageset-worker-eastus", + "non-Azure image": "projects/example/global/images/worker-image", + } + want := map[string]string{ + "gallery with redundant image name": "Azure gallery win2022@1.0.0", + "gallery with distinct image name": "Azure gallery gallery/image@2.0", + "managed image": "Azure image imageset-worker-eastus", + "non-Azure image": "projects/example/global/images/worker-image", + } + + for name, input := range tests { + t.Run(name, func(t *testing.T) { + if got := compactAzureImageReference(input); got != want[name] { + t.Fatalf("compactAzureImageReference() = %q, want %q", got, want[name]) + } + }) + } +} + +func TestCompactImageReferencesPreservesConfiguredSet(t *testing.T) { + input := "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/one,/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/two" + want := "Azure image one, Azure image two" + if got := compactImageReferences(input); got != want { + t.Fatalf("compactImageReferences() = %q, want %q", got, want) + } +} + +func TestCompactImageReferencesSummarizesRegionalAzureSet(t *testing.T) { + input := strings.Join([]string{ + "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/imageset-abcdefghijklmnopqrst-westus2-fuzzing", + "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/images/imageset-zyxwvutsrqponmlkjihg-eastus-fuzzing", + }, ",") + want := "Azure image set fuzzing (eastus, westus2)" + if got := compactImageReferences(input); got != want { + t.Fatalf("compactImageReferences() = %q, want %q", got, want) + } +} + +func TestImageHoverTitleSeparatesImageSetWithLineBreaks(t *testing.T) { + if got := imageHoverTitle("/azure/one,/azure/two"); got != "/azure/one /azure/two" { + t.Fatalf("imageHoverTitle() = %q, want encoded line break", got) + } +} + +func TestRenderReadmeShowsFullAzureImageOnHover(t *testing.T) { + fullImage := "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/galleries/win2022/images/win2022/versions/1.0.0" + workers := []WorkerInfo{ + {Implementation: "generic-worker", Imageset: fullImage, ImageStatus: imageStatusKnown, Details: map[string]string{"revision": "1234567890"}}, + {Implementation: "generic-worker", Imageset: "projects/example/global/images/linux", ImageStatus: imageStatusKnown, Details: map[string]string{"revision": "1234567890"}}, + } + + got := renderReadme(WorkerSnapshot{Workers: workers}) + + want := `Azure gallery win2022@1.0.0` + if !strings.Contains(got, want) { + t.Fatalf("rendered README does not contain Azure hover label %q", want) + } + if strings.Contains(got, ` { + const workers = Array.isArray(data) ? data : data.workers const implementations = {} const versions = {} const imagesets = {} @@ -20,7 +21,7 @@ // indistinguishable once merged into a single bucket const versionsByImplementation = {} const inc = (dict, key) => dict[key] = (dict[key] || 0) + 1 - data.forEach(worker => { + workers.forEach(worker => { inc(implementations, worker.Implementation) inc(versions, worker.Version) inc(imagesets, worker.Imageset) @@ -60,7 +61,7 @@ do revdate=$(git show --no-patch --no-notes --date=short --pretty='%cd' "$rev") echo "Fetching ${revdate} version ${rev}" git show "${rev}:${WORKERS_FILE}" > "${PUBLIC_DIR}/data/${revdate}.json" -done < <(git rev-list master -- "${WORKERS_FILE}") +done < <(git rev-list --reverse HEAD -- "${WORKERS_FILE}") node -e "${nodescript}" "${PWD}" ${PUBLIC_DIR}/data/*.json > $HISTORICAL_DATA rm -rf "${PUBLIC_DIR}/data/" diff --git a/docs/index-local.html b/docs/index-local.html new file mode 100644 index 000000000..f9d468b15 --- /dev/null +++ b/docs/index-local.html @@ -0,0 +1,20 @@ + + + + + + Local Worker Version Report + + +

Opening the worker version report with local data…

+ + + + diff --git a/docs/index.html b/docs/index.html index a60e5aa1d..9f0e09b6d 100644 --- a/docs/index.html +++ b/docs/index.html @@ -18,38 +18,235 @@ } main { - margin: 20px 40px; + margin: 20px 0; } - .content { - margin-right: 250px; + .report-layout { + display: grid; + grid-template-columns: max-content minmax(0, 1fr); + gap: 12px; + align-items: start; } - @media screen and (max-width: 1400px) { - .content { - margin-right: auto; - } + .content-column { + min-width: 0; + box-sizing: border-box; + padding-right: 40px; } .toc { - position: fixed; - right: 15px; - top: 5px; - background: #fff; - padding: 15px 25px; - border: 1px solid #eee; - /* above the sticky table headers/columns from table-utils.js */ - z-index: 10; + position: sticky; + top: 20px; + max-height: calc(100vh - 40px); + overflow-y: auto; + box-sizing: border-box; + padding: 0 20px 12px 40px; + border-right: 1px solid #d0d7de; + } + + .toc-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 8px; + padding-bottom: 8px; + } + + .toc-title { + padding-top: 16px; + font-size: 16px; + font-weight: 700; + } + + .toc-title::after { + display: block; + width: 100%; + margin-top: 8px; + border-bottom: 2px solid #d0d7de; + content: ""; + } + + .toc-toggle { + flex: none; + display: inline-flex; + width: 32px; + height: 32px; + padding: 0; + align-items: center; + justify-content: center; + border: 1px solid #d0d7de; + border-radius: 6px; + background: #f6f8fa; + color: #24292f; + font: inherit; + cursor: pointer; + } + + .toc-toggle svg { + width: 16px; + height: 16px; + fill: currentColor; + } + + .toc-toggle .toc-expand-icon, + .toc-toggle[aria-expanded="false"] .toc-collapse-icon { + display: none; + } + + .toc-toggle[aria-expanded="false"] .toc-expand-icon { + display: block; + } + + .report-layout:not(.toc-collapsed) .toc-toggle { + margin-top: 11px; + } + + .toc-toggle:hover { + background: #eef1f4; + } + + .toc-toggle:focus-visible { + outline: 2px solid #0969da; + outline-offset: 2px; + } + + .report-layout.toc-collapsed { + grid-template-columns: 32px minmax(0, 1fr); + column-gap: 16px; + padding-left: 20px; + } + + .report-layout.toc-collapsed .toc { + width: 32px; + top: 31px; + padding: 0; + overflow: visible; + border-right: 0; + } + + .report-layout.toc-collapsed .toc-title { + display: none; + } + + .report-layout.toc-collapsed .toc-header { + padding-bottom: 0; + } + + .report-layout.toc-collapsed .content-column { + padding-left: 20px; + } + + @media screen and (max-width: 900px) { + main { + margin: 16px; + } + + .report-layout, + .report-layout.toc-collapsed { + grid-template-columns: minmax(0, 1fr); + gap: 12px; + padding-left: 0; + } + + .report-layout .toc { + position: static; + width: 100%; + margin-left: 0; + max-height: none; + overflow: visible; + padding: 0 0 12px; + border-right: 0; + border-bottom: 1px solid #d0d7de; + } + + .toc-title { + padding-top: 6px; + } + + .report-layout.toc-collapsed .toc { + width: 32px; + top: auto; + padding: 0; + border-bottom: 0; + } + + .report-layout .content-column, + .report-layout.toc-collapsed .content-column { + padding-right: 0; + padding-left: 0; + } } .toc ul { - margin-left: 20px; - padding-left: 0; + margin: 8px 0 0; + padding: 0; + list-style: none; } .toc ul ul { - margin-left: auto; - padding-left: 15px; + margin-top: 3px; + padding-left: 12px; + } + + .toc li + li { + margin-top: 4px; + } + + .toc li li + li { + margin-top: 2px; + } + + .toc a { + color: #24292f; + line-height: 1.35; + text-decoration: none; + } + + .toc > ul > li > a { + font-weight: 600; + } + + .toc ul ul a { + color: #57606a; + font-size: 0.92em; + } + + @media screen and (min-width: 901px) { + .toc a { + white-space: nowrap; + } + } + + .heading-permalink { + margin-left: 0.35em; + color: #57606a; + font-size: 0.75em; + font-weight: 400; + opacity: 0.35; + text-decoration: none; + } + + h2:hover > .heading-permalink, + h3:hover > .heading-permalink, + .heading-permalink:hover, + .heading-permalink:focus-visible { + opacity: 1; + } + + .heading-permalink:hover { + color: #0969da; + text-decoration: none; + } + + .heading-permalink:focus-visible { + outline: 2px solid #0969da; + outline-offset: 2px; + } + + @media (hover: none) { + .heading-permalink { + opacity: 0.65; + } } .row { @@ -74,10 +271,22 @@ width: 70%; } + .stale-probe-notice { + display: flex; + align-items: baseline; + gap: 0.6rem; + padding: 0.75rem 1rem; + margin: 0.75rem 0 1rem; + background: #fff8e6; + border: 1px solid #e6cb83; + border-radius: 6px; + color: #684b10; + } + .source { margin: 0 0 10px; - color: #656d76; - font-size: 14px; + color: inherit; + font-size: inherit; } .source span + span::before { @@ -194,40 +403,35 @@
-
- -

-

Looking for the big picture? The docker-worker → - generic-worker migration timeline animates the fleet balance and the version rollout month by - month.

-

Disclaimer: Values below represent worker pools.
- They do not contain number of unique imagesets used for those pools, or how many tasks are being executed by those - worker pools.

-
-
-
-

Disclaimer: Those values represent number of worker pools, and not number of - workers. This also doesn't include total number of tasks run on those worker pools.

-

Loading charts…

-
-
+
+ +
+
+
+
+

Graphs

+

Each point shows the number of worker pools observed in a snapshot.

+

Loading charts…

+

Worker pools by implementation

+
+

Worker pools by version

+
+
@@ -238,14 +442,29 @@ crossorigin="anonymous" referrerpolicy="no-referrer"> diff --git a/docs/index.test.mjs b/docs/index.test.mjs new file mode 100644 index 000000000..205214acd --- /dev/null +++ b/docs/index.test.mjs @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import test from 'node:test' + +const html = await readFile(new URL('./index.html', import.meta.url), 'utf8') + +test('the report page has one syntactically valid inline module', () => { + const modules = [...html.matchAll(/ + diff --git a/test_fetch_and_generate.py b/test_fetch_and_generate.py new file mode 100644 index 000000000..60bcfc76d --- /dev/null +++ b/test_fetch_and_generate.py @@ -0,0 +1,67 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from fetch_and_generate import load_taskcluster_token + + +class LoadTaskclusterTokenTests(unittest.TestCase): + def test_loads_missing_credentials_from_json(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + token_file = Path(temp_dir) / ".tc_token" + token_file.write_text( + json.dumps({"clientId": "file-client", "accessToken": "file-token"}), + encoding="utf-8", + ) + env: dict[str, str] = {} + + loaded = load_taskcluster_token(env, token_file) + + self.assertTrue(loaded) + self.assertEqual(env["TASKCLUSTER_CLIENT_ID"], "file-client") + self.assertEqual(env["TASKCLUSTER_ACCESS_TOKEN"], "file-token") + + def test_token_file_takes_precedence_over_environment(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + token_file = Path(temp_dir) / ".tc_token" + token_file.write_text( + json.dumps({"clientId": "file-client", "accessToken": "file-token"}), + encoding="utf-8", + ) + env = { + "TASKCLUSTER_CLIENT_ID": "environment-client", + "TASKCLUSTER_ACCESS_TOKEN": "environment-token", + } + + loaded = load_taskcluster_token(env, token_file) + + self.assertTrue(loaded) + self.assertEqual(env["TASKCLUSTER_CLIENT_ID"], "file-client") + self.assertEqual(env["TASKCLUSTER_ACCESS_TOKEN"], "file-token") + + def test_environment_is_unchanged_when_token_file_is_absent(self) -> None: + env = { + "TASKCLUSTER_CLIENT_ID": "environment-client", + "TASKCLUSTER_ACCESS_TOKEN": "environment-token", + } + + loaded = load_taskcluster_token(env, Path("/does/not/exist/.tc_token")) + + self.assertFalse(loaded) + self.assertEqual(env["TASKCLUSTER_CLIENT_ID"], "environment-client") + self.assertEqual(env["TASKCLUSTER_ACCESS_TOKEN"], "environment-token") + + def test_rejects_malformed_json_when_credentials_are_needed(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + token_file = Path(temp_dir) / ".tc_token" + token_file.write_text("not json", encoding="utf-8") + + with self.assertRaisesRegex( + RuntimeError, "could not read Taskcluster token file" + ): + load_taskcluster_token({}, token_file) + + +if __name__ == "__main__": + unittest.main()