twitcher/.github/workflows/queue_commits.yml at main · bevyengine/twitcher · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: Queue Commits From Bevy Main

on:
  workflow_dispatch:
  schedule:
    - cron: "7,27,47 * * * *"

env:
  SUITES: "build benches rendering"

jobs:
  queue-latest-commit:
    name: Queue Latest Commit
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout Bevy main branch
        uses: actions/checkout@v4
        with:
          repository: "bevyengine/bevy"
          ref: "main"
      - uses: actions/checkout@v4
        with:
          ref: "queue"
          path: "queue"
      - uses: actions/checkout@v4
        with:
          ref: "results"
          path: "results"
      - name: Queue latest commit
        id: queue
        run: |
          set -euo pipefail
          gitref=$(git rev-parse HEAD)
          added=0
          for suite in $SUITES; do
            # Already waiting for this suite?
            if [ -e "queue/$suite/$gitref" ]; then
              echo "$suite: already queued"
              continue
            fi
            dir="results/${gitref:0:1}/${gitref:1:1}/$gitref"
            if [ -f "$dir/stats.$suite.json" ] || [ -f "$dir/stats.json" ]; then
              echo "$suite: already collected"
              continue
            fi
            mkdir -p "queue/$suite"
            touch "queue/$suite/$gitref"
            echo "$suite: queued $gitref"
            added=$((added + 1))
          done
          echo "ADDED=$added" >> "$GITHUB_OUTPUT"
      - name: Commit
        if: steps.queue.outputs.ADDED > 0
        run: |
          cd queue
          git config user.name 'Workflow'
          git config user.email '<>'

          git add .
          git commit -m "Queued new commit for ${{ steps.queue.outputs.ADDED }} suite(s)"
          git push

  queue-history-commits:
    needs: [queue-latest-commit]
    name: Backfill History Per Suite
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout Bevy main branch
        uses: actions/checkout@v4
        with:
          repository: "bevyengine/bevy"
          ref: "main"
          path: "bevy"
          fetch-depth: 1000
      - uses: actions/checkout@v4
        with:
          ref: "queue"
          path: "queue"
      - uses: actions/checkout@v4
        with:
          ref: "results"
          path: "results"
      - uses: actions/checkout@v4
        with:
          path: "twitcher"
      - name: Backfill each idle suite
        id: queue
        run: |
          set -euo pipefail
          max=3
          added=0
          summary=""

          commits=$(cd bevy && git log --no-abbrev-commit --pretty=format:%H | sort)

          busy=""
          owners=0
          for machine in twitcher/machines/*.toml; do
            owners=$((owners + 1))
            owned=$(sed -n 's/^suites *= *\[\(.*\)\].*/\1/p' "$machine" | tr -d '" ' | tr ',' ' ')
            pending=""
            for suite in $owned; do
              [ -d "queue/$suite" ] && pending="${pending:+$pending }$suite"
            done
            if [ -n "$pending" ]; then
              echo "$(basename "$machine" .toml): queued work in $pending, holding back: $owned"
              busy="${busy:+$busy }$owned"
            fi
          done
          for suite in $SUITES; do
            if [ -d "queue/$suite" ]; then
              echo "$suite: still queued, not backfilling"
              continue
            fi
            case " $busy " in
              *" $suite "*)
                echo "$suite: idle, but its machine has queued work, not backfilling"
                continue
                ;;
            esac

            i=0
            missing=0
            for commit in $commits; do
              dir="results/${commit:0:1}/${commit:1:1}/$commit"
              if [ -f "$dir/stats.$suite.json" ] || [ -f "$dir/stats.json" ]; then
                continue
              fi
              missing=$((missing + 1))
              if [ "$i" -lt "$max" ]; then
                mkdir -p "queue/$suite"
                touch "queue/$suite/$commit"
                i=$((i + 1))
              fi
            done
            echo "$suite: queued $i of $missing missing commits"
            added=$((added + i))
            [ "$i" -gt 0 ] && summary="$summary $suite=$i/$missing"
          done

          echo "ADDED=$added" >> "$GITHUB_OUTPUT"
          echo "SUMMARY=$summary" >> "$GITHUB_OUTPUT"
      - name: Commit
        if: steps.queue.outputs.ADDED > 0
        run: |
          cd queue
          git config user.name 'Workflow'
          git config user.email '<>'

          git add .
          git commit -m "Backfilled history commits:${{ steps.queue.outputs.SUMMARY }}"
          git push