77import argparse
88import hashlib
99import json
10+ import os
1011from dataclasses import dataclass
1112from pathlib import Path
12- from urllib .parse import urlparse
13+ from urllib .parse import quote , urlparse
1314from urllib .request import Request , urlopen
1415
1516
@@ -19,6 +20,9 @@ class Source:
1920 url : str
2021 sha256 : str
2122 destination : Path
23+ repository : str | None = None
24+ ref : str | None = None
25+ path : str | None = None
2226
2327
2428def load_sources (manifest_path : Path ) -> list [Source ]:
@@ -46,23 +50,34 @@ def load_sources(manifest_path: Path) -> list[Source]:
4650 raise ValueError (f"sources[{ index } )
4751 _validate_immutable_url (url , f"sources[{ index } )
4852
53+ repository = _optional_string (raw .get ("repository" ), f"sources[{ index } )
54+ ref = _optional_string (raw .get ("ref" ), f"sources[{ index } )
55+ path = _optional_string (raw .get ("path" ), f"sources[{ index } )
56+ tracking = (repository , ref , path )
57+ if any (value is not None for value in tracking ) and not all (
58+ value is not None for value in tracking
59+ ):
60+ raise ValueError (
61+ f"sources[{ index }
62+ )
63+
4964 sources .append (
5065 Source (
5166 name = name ,
5267 url = url ,
5368 sha256 = digest ,
5469 destination = Path (destination ),
70+ repository = repository ,
71+ ref = ref ,
72+ path = path ,
5573 )
5674 )
5775
5876 return sources
5977
6078
6179def fetch (source : Source ) -> bytes :
62- request = Request (source .url , headers = {"User-Agent" : "github-workflows-sync" })
63- with urlopen (request , timeout = 30 ) as response :
64- content = response .read ()
65-
80+ content = _download (source .url )
6681 actual = hashlib .sha256 (content ).hexdigest ()
6782 if actual != source .sha256 :
6883 raise ValueError (
@@ -91,6 +106,78 @@ def check(sources: list[Source], root: Path) -> None:
91106 raise ValueError ("generated templates are out of date: " + ", " .join (drift ))
92107
93108
109+ def refresh (manifest_path : Path , root : Path , token : str | None = None ) -> None :
110+ payload = json .loads (manifest_path .read_text (encoding = "utf-8" ))
111+ raw_sources = payload .get ("sources" )
112+ if not isinstance (raw_sources , list ):
113+ raise ValueError ("manifest.sources must be an array" )
114+
115+ # Validate the current manifest before mutating it.
116+ load_sources (manifest_path )
117+
118+ for index , raw in enumerate (raw_sources ):
119+ if not isinstance (raw , dict ):
120+ raise ValueError (f"manifest.sources[{ index } )
121+
122+ repository = raw .get ("repository" )
123+ ref = raw .get ("ref" )
124+ path = raw .get ("path" )
125+ if not all (isinstance (value , str ) and value for value in (repository , ref , path )):
126+ continue
127+
128+ commit = _latest_commit (repository , ref , path , token )
129+ url = f"https://raw.githubusercontent.com/{ repository } { commit } { path }
130+ content = _download (url )
131+ digest = hashlib .sha256 (content ).hexdigest ()
132+
133+ raw ["url" ] = url
134+ raw ["sha256" ] = digest
135+
136+ destination = _safe_destination (root , Path (str (raw ["destination" ])))
137+ destination .parent .mkdir (parents = True , exist_ok = True )
138+ destination .write_bytes (content )
139+
140+ manifest_path .write_text (
141+ json .dumps (payload , indent = 2 , sort_keys = False ) + "\n " ,
142+ encoding = "utf-8" ,
143+ )
144+
145+
146+ def _latest_commit (repository : str , ref : str , path : str , token : str | None ) -> str :
147+ url = (
148+ f"https://api.github.com/repos/{ repository }
149+ f"?sha={ quote (ref , safe = '' )} { quote (path , safe = '' )}
150+ )
151+ headers = {
152+ "Accept" : "application/vnd.github+json" ,
153+ "User-Agent" : "github-workflows-sync" ,
154+ "X-GitHub-Api-Version" : "2022-11-28" ,
155+ }
156+ if token :
157+ headers ["Authorization" ] = f"Bearer { token }
158+
159+ request = Request (url , headers = headers )
160+ with urlopen (request , timeout = 30 ) as response :
161+ payload = json .load (response )
162+
163+ if not isinstance (payload , list ) or not payload :
164+ raise ValueError (
165+ f"cannot resolve latest commit for { repository } { ref } { path }
166+ )
167+ commit = payload [0 ].get ("sha" )
168+ if not isinstance (commit , str ) or len (commit ) != 40 :
169+ raise ValueError (
170+ f"invalid commit returned for { repository } { ref } { path }
171+ )
172+ return commit
173+
174+
175+ def _download (url : str ) -> bytes :
176+ request = Request (url , headers = {"User-Agent" : "github-workflows-sync" })
177+ with urlopen (request , timeout = 30 ) as response :
178+ return response .read ()
179+
180+
94181def _validate_immutable_url (url : str , path : str ) -> None :
95182 parsed = urlparse (url )
96183 if parsed .scheme != "https" :
@@ -125,20 +212,35 @@ def _non_empty_string(value: object, path: str) -> str:
125212 return value
126213
127214
215+ def _optional_string (value : object , path : str ) -> str | None :
216+ if value is None :
217+ return None
218+ if not isinstance (value , str ) or not value :
219+ raise ValueError (f"{ path } )
220+ return value
221+
222+
128223def main () -> int :
129224 parser = argparse .ArgumentParser ()
130- parser .add_argument ("command" , choices = ("sync" , "check" ))
225+ parser .add_argument ("command" , choices = ("sync" , "check" , "refresh" ))
131226 parser .add_argument ("manifest" , type = Path )
132227 args = parser .parse_args ()
133228
134229 root = Path .cwd ()
135- sources = load_sources (args .manifest )
136230
137231 try :
138- if args .command == "sync" :
139- sync (sources , root )
232+ if args .command == "refresh" :
233+ refresh (
234+ args .manifest ,
235+ root ,
236+ token = os .environ .get ("GITHUB_TOKEN" ),
237+ )
140238 else :
141- check (sources , root )
239+ sources = load_sources (args .manifest )
240+ if args .command == "sync" :
241+ sync (sources , root )
242+ else :
243+ check (sources , root )
142244 except ValueError as error :
143245 parser .error (str (error ))
144246
0 commit comments