@@ -1336,15 +1344,15 @@
${service.displayName}
const response = await fetch('/api/build-info');
if (response.ok) {
const data = await response.json();
- const version = data.version || 'dev';
+ const version = data.version || 'unknown';
const buildDate = data.build_date || '';
const dateStr = buildDate ? `, ${buildDate}` : '';
const vPrefix = version.startsWith('v') ? '' : 'v';
document.getElementById('buildInfo').textContent =
- `Fleet Management ${vPrefix}${version}${dateStr}`;
+ `Fleet Manager image build: ${version === 'unknown' ? 'unavailable' : vPrefix + version}${dateStr}`;
}
} catch (e) {
- document.getElementById('buildInfo').textContent = 'Fleet Management (dev build)';
+ document.getElementById('buildInfo').textContent = 'Fleet Manager image build: unavailable';
}
}
@@ -1354,19 +1362,25 @@
${service.displayName}
}[char]));
}
+ let versionsEpoch = 0;
+ let legacyUpdateBranch = 'main';
+ let legacyServiceChannels = {};
// Load versions
async function loadVersions() {
const grid = document.getElementById('versionsGrid');
+ const epoch = ++versionsEpoch;
try {
const response = await fetch('/api/versions');
if (!response.ok) throw new Error('Failed to fetch versions');
const data = await response.json();
+ if (epoch !== versionsEpoch) return;
const settings = data._settings || {};
+ legacyUpdateBranch = settings.target_branch || 'main';
// Update branch selector
const branchSelect = document.getElementById('targetBranch');
- branchSelect.value = settings.target_branch || 'main';
+ // Candidate discovery filter never mutates installed component channels.
// Render version cards
let html = '';
@@ -1375,8 +1389,9 @@
${service.displayName}
for (const [name, info] of Object.entries(data)) {
if (name === '_settings') continue;
+ legacyServiceChannels[name] = info.selected_channel || legacyUpdateBranch;
const statusClass = info.running ? 'running' : 'stopped';
- const displayName = name.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
+ const displayName = name === 'cryptolabs-proxy' ? 'Fleet Manager (proxy)' : name.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
let versionText = '';
let detailsText = '';
@@ -1387,8 +1402,8 @@
${service.displayName}
const readable = installed && !['latest', 'main', 'dev', 'Pinned'].includes(installed)
? escapeVersionText(displayVersion) : '';
if (info.latest_status === 'latest') {
- versionText = '
✓ Latest ';
- if (readable) detailsText = `
${readable} `;
+ versionText = readable || (info.pinned ? 'Pinned image' : 'Installed image');
+ detailsText = `
✓ Latest verified image comparison (${escapeVersionText(legacyUpdateBranch)}) `;
} else {
versionText = readable || (info.pinned ? 'Pinned image' : 'Installed');
}
@@ -1402,6 +1417,8 @@
${service.displayName}
versionText = 'Not running';
}
+ detailsText = `
Observed image: ${versionText} ` + detailsText;
+ versionText = info.selected_release ? `Selected: ${escapeVersionText(info.selected_release)} (${escapeVersionText(info.selected_channel || 'main')})` : 'No selected release';
let updateBtn = '';
if (info.running && info.latest_status !== 'latest') {
updateBtn = `
${info.latest_status === 'update_available' ? 'Update available' : 'Unable to verify · Check Updates'} `;
@@ -1412,6 +1429,8 @@
${service.displayName}
updateBtn += '
Pinned image · managed separately ';
} else if (!info.running) {
updateBtn = '
Use service setup to start ';
+ } else if (info.selected_release) {
+ updateBtn += '
Use admitted releases above ';
} else if (info.update_available === true) {
updateBtn += `
⬆️ Update `;
updatableCount++;
@@ -1451,7 +1470,7 @@
${service.displayName}
}
// Enable/disable Update Running button based on whether any services need updating
- const updateAllBtn = document.querySelector('.update-buttons .btn-primary');
+ const updateAllBtn = document.querySelector('.legacy-update-all');
if (updateAllBtn) {
if (updatableCount > 0) {
updateAllBtn.disabled = false;
@@ -1470,25 +1489,14 @@
${service.displayName}
loadWatchdogStatus();
} catch (e) {
+ if (epoch !== versionsEpoch) return;
console.error('Failed to load versions:', e);
grid.innerHTML = `
Failed to load versions: ${e.message}
`;
}
}
async function saveBranchSetting() {
- const branch = document.getElementById('targetBranch').value;
- try {
- const response = await fetch('/api/update-settings', {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({ branch })
- });
- if (!response.ok) throw new Error('Failed to save');
- showUpdateStatus('info', `Target branch set to: ${branch}`);
- loadVersions();
- } catch (e) {
- showUpdateStatus('error', `Failed to save branch setting: ${e.message}`);
- }
+ return loadReleaseCandidates();
}
let activeUpdateJob = null;
@@ -1502,13 +1510,16 @@
${service.displayName}
} catch (e) { /* Storage may be disabled; this page can still poll. */ }
}
- async function pollUpdateJob(id, failures = 0) {
+ async function pollUpdateJob(id, failures = 0, expected = id) {
+ if (activeUpdateJob && activeUpdateJob !== expected) return;
activeUpdateJob = id;
clearTimeout(updatePollTimer);
try {
const response = await fetch(`/api/update-status?id=${encodeURIComponent(id)}`);
if (!response.ok) throw new Error('Update status unavailable');
const job = await response.json();
+ if (activeUpdateJob !== expected) return;
+ if (!id && job.id) { id = job.id; expected = job.id; activeUpdateJob = id; rememberUpdateJob(id); }
if (job.state === 'queued' || job.state === 'running') {
showUpdateStatus('info', 'Update operation in progress. Services restart only after their images are ready.');
updatePollTimer = setTimeout(() => pollUpdateJob(id), 2000);
@@ -1523,6 +1534,7 @@
${service.displayName}
loadVersions();
loadServices();
} catch (e) {
+ if (activeUpdateJob !== expected) return;
if (failures >= 60) {
showUpdateStatus('error', 'Update status is unavailable. Reload to reconnect before starting another update.');
return;
@@ -1537,12 +1549,12 @@
${service.displayName}
showUpdateStatus('info', 'An update operation is already in progress.');
return;
}
- const branch = document.getElementById('targetBranch').value;
+ const branch = legacyServiceChannels[service] || legacyUpdateBranch;
showUpdateStatus('info', action === 'pull' ? 'Checking images…' : 'Preparing update…');
try {
const response = await fetch(action === 'pull' ? '/api/pull' : '/api/update', {
method: 'POST',
- headers: { 'Content-Type': 'application/json' },
+ headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': await releaseToken() },
body: JSON.stringify({ service, branch })
});
const data = await response.json();
@@ -1590,6 +1602,206 @@
${service.displayName}
}
}
+ const RELEASE_OPERATION_KEY = 'fleet-release-operation';
+ const RELEASE_APPLY_KEY = 'fleet-release-apply';
+ let releaseSession = null;
+ let releaseListing = null;
+ let releasePreview = null;
+ let releaseExpected = null;
+ let releaseEpoch = 0;
+ let releaseOperationId = null;
+ let releasePollTimer = null;
+ let releaseBusy = false;
+
+ async function releaseToken() {
+ const response = await fetch('/api/releases/session', {cache:'no-store'});
+ const data = await response.json();
+ if (!response.ok) throw new Error(data.error || 'Current administrator access required.');
+ releaseSession = data;
+ return data.csrf_token;
+ }
+
+ async function releaseRequest(path, body) {
+ const options = {cache:'no-store'};
+ if (body !== undefined) {
+ options.method = 'POST';
+ options.headers = {'Content-Type':'application/json', 'X-CSRF-Token':await releaseToken()};
+ options.body = JSON.stringify(body);
+ }
+ const response = await fetch(path, options);
+ const data = await response.json();
+ if (!response.ok) {
+ const error = new Error(data.error || 'Release owner unavailable.');
+ error.status = response.status;
+ throw error;
+ }
+ return data;
+ }
+
+ async function loadReleaseCandidates() {
+ const epoch = ++releaseEpoch;
+ releasePreview = null;
+ document.getElementById('releaseApplyButton').disabled = true;
+ document.getElementById('releasePreviewButton').disabled = true;
+ const message = document.getElementById('releaseAvailability');
+ const container = document.getElementById('releaseCandidates');
+ container.replaceChildren();
+ document.getElementById('releasePreview').textContent = '';
+ try {
+ await releaseToken();
+ const channel = document.getElementById('targetBranch').value;
+ const listing = await releaseRequest('/api/releases/candidates?channel=' + encodeURIComponent(channel));
+ if (epoch !== releaseEpoch) return;
+ releaseListing = listing;
+ releaseExpected = listing.selected;
+ const choices = new Map();
+ for (const candidate of listing.candidates || []) {
+ for (const component of candidate.components || []) {
+ if (component.channel !== channel) continue;
+ if (!choices.has(component.component)) choices.set(component.component, []);
+ choices.get(component.component).push({candidate, component});
+ }
+ }
+ for (const [name, available] of choices) {
+ const row = document.createElement('div');
+ row.style.cssText = 'display:flex;flex-wrap:wrap;gap:.75rem;align-items:center;margin:.75rem 0';
+ const label = document.createElement('label');
+ const check = document.createElement('input');
+ check.type = 'checkbox'; check.dataset.component = name;
+ check.disabled = releaseBusy;
+ label.append(check, document.createTextNode(' ' + (name === 'cryptolabs-proxy' ? 'Fleet Manager (proxy)' : name)));
+ const select = document.createElement('select');
+ select.dataset.releaseComponent = name;
+ select.setAttribute('aria-label', name + ' admitted available release');
+ for (const {candidate, component} of available) {
+ const option = document.createElement('option');
+ option.value = candidate.candidate_id;
+ option.textContent = `${component.release || component.identity} (${component.channel})`;
+ select.append(option);
+ }
+ select.disabled = check.disabled;
+ const current = document.createElement('span');
+ const selected = listing.selected && listing.selected.components && listing.selected.components[name];
+ current.textContent = selected ? `Selected: ${selected.release || selected.identity} (${selected.channel})` : 'Selected release unavailable';
+ row.append(label, current, select);
+ container.append(row);
+ }
+ const admitted = choices.size > 0;
+ message.textContent = !admitted ? `No admitted ${channel === 'main' ? 'stable' : 'dev'} releases are available.`
+ : listing.writes_enabled ? 'Choose components, then review the exact update scope.'
+ : 'Admitted releases found. Updates remain disabled until the installed owner and trust checks pass.';
+ document.getElementById('releasePreviewButton').disabled = !admitted || releaseBusy;
+ // Dev is available only when the installed owner actually discovers an admitted dev release.
+ const dev = await releaseRequest('/api/releases/candidates?channel=dev').catch(() => ({candidates:[]}));
+ if (epoch !== releaseEpoch) return;
+ const option = document.querySelector('#targetBranch option[value="dev"]');
+ const hasDev = (dev.candidates || []).some(c => (c.components || []).some(p => p.channel === 'dev'));
+ option.disabled = !hasDev;
+ option.textContent = hasDev ? 'dev (admitted development)' : 'dev (unavailable until admitted)';
+ } catch (error) {
+ if (epoch !== releaseEpoch) return;
+ message.textContent = error.message + ' No update started.';
+ }
+ }
+
+ async function previewRelease() {
+ if (releaseBusy || !releaseListing || !releaseExpected) return;
+ const components = [], candidates = new Set();
+ for (const check of document.querySelectorAll('#releaseCandidates input:checked')) {
+ const name = check.dataset.component;
+ components.push(name);
+ const select = Array.from(document.querySelectorAll('#releaseCandidates select')).find(s => s.dataset.releaseComponent === name);
+ candidates.add(select.value);
+ }
+ const previewText = document.getElementById('releasePreview');
+ if (!components.length) { previewText.textContent = 'Choose at least one component.'; return; }
+ if (components.includes('cryptolabs-proxy') && components.length !== 1) {
+ previewText.textContent = 'Review Fleet Manager separately; mixed replacement groups require an admitted owner sequence.'; return;
+ }
+ const epoch = releaseEpoch;
+ try {
+ const result = await releaseRequest('/api/releases/preview', {
+ candidate_ids:Array.from(candidates), components,
+ expected_selection_id:releaseExpected.selection_id,
+ expected_selection_sha256:releaseExpected.selection_sha256,
+ operation_kind:components[0] === 'cryptolabs-proxy' ? 'proxy-adopt' : 'services-update'
+ });
+ if (epoch !== releaseEpoch || releaseBusy) return;
+ releasePreview = result;
+ previewText.textContent = `Update scope: ${result.components.join(', ')}\nTarget manifest: ${result.manifest_sha256}\n`
+ + (result.owner_plan ? `Fleet Manager target: ${result.owner_plan.image}\n` : '')
+ + (result.disconnect_expected ? 'Fleet Manager will briefly disconnect. Reconnect to this same operation. Retained rollback is owned by the host.\n' : '')
+ + `Preview: ${result.preview_id}`;
+ document.getElementById('releaseApplyButton').disabled = !releaseListing.writes_enabled;
+ } catch (error) { if (epoch === releaseEpoch) previewText.textContent = error.message; }
+ }
+
+ async function applyRelease() {
+ if (releaseBusy) return;
+ let body = null;
+ try { body = JSON.parse(localStorage.getItem(RELEASE_APPLY_KEY)); } catch (e) {}
+ if (!body) {
+ if (!releasePreview || !releaseExpected || !releaseListing.writes_enabled) return;
+ body = {operation_id:crypto.randomUUID(),preview_id:releasePreview.preview_id,preview_sha256:releasePreview.preview_sha256,
+ expected_selection_id:releaseExpected.selection_id,expected_selection_sha256:releaseExpected.selection_sha256};
+ }
+ releaseOperationId = body.operation_id;
+ try {
+ localStorage.setItem(RELEASE_OPERATION_KEY,releaseOperationId);
+ localStorage.setItem(RELEASE_APPLY_KEY,JSON.stringify(body));
+ } catch (e) {
+ document.getElementById('releaseOperation').textContent = 'Browser storage is required to retain this operation through a disconnect.'; return;
+ }
+ releaseBusy = true;
+ document.getElementById('releaseApplyButton').disabled = true;
+ document.getElementById('releasePreviewButton').disabled = true;
+ document.getElementById('releaseOperation').textContent = 'Submitting operation ' + releaseOperationId;
+ try { await releaseRequest('/api/releases/apply',body); }
+ catch (error) { document.getElementById('releaseOperation').textContent = 'Reply unavailable. Reconnecting to the retained operation; retry uses the same identity.'; }
+ return pollReleaseOperation(releaseOperationId);
+ }
+
+ async function pollReleaseOperation(id, failures = 0) {
+ if (releaseOperationId !== id) return;
+ clearTimeout(releasePollTimer);
+ try {
+ const operation = await releaseRequest('/api/releases/operations/' + encodeURIComponent(id));
+ if (releaseOperationId !== id) return;
+ const terminal = ['completed','failed-restored','failed','selection-conflict','recovery-required'].includes(operation.state);
+ document.getElementById('releaseOperation').textContent = `Operation ${id}: ${operation.state}` + (operation.reason ? ` · ${operation.reason}` : '');
+ document.getElementById('releaseRecoverButton').hidden = operation.state !== 'recovery-required';
+ if (!terminal) {
+ releaseBusy = true;
+ releasePollTimer = setTimeout(() => pollReleaseOperation(id),2000);
+ return;
+ }
+ releaseBusy = operation.state === 'recovery-required';
+ if (!releaseBusy) {
+ localStorage.removeItem(RELEASE_APPLY_KEY);
+ loadVersions(); loadServices(); loadReleaseCandidates();
+ }
+ } catch (error) {
+ if (releaseOperationId !== id) return;
+ document.getElementById('releaseOperation').textContent = error.status === 404
+ ? `Operation ${id} has no confirmed receipt. Retry Apply uses the same request.`
+ : `Reconnecting to operation ${id}. The installed owner retains progress through a disconnect.`;
+ if (error.status === 404) {
+ releaseBusy = false;
+ document.getElementById('releaseApplyButton').disabled = false;
+ return;
+ }
+ if (failures < 60) releasePollTimer = setTimeout(() => pollReleaseOperation(id,failures+1),3000);
+ }
+ }
+
+ async function recoverRelease() {
+ if (!releaseOperationId) return;
+ try {
+ await releaseRequest('/api/releases/operations/' + encodeURIComponent(releaseOperationId) + '/recover', {operation_id:releaseOperationId});
+ await pollReleaseOperation(releaseOperationId);
+ } catch (error) { document.getElementById('releaseOperation').textContent = error.message; }
+ }
+
// DC Watchdog agent status - integrated into versions grid (single card only)
let watchdogStatusLoading = false;
async function loadWatchdogStatus() {
@@ -1843,9 +2055,13 @@
${service.displayName}
loadServices();
loadBuildInfo();
loadVersions(); // This calls loadWatchdogStatus() at the end
+ loadReleaseCandidates();
try {
const pendingUpdate = localStorage.getItem(UPDATE_JOB_KEY);
if (pendingUpdate) pollUpdateJob(pendingUpdate);
+ else pollUpdateJob('').catch(() => {});
+ const operation = localStorage.getItem(RELEASE_OPERATION_KEY);
+ if (operation) { releaseOperationId = operation; releaseBusy = true; pollReleaseOperation(operation); }
} catch (e) { /* Browser storage is optional. */ }
// Auto-refresh every 30 seconds
diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index 8ef306f..7668ab8 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -132,6 +132,43 @@ http {
proxy_pass http://health_api/api/build-info;
}
+ # Fleet release writes and status use live session authority in Flask.
+ location ^~ /api/releases/ {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+ location = /api/update {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+ location = /api/pull {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+ location = /api/update-settings {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+ location = /api/update-status {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+
# Version management and update endpoints (protected)
location /api/versions {
auth_request /_auth_check;
diff --git a/requirements.md b/requirements.md
index d2fc55b..9cdb220 100644
--- a/requirements.md
+++ b/requirements.md
@@ -1,3 +1,23 @@
+# Fleet Manager bounded work-item authority
+
+REQ-FLEETUPDATE-001 is an administrative label for the exact five-step revision 2
+Fleet Manager plan approved by the owner on 27 October 2026. It adds no requirement
+semantics or approval.
+
+Canonical authority: [owner receipt](../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-20261027/owner-approval.json),
+[approved plan](../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-plan/plan.json),
+and [shared work-item requirements](../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-20261027/requirements.md).
+Approved plan SHA256: `ec14802b47a1078ad6b1d051ba2943d43c77160e5bfe119c366eb4209caec516`.
+
+Repair truthful version/job status and let a Fleet administrator preview and
+update admitted managed components, including Fleet Manager itself through the
+installed host adoption owner. First installation remains operator-owned;
+routine later updates use the UI or equivalent CLI.
+
+Production trust/admission, bootstrap, packaged Linux and browser acceptance,
+retained rollback and final owner acceptance remain held until independently
+verified. No feature version bump or publication is part of this worktree change.
+
# CryptoLabs Proxy scoped requirements
diff --git a/scripts/health-api.py b/scripts/health-api.py
index 8d87886..a34ca12 100644
--- a/scripts/health-api.py
+++ b/scripts/health-api.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python3
+# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
"""
Health API server for CryptoLabs Proxy.
Detects running Docker containers, reports their status, and manages updates.
@@ -444,6 +445,7 @@ def get_vpm_prerequisite_for_display():
return value
+# TRACEWEAVER: entrypoint=get_all_versions; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
def get_all_versions():
"""Get version info for all services."""
versions = {}
@@ -486,6 +488,18 @@ def get_all_versions():
if running:
version_info.update(update_status(name, settings.get('branch', 'main')))
+ from cryptolabs_proxy.updates import installed_selection
+ try:
+ snapshot=installed_selection()
+ except UpdateError:
+ snapshot=None
+ for name,info in versions.items():
+ info['component']=name
+ info['observed_source']='running-image-metadata'
+ selected=snapshot['manifest']['components'].get(name) if snapshot else None
+ info['selected_release']=selected['release'] if selected else None
+ info['selected_channel']=snapshot['manifest'].get('component_channels',{}).get(name,'main') if selected else None
+ info['admitted_available_release']=None
# Add configured branch
versions['_settings'] = {
'target_branch': settings.get('branch', 'main'),
@@ -506,10 +520,13 @@ def get_build_info():
if '=' in line:
key, value = line.split('=', 1)
info[key.lower()] = value
+ info.update(component='cryptolabs-proxy',metadata_source='image-build-info')
return info
except FileNotFoundError:
return {
- 'version': 'dev',
+ 'component': 'cryptolabs-proxy',
+ 'metadata_source': 'unavailable',
+ 'version': 'unknown',
'branch': 'unknown',
'commit': 'unknown',
'build_date': 'unknown',
@@ -613,6 +630,10 @@ def do_GET(self):
self.end_headers()
def do_POST(self):
+ # Browser writes belong exclusively to live-admin Flask authority.
+ if urlparse(self.path).path in ('/api/update','/api/pull','/api/update-settings'):
+ self.send_json({'error':'Use the authenticated release API.'},403)
+ return
parsed = urlparse(self.path)
path = parsed.path
diff --git a/src/cryptolabs_proxy/auth.py b/src/cryptolabs_proxy/auth.py
index e594edf..d136d76 100644
--- a/src/cryptolabs_proxy/auth.py
+++ b/src/cryptolabs_proxy/auth.py
@@ -1,3 +1,4 @@
+# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
# TRACEWEAVER: file-role=account-pairing-routes; req=REQ-PAIR-004; trace=TRACE-PAIR-004; ver=VER-PAIR-004
"""
CryptoLabs Proxy - Unified Authentication Module
@@ -607,6 +608,7 @@ def verify_proxy_auth(headers: dict) -> Optional[dict]:
# FLASK INTEGRATION HELPERS
# =============================================================================
+# TRACEWEAVER: entrypoint=create_flask_auth_app; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
def create_flask_auth_app():
"""
Create a Flask app for the proxy authentication API.
@@ -1294,6 +1296,144 @@ def derive(purpose: bytes) -> str:
'csrf_token': derive(b'cryptolabs/vpm/csrf/v1:'),
}, 200
+ def release_authority(mutation=False):
+ from .release_client import ReleaseBrokerError
+ user,status=current_admin_user()
+ if status!=200: raise ReleaseBrokerError('admin_required','Current administrator access required.',status)
+ cookie=request.cookies.get(app.config['SESSION_COOKIE_NAME'])
+ if not cookie: raise ReleaseBrokerError('admin_required','Current administrator access required.',401)
+ binding=session.get('_release_session')
+ valid=(isinstance(binding,dict) and binding.get('principal')==user['username']
+ and isinstance(binding.get('nonce'),str) and len(binding['nonce'])==64
+ and all(c in '0123456789abcdef' for c in binding['nonce']))
+ if not valid:
+ if mutation:
+ raise ReleaseBrokerError('csrf_or_origin','Release session token must be issued before mutation.',403)
+ # This integrity-protected identifier is visible in Flask's signed
+ # cookie. Authority still depends on the server HMAC and live user.
+ binding={'principal':user['username'],'nonce':secrets.token_hex(32)}
+ session['_release_session']=binding
+ material=json.dumps([user['username'],binding['nonce']],separators=(',',':')).encode('utf-8')
+ def derive(purpose):
+ return hmac.new(AUTH_SECRET_KEY.encode(),purpose+material,hashlib.sha256).hexdigest()
+ token=derive(b'cryptolabs/releases/csrf/v1:')
+ if mutation:
+ # Only an installed canonical origin grants browser mutation authority.
+ origin=app.config.get('RELEASE_PUBLIC_ORIGIN') or os.environ.get('FLEET_PUBLIC_ORIGIN','')
+ supplied=request.headers.get('X-CSRF-Token','')
+ if not origin or request.headers.get('Origin')!=origin or not hmac.compare_digest(supplied,token):
+ raise ReleaseBrokerError('csrf_or_origin','Release origin and session token required.',403)
+ return {'kind':'fleet-admin','username':user['username'],'subject':derive(b'cryptolabs/releases/subject/v1:')},token
+
+ def release_error(error):
+ return jsonify({'error':error.message,'code':error.code}),error.status
+
+ def release_dispatch(action,params):
+ from .release_client import ReleaseBrokerClient,ReleaseBrokerError,validate_params
+ try:
+ actor,_=release_authority(request.method=='POST')
+ validate_params(action,params)
+ return jsonify(ReleaseBrokerClient().request(action,params,actor=actor))
+ except ReleaseBrokerError as error: return release_error(error)
+
+ def release_body():
+ from .release_client import strict_json,ReleaseBrokerError,MAX_REQUEST
+ if request.content_length is None or request.content_length>MAX_REQUEST:
+ raise ReleaseBrokerError('invalid_request','Bounded JSON body required.',400)
+ try:
+ value=strict_json(request.get_data(cache=False))
+ if not isinstance(value,dict): raise ValueError()
+ return value
+ except (ValueError,UnicodeError): raise ReleaseBrokerError('invalid_request','Invalid release JSON.',400) from None
+
+ @app.route('/api/releases/session')
+ def release_session():
+ from .release_client import ReleaseBrokerError,SOCKET_PATH
+ try:
+ actor,token=release_authority()
+ return jsonify({'authenticated':True,'username':actor['username'],'csrf_token':token,
+ 'broker_available':Path(SOCKET_PATH).exists(),
+ 'origin_configured':bool(app.config.get('RELEASE_PUBLIC_ORIGIN') or os.environ.get('FLEET_PUBLIC_ORIGIN'))})
+ except ReleaseBrokerError as error: return release_error(error)
+
+ @app.route('/api/releases/candidates')
+ def release_candidates():
+ if set(request.args)-{'channel'}: return jsonify({'error':'Invalid discovery query.'}),400
+ return release_dispatch('list',{'channel':request.args.get('channel','main')})
+
+ @app.route('/api/releases/preview',methods=['POST'])
+ @app.route('/api/releases/apply',methods=['POST'])
+ def release_mutation():
+ from .release_client import ReleaseBrokerError
+ try:
+ release_authority(True)
+ return release_dispatch(request.path.rsplit('/',1)[-1],release_body())
+ except ReleaseBrokerError as error: return release_error(error)
+
+ @app.route('/api/releases/operations/
')
+ def release_operation_status(operation_id):
+ return release_dispatch('status',{'operation_id':operation_id})
+
+ @app.route('/api/releases/operations//recover',methods=['POST'])
+ def release_operation_recover(operation_id):
+ from .release_client import ReleaseBrokerError
+ try:
+ release_authority(True)
+ body=release_body()
+ if body not in ({},{'operation_id':operation_id}):
+ raise ReleaseBrokerError('invalid_request','Recovery identifier must match operation.',400)
+ return release_dispatch('recover',{'operation_id':operation_id})
+ except ReleaseBrokerError as error: return release_error(error)
+
+ @app.route('/api/update-status')
+ def legacy_update_status():
+ from .release_client import ReleaseBrokerError
+ from .updates import job_status,UpdateError
+ try:
+ release_authority()
+ return jsonify(job_status(request.args.get('id','')))
+ except ReleaseBrokerError as error: return release_error(error)
+ except UpdateError as error: return jsonify({'error':str(error)}),404
+
+ @app.route('/api/update',methods=['POST'])
+ @app.route('/api/pull',methods=['POST'])
+ @app.route('/api/update-settings',methods=['GET','POST'])
+ def legacy_release_mutation():
+ from .release_client import ReleaseBrokerError
+ from . import updates
+ try:
+ release_authority(request.method=='POST')
+ settings_path=DATA_DIR/'update-settings.json'
+ settings={'branch':'main','auto_update':True,'update_schedule':'daily'}
+ try: settings.update(json.loads(settings_path.read_text()))
+ except (OSError,ValueError,TypeError): pass
+ if request.method=='GET': return jsonify(settings)
+ body=release_body()
+ snapshot=updates.installed_selection()
+ if snapshot:
+ raise ReleaseBrokerError('release_owner_required','Selected releases require the Fleet Manager admitted release owner. Preview updates through /api/releases.',409)
+ if request.path=='/api/update-settings':
+ if set(body)-{'branch','auto_update','update_schedule'}:
+ raise ReleaseBrokerError('invalid_request','Invalid settings fields.',400)
+ if ('branch' in body and body['branch'] not in ('main','dev') or
+ 'auto_update' in body and type(body['auto_update']) is not bool or
+ 'update_schedule' in body and body['update_schedule'] not in ('daily','weekly','manual')):
+ raise ReleaseBrokerError('invalid_request','Invalid update settings.',400)
+ settings.update(body)
+ updates._write_json(settings_path,settings)
+ return jsonify({'success':True,'settings':settings})
+ if set(body)-{'service','branch','selection_id','selection_sha256'}:
+ raise ReleaseBrokerError('invalid_request','Invalid updater fields.',400)
+ branch=body.get('branch',settings['branch'])
+ if 'selection_id' in body or 'selection_sha256' in body:
+ raise ReleaseBrokerError('selection_conflict','Installed selection unavailable.',409)
+ job=updates.submit_job(body.get('service','all'),branch,'pull' if request.path=='/api/pull' else 'update')
+ return jsonify({'success':True,'job':job}),202
+ except ReleaseBrokerError as error: return release_error(error)
+ except updates.UpdateBusy as error: return jsonify({'error':str(error),'job':error.job}),409
+ except updates.UpdateError as error: return jsonify({'error':str(error)}),400
+ except (OSError,ValueError,TypeError): return jsonify({'error':'Updater unavailable; no update confirmed.'}),503
+
def reset_pair_session():
binding = session.get('pair_session')
if binding:
@@ -1324,6 +1464,7 @@ def login():
error = 'Passwords do not match'
else:
create_user(username, password, 'admin')
+ session.pop('_release_session', None)
reset_pair_session()
session['logged_in'] = True
session['username'] = username
@@ -1338,6 +1479,7 @@ def login():
# Normal login
user = verify_user(username, password)
if user:
+ session.pop('_release_session', None)
reset_pair_session()
session['logged_in'] = True
session['username'] = user['username']
diff --git a/src/cryptolabs_proxy/config.py b/src/cryptolabs_proxy/config.py
index 5d4de6b..0f1e42a 100644
--- a/src/cryptolabs_proxy/config.py
+++ b/src/cryptolabs_proxy/config.py
@@ -1,3 +1,4 @@
+# TRACEWEAVER: file-role=fleet-release-compose-consumer; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
"""Configuration management for CryptoLabs Proxy."""
import os
@@ -59,6 +60,7 @@ def render_vpm_internal_auth_blocks() -> str:
return get_jinja_env().get_template("vast-price-manager-auth-blocks.conf.j2").render()
+# TRACEWEAVER: entrypoint=generate_docker_compose; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
def generate_docker_compose(config_dir: Path, domain: str = None, use_letsencrypt: bool = False,
*, release_selection_dir=None, platform=None):
"""Generate docker-compose.yml from template."""
@@ -81,7 +83,10 @@ def generate_docker_compose(config_dir: Path, domain: str = None, use_letsencryp
proxy_image = selected['cryptolabs-proxy']['reference']
proxy_origin = selected['cryptolabs-proxy']['origin']
+ broker_enabled=registry.config.get('release_broker_enabled',False)
+ if type(broker_enabled) is not bool: raise ValueError('release_broker_enabled must be boolean')
content = template.render(
+ release_broker_enabled=broker_enabled,
domain=domain,
use_letsencrypt=use_letsencrypt,
release_selection_dir=str(Path(release_selection_dir).resolve()) if release_selection_dir else None,
diff --git a/src/cryptolabs_proxy/release_client.py b/src/cryptolabs_proxy/release_client.py
new file mode 100644
index 0000000..6fe64db
--- /dev/null
+++ b/src/cryptolabs_proxy/release_client.py
@@ -0,0 +1,103 @@
+# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
+"""Bounded client for the installed Fleet release owner, with no remote paths."""
+import json
+import os
+from pathlib import Path
+import re
+import socket
+import stat
+import uuid
+
+from .release_selection import COMPONENTS
+
+SOCKET_PATH = '/run/dc-overview-release-broker.sock'
+MAX_REQUEST = 65536
+MAX_RESPONSE = 1048576
+
+
+class ReleaseBrokerError(ValueError):
+ def __init__(self, code, message='Release owner unavailable.', status=503):
+ self.code, self.message, self.status = code, message, status
+ super().__init__(message)
+
+
+def strict_json(raw):
+ if isinstance(raw,bytes): raw=raw.decode('utf-8')
+ def unique(pairs):
+ value = {}
+ for key, item in pairs:
+ if key in value: raise ValueError('duplicate_field')
+ value[key] = item
+ return value
+ return json.loads(raw, object_pairs_hook=unique, parse_constant=lambda _: (_ for _ in ()).throw(ValueError('invalid_number')))
+
+
+def validate_params(action, params):
+ fields = {'list': {'channel'}, 'preview': {'candidate_ids','components','expected_selection_id','expected_selection_sha256','operation_kind'},
+ 'apply': {'operation_id','preview_id','preview_sha256','expected_selection_id','expected_selection_sha256'},
+ 'status': {'operation_id'}, 'recover': {'operation_id'}}
+ if action not in fields or not isinstance(params, dict) or set(params) != fields[action]:
+ raise ReleaseBrokerError('invalid_request', 'Invalid release request fields.', 400)
+ for key, value in params.items():
+ if key in ('candidate_ids','components'):
+ if (not isinstance(value, list) or not 1 <= len(value) <= 32
+ or any(not isinstance(x,str) or not 1 <= len(x) <= 128 for x in value)
+ or len(set(value)) != len(value) or key == 'components' and any(x not in COMPONENTS for x in value)):
+ raise ReleaseBrokerError('invalid_request','Invalid release component scope.',400)
+ elif not isinstance(value,str) or not 1 <= len(value) <= 128:
+ raise ReleaseBrokerError('invalid_request','Invalid release request value.',400)
+ elif key.endswith('_sha256') and not re.fullmatch('[0-9a-f]{64}',value):
+ raise ReleaseBrokerError('invalid_request','Invalid release digest.',400)
+ elif key == 'operation_id':
+ try:
+ if str(uuid.UUID(value)) != value: raise ValueError()
+ except ValueError: raise ReleaseBrokerError('invalid_request','Invalid operation identifier.',400) from None
+ if 'channel' in params and params['channel'] not in ('main','dev'):
+ raise ReleaseBrokerError('invalid_channel','Use main or dev.',400)
+ if 'operation_kind' in params and params['operation_kind'] not in ('services-update','proxy-adopt'):
+ raise ReleaseBrokerError('invalid_request','Invalid release operation.',400)
+ return params
+
+
+class ReleaseBrokerClient:
+# TRACEWEAVER: entrypoint=request; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
+ def request(self, action, params, *, actor):
+ validate_params(action,params)
+ if (not isinstance(actor,dict) or set(actor) != {'kind','username','subject'}
+ or actor.get('kind') != 'fleet-admin' or any(not isinstance(actor[k],str) or not 1<=len(actor[k])<=128 for k in ('username','subject'))):
+ raise ReleaseBrokerError('invalid_actor','Release administrator identity required.',403)
+ raw=json.dumps({'schema_version':1,'action':action,'params':params,'actor':actor},separators=(',',':'),allow_nan=False).encode()+b'\n'
+ if len(raw)>MAX_REQUEST: raise ReleaseBrokerError('request_too_large','Release request too large.',413)
+ try:
+ metadata=Path(SOCKET_PATH).lstat()
+ if not stat.S_ISSOCK(metadata.st_mode) or metadata.st_uid!=0 or metadata.st_mode & 0o077:
+ raise OSError('unsafe_owner_socket')
+ with socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) as connection:
+ connection.settimeout(3)
+ connection.connect(SOCKET_PATH)
+ connection.settimeout(15)
+ connection.sendall(raw)
+ connection.shutdown(socket.SHUT_WR)
+ response=bytearray()
+ while True:
+ chunk=connection.recv(min(65536,MAX_RESPONSE+1-len(response)))
+ if not chunk: break
+ response.extend(chunk)
+ if len(response)>MAX_RESPONSE: raise ValueError('response_too_large')
+ value=strict_json(bytes(response))
+ if (not isinstance(value,dict) or type(value.get('schema_version')) is not int or value['schema_version']!=1
+ or type(value.get('ok')) is not bool): raise ValueError('invalid_response')
+ if value['ok']:
+ if set(value)!={'schema_version','ok','result'} or not isinstance(value['result'],dict): raise ValueError('invalid_response')
+ return value['result']
+ if set(value)!={'schema_version','ok','error'}: raise ValueError('invalid_response')
+ error=value['error']
+ if (not isinstance(error,dict) or set(error)!={'code','message','status'}
+ or type(error['status']) is not int or not 400<=error['status']<=599
+ or not isinstance(error['code'],str) or not re.fullmatch('[a-z][a-z0-9_-]{0,79}',error['code'])
+ or not isinstance(error['message'],str) or len(error['message'])>512): raise ValueError('invalid_response')
+ raise ReleaseBrokerError(error['code'],error['message'],error['status'])
+ except ReleaseBrokerError:
+ raise
+ except (OSError,ValueError,UnicodeError,TypeError):
+ raise ReleaseBrokerError('owner_unavailable','Release owner unavailable; no update confirmed.',503) from None
diff --git a/src/cryptolabs_proxy/release_selection.py b/src/cryptolabs_proxy/release_selection.py
index ab3be7b..4b54065 100644
--- a/src/cryptolabs_proxy/release_selection.py
+++ b/src/cryptolabs_proxy/release_selection.py
@@ -1,3 +1,4 @@
+# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
"""Pure, bounded contract1 consumer; approval remains with the host operator.
Hashes identify exact imported bytes, not publisher trust. Never interpret data
@@ -71,13 +72,14 @@ def unique(pairs):
raise SelectionError('invalid_json') from None
+# TRACEWEAVER: entrypoint=validate; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
def validate(selection_bytes, manifest_bytes, *, selection_id=None, selection_sha256=None):
"""Validate exact bytes, all declared artifacts and every enabled role."""
selection = _json(selection_bytes)
manifest = _json(manifest_bytes)
_fields(selection, ('schema_version', 'selection_id', 'manifest_sha256', 'components'))
- _fields(manifest, ('schema_version', 'selection_contract', 'baseline_id', 'components'))
- if type(selection['schema_version']) is not int or selection['schema_version'] != 1 or type(manifest['schema_version']) is not int or manifest['schema_version'] != 1 or type(manifest['selection_contract']) is not int or manifest['selection_contract'] != 1:
+ _fields(manifest, ('schema_version', 'selection_contract', 'baseline_id', 'components'), ('component_channels',))
+ if type(selection['schema_version']) is not int or selection['schema_version'] != 1 or type(manifest['schema_version']) is not int or manifest['schema_version'] not in (1, 2) or type(manifest['selection_contract']) is not int or manifest['selection_contract'] != manifest['schema_version']:
raise SelectionError('unsupported_contract')
_identifier(selection['selection_id']); _identifier(manifest['baseline_id'])
_hash(selection['manifest_sha256'])
@@ -91,6 +93,11 @@ def validate(selection_bytes, manifest_bytes, *, selection_id=None, selection_sh
raise SelectionError('selection_changed')
if not isinstance(manifest['components'], dict) or not isinstance(selection['components'], dict) or not selection['components']:
raise SelectionError('invalid_components')
+ channels = manifest.get('component_channels', {})
+ if (not isinstance(channels, dict) or set(channels) - set(manifest['components'])
+ or any(type(v) is not str or v not in ('main', 'dev') for v in channels.values())
+ or (manifest['schema_version'] == 1 and 'component_channels' in manifest)):
+ raise SelectionError('invalid_component_channels')
for name, component in manifest['components'].items():
if name not in COMPONENTS:
raise SelectionError('unknown_component')
diff --git a/src/cryptolabs_proxy/standard_adoption.py b/src/cryptolabs_proxy/standard_adoption.py
index 0d67389..32d80de 100644
--- a/src/cryptolabs_proxy/standard_adoption.py
+++ b/src/cryptolabs_proxy/standard_adoption.py
@@ -1,3 +1,4 @@
+# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
"""Explicit standard configuration adoption, without Nginx passthrough.
Public manifests contain identities and declarative fields only. Inspected
@@ -22,7 +23,7 @@
build_recreate_request, endpoint_configurations, wait_for_container_healthy,
wait_for_proxy_ready, PROXY_READY_TIMEOUT, _snapshot_registry_files,
_write_registry_backup, _restore_registry_files)
-from .release_selection import load, targets, sha256, SelectionError
+from .release_selection import load, targets, sha256, SelectionError, validate as validate_selection
from .services import ServiceRegistry, DEFAULT_SERVICES
@@ -52,6 +53,9 @@ def auth_state(data, *, allow_active_jobs=False):
if p.is_absolute() or '..' in p.parts or not p.parts or p.parts[0]!='auth' or not(member.isfile() or member.isdir()): raise AdoptionError('auth_snapshot_unsupported')
if member.isdir(): continue
content=archive.extractfile(member).read(); name='/'.join(p.parts[1:])
+ # PairStore creates this empty mutex on reads. Preserve it on
+ # disk without treating first-read presence as account state.
+ if name=='account_link.lock' and not content: continue
if name=='users.json':
try: validate_identity_users(json.loads(content))
except (ValueError,UnicodeError): raise AdoptionError('auth_identity_store_invalid; operator recovery required') from None
@@ -78,6 +82,42 @@ def _canonical(value):
return json.dumps(value,sort_keys=True,separators=(',',':')).encode()
+def _durable_private(path, content):
+ """Atomic, fsynced owner journal writes; never expose inspected values."""
+ import os as filesystem
+ import uuid
+ path=Path(path)
+ temporary=path.with_name('.'+path.name+'-'+uuid.uuid4().hex)
+ descriptor=filesystem.open(temporary,filesystem.O_WRONLY|filesystem.O_CREAT|filesystem.O_EXCL|filesystem.O_NOFOLLOW,0o600)
+ try:
+ with filesystem.fdopen(descriptor,'wb') as stream:
+ stream.write(content); stream.flush(); filesystem.fsync(stream.fileno())
+ filesystem.replace(temporary,path)
+ descriptor=filesystem.open(path.parent,filesystem.O_RDONLY|filesystem.O_DIRECTORY)
+ try: filesystem.fsync(descriptor)
+ finally: filesystem.close(descriptor)
+ finally:
+ try: temporary.unlink()
+ except FileNotFoundError: pass
+
+
+def _restoration_evidence(engine, source, plan, expected, config_dir):
+ serving=engine.inspect('cryptolabs-proxy')
+ sanitized=[e for e in source['Config'].get('Env',[]) if not e.startswith('FLEET_ADMIN_PASS=')]+['FLEET_ADMIN_PASS=']
+ if (not serving.get('State',{}).get('Running') or serving['Image']!=source['Image']
+ or serving['Config'].get('Env',[])!=sanitized
+ or sha256(engine.read_config('cryptolabs-proxy'))!=plan['config_sha256']
+ or auth_state(engine.archive_auth('cryptolabs-proxy'),allow_active_jobs=True)!=expected):
+ raise AdoptionError('source_recovery_unverified')
+ registry=_snapshot_registry_files(ServiceRegistry(config_dir))
+ registry_sha=sha256(_canonical({p.name:sha256(data) if data is not None else None for p,data in registry.items()}))
+ if registry_sha!=plan['registry_sha256']: raise AdoptionError('source_registry_recovery_unverified')
+ return {'schema_version':1,'plan_id':plan['plan_id'],'runtime_sha256':_runtime_identity(serving),
+ 'container_id':serving['Id'],'image_id':serving['Image'],'config_sha256':plan['config_sha256'],
+ 'selection_sha256':plan['selection_sha256'],'registry_sha256':registry_sha,
+ 'auth_state_sha256':sha256(_canonical(expected)),'startup_verified':True,'health_verified':True}
+
+
def parse_nginx(data):
"""Bounded structural comparison; unknown directives are never executed."""
if not isinstance(data,bytes) or len(data)>MAX_CONFIG:
@@ -142,8 +182,10 @@ def _safe_upstream(words):
def _declaration(value):
required={'schema_version','domain','tls','release_selection_dir','services','obsolete_overlays','obsolete_owner_labels'}
- if not isinstance(value,dict) or set(value)-required-{'host_forward_groups','acme_challenge','hostname_policy','http_policy'} or not required<=set(value) or type(value['schema_version']) is not int or value['schema_version']!=1:
+ if not isinstance(value,dict) or set(value)-required-{'host_forward_groups','acme_challenge','hostname_policy','http_policy','release_broker_enabled'} or not required<=set(value) or type(value['schema_version']) is not int or value['schema_version']!=1:
raise AdoptionError('declaration_schema_invalid')
+ if 'release_broker_enabled' in value and type(value['release_broker_enabled']) is not bool:
+ raise AdoptionError('release_broker_policy_invalid')
domain=value['domain']
if not isinstance(domain,str) or not DOMAIN.fullmatch(domain) or '..' in domain or domain.endswith('.') or value['tls'] not in ('self-signed','letsencrypt'):
raise AdoptionError('domain_or_tls_invalid')
@@ -187,6 +229,22 @@ def render_standard(declaration):
return get_jinja_env().get_template('nginx.conf.j2').render(domain=declaration['domain'],letsencrypt=declaration['tls']=='letsencrypt',services=declaration['services'],host_forward_groups=declaration.get('host_forward_groups',[]),acme_challenge=declaration.get('acme_challenge'),hostname_policy=validate_hostname_policy(declaration.get('hostname_policy'),declaration['domain']),http_policy=validate_http_policy(declaration.get('http_policy'),declaration['domain'],declaration.get('acme_challenge'))).encode()
+PREVIOUS_STANDARD_SOURCE = '444d7c0e06e0f9935971c51872e9f37bd8869282'
+PREVIOUS_STANDARD_TEMPLATES = {'previous-standard-nginx-444d7c0.conf.j2': 'f12e2b9800cbaa0e49e685506cfaad38eb7e568fa6d239fa15277af59073d8de', 'vast-price-manager-auth-blocks.conf.j2': 'b9d7013434557492887b0600bc2afc92227ee17f9efebe55b0cccc4b16bd3f81', 'acme-challenge.conf.j2': '7e594162700825757c9d402971094448aa1834d1f7121ac1c02f152f345e149a', 'fleet-api-routes.conf.j2': '6991ca6bf1dd4b258e8ee7851fc69ef50b19dfe2353570e4911a0ebadc68ed1b', 'vast-price-manager.conf.j2': '32483d169a09c42653396e66c5c7efe1995bd599f44f11ee64fa2f0731a0a2f8', 'standard-service-routes.conf.j2': 'ef0f55aae48e35127d288781e68281a371505ba27384c8fd04dff9200dce3963', 'exporter-auth-routes.conf.j2': '450f4c418c09474ac45b6dd314c47cc090fe7d230a0206b30e48c7975023133b', 'host-forwarding.conf.j2': '35c4a1fe1d3001865a77d7653350b7a61968953b94172dc7063bc31a42c0f9f7'}
+
+
+def render_previous_standard(declaration):
+ env=get_jinja_env()
+ for name,expected in PREVIOUS_STANDARD_TEMPLATES.items():
+ raw=env.loader.get_source(env,name)[0].encode()
+ if sha256(raw)!=expected: raise AdoptionError('previous_standard_profile_unavailable')
+ return env.get_template('previous-standard-nginx-444d7c0.conf.j2').render(
+ domain=declaration['domain'],letsencrypt=declaration['tls']=='letsencrypt',services=declaration['services'],
+ host_forward_groups=declaration.get('host_forward_groups',[]),acme_challenge=declaration.get('acme_challenge'),
+ hostname_policy=validate_hostname_policy(declaration.get('hostname_policy'),declaration['domain']),
+ http_policy=validate_http_policy(declaration.get('http_policy'),declaration['domain'],declaration.get('acme_challenge'))).encode()
+
+
def _policy_healthcheck(source, declaration):
try:
return policy_healthcheck(source.get('Config',{}).get('Healthcheck'),declaration['domain'],declaration.get('hostname_policy'),declaration.get('http_policy'),letsencrypt=declaration['tls']=='letsencrypt')
@@ -248,7 +306,7 @@ class AdoptionEngine(DockerEngine):
def prove_recovery_auth(self,source,expected,identity):
"""Execute the source image's auth startup with no network or ports.
- The restored store remains private until this actual old initializer
+ The stopped source store stays quiescent while this actual old initializer
proves it preserves every file with the obsolete bootstrap disabled.
"""
from .migration import _mount_request
@@ -581,6 +639,7 @@ def visit(items):
visit(nodes); return nodes
actual=normalize(tree); expected=normalize(parse_nginx(render_standard(declaration)))
if actual==expected: return 'standard'
+ if actual==normalize(parse_nginx(render_previous_standard(declaration))): return 'standard-444d7c0'
legacy=get_jinja_env().loader.get_source(get_jinja_env(),'legacy-bootstrap-nginx.conf')[0].encode()
profiles=[legacy]
for enabled in (False,True): profiles.append(render_managed_vpm_config(legacy,enabled))
@@ -618,7 +677,8 @@ def visit(items):
context=_safe(' '.join(mismatch) if mismatch[0]=='location' else mismatch[0])
raise AdoptionError('unsupported_source_behavior: '+context+' requires a supported declarative product profile')
- def plan(self,settings,*,environment=None):
+# TRACEWEAVER: entrypoint=plan; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
+ def plan(self,settings,*,environment=None,proposed_selection=None):
declaration=_declaration(settings); source=self.engine.inspect('cryptolabs-proxy')
if not source.get('State',{}).get('Running'): raise AdoptionError('source_not_running')
healthcheck=_policy_healthcheck(source,declaration)
@@ -642,7 +702,13 @@ def plan(self,settings,*,environment=None):
if len(matching)!=1 or matching[0].get('Type')!='bind' or matching[0].get('RW') is not False: raise AdoptionError('overlay_mount_unproven')
if sha256(self.engine.overlay_bytes(destination))!=overlay['sha256']: raise AdoptionError('overlay_source_changed')
if overlay['replacement_sha256']!=sha256(Path(__file__).with_name('auth.py').read_bytes()): raise AdoptionError('overlay_replacement_module_unproven')
- snapshot=load(declaration['release_selection_dir']); image=self.engine.image(source['Image']); platform=image.get('Os','')+'/'+image.get('Architecture','')
+ if proposed_selection is None:
+ snapshot=load(declaration['release_selection_dir'])
+ else:
+ if not isinstance(proposed_selection,dict): raise SelectionError('invalid_snapshot')
+ snapshot=validate_selection(proposed_selection['selection_bytes'].encode(), proposed_selection['manifest_bytes'].encode(),
+ selection_id=proposed_selection['selection_id'],selection_sha256=proposed_selection['selection_sha256'])
+ image=self.engine.image(source['Image']); platform=image.get('Os','')+'/'+image.get('Architecture','')
selected=targets(snapshot,platform); artifact=selected.get('cryptolabs-proxy')
if not artifact or artifact['owner']!='proxy-updater': raise AdoptionError('approved_proxy_required')
candidate=render_standard(declaration)
@@ -680,26 +746,68 @@ def adopt(self,settings,identity,*,environment=None):
backup=self.backup_root/identity
if backup.exists(): raise AdoptionError('private_recovery_snapshot_exists; inspect prior outcome before retry')
backup.mkdir(parents=True,mode=0o700); self.backup_root.chmod(0o700); backup.chmod(0o700)
- write=CustomConfigMigrator._write_private
- write(backup/'inspect.json',_canonical(source)); write(backup/'source.nginx.conf',self.engine.read_config('cryptolabs-proxy')); _write_registry_backup(backup,previous)
+ import os as filesystem
+ for parent in (self.backup_root,self.config_dir):
+ descriptor=filesystem.open(parent,filesystem.O_RDONLY|filesystem.O_DIRECTORY)
+ try: filesystem.fsync(descriptor)
+ finally: filesystem.close(descriptor)
+ write=_durable_private
+ write(backup/'declaration.json',_canonical(declaration)); write(backup/'inspect.json',_canonical(source)); write(backup/'source.nginx.conf',self.engine.read_config('cryptolabs-proxy')); _write_registry_backup(backup,previous)
candidate=backup/'candidate.nginx.conf'; write(candidate,render_standard(declaration)); write(backup/'plan.json',_canonical(plan))
+ write(backup/'registry-snapshot.json',(backup/'registry-snapshot.json').read_bytes())
+ journal={'schema_version':1,'plan_id':identity,'source_id':source['Id'],
+ 'source_image_id':source['Image'],'candidate_may_have_started':False}
+ def phase(value,**details):
+ journal.update(details); journal['phase']=value
+ write(backup/'phase.json',_canonical(journal))
+ def mutation(action,name,invoke,**details):
+ try: before=engine.inspect(name)
+ except Exception: before=None
+ phase(action+'-requested',target=name,prior_id=before.get('Id') if before else None,**details)
+ result=invoke()
+ try: after=engine.inspect(name)
+ except Exception: after=None
+ if action=='create' and after and after.get('Image')==journal.get('candidate_image_id'):
+ journal['candidate_container_id']=after['Id']
+ phase(action+'-done',target=name,observed_id=after.get('Id') if after else None,**details)
+ return result
+ engine=self.engine
+ phase('candidate-pull-requested')
self.engine.pull(plan['image']); artifact=self.engine.image(plan['image'])
+ phase('candidate-pulled',candidate_image_id=artifact['Id'])
if plan['image'] not in artifact.get('RepoDigests',[]) or artifact.get('Os','')+'/'+artifact.get('Architecture','')!=plan['platform']: raise AdoptionError('candidate_artifact_mismatch')
if self.engine.artifact_file_sha256(plan['image'],AUTH_OVERLAY,identity)!=sha256(Path(__file__).with_name('auth.py').read_bytes()): raise AdoptionError('candidate_auth_module_mismatch')
for overlay in declaration['obsolete_overlays']:
if self.engine.artifact_file_sha256(plan['image'],overlay['destination'],identity)!=overlay['replacement_sha256']: raise AdoptionError('overlay_artifact_module_mismatch')
write(backup/'retired-auth.py',self.engine.overlay_bytes(overlay['destination']))
+ phase('candidate-validate-requested')
self.engine.validate(plan['image'],candidate,source,identity)
+ phase('candidate-validated')
# Recheck source and all policy/file identities after pull/test.
if self.plan(settings,environment=environment)!=plan: raise AdoptionError('preflight_changed')
+ phase('updater-guard-requested')
guard=exclusion.enter_context(self.engine.updater_guard(source,plan['image'],identity))
+ try: held=self.engine.inspect('cryptolabs-adoption-lock-'+identity)
+ except Exception: held=None
+ phase('updater-held',guard_id=held.get('Id') if held else None)
# Reject queued/running jobs and take an identity while the
# dispatch and operation authorities are both excluded.
- prestop_auth=auth_state(self.engine.archive_auth(plan['container_id']))
+ prestop_archive=self.engine.archive_auth(plan['container_id'])
+ prestop_auth=auth_state(prestop_archive)
+ write(backup/'prestop-auth.tar',prestop_archive)
+ phase('source-ready-to-stop',prestop_auth=prestop_auth)
if self.plan(settings,environment=environment)!=plan: raise AdoptionError('preflight_changed')
clone=deepcopy(source); clone['Mounts']=[m for m in source['Mounts'] if m.get('Destination') not in ('/etc/nginx/nginx.conf','/etc/cryptolabs/releases') and m.get('Destination') not in {o['destination'] for o in declaration['obsolete_overlays']}]
auth_mount=next(m for m in source['Mounts'] if m['Destination']=='/data/auth')
clone['Mounts'].append({'Type':'bind','Source':declaration['release_selection_dir'],'Destination':'/etc/cryptolabs/releases','RW':False})
+ if declaration.get('release_broker_enabled'):
+ socket_path='/run/dc-overview-release-broker.sock'
+ metadata=Path(socket_path).lstat()
+ import stat
+ if not stat.S_ISSOCK(metadata.st_mode) or metadata.st_uid!=0 or metadata.st_mode & 0o077:
+ raise AdoptionError('release_broker_owner_unavailable')
+ clone['Mounts']=[m for m in clone['Mounts'] if m.get('Destination')!=socket_path]
+ clone['Mounts'].append({'Type':'bind','Source':socket_path,'Destination':socket_path,'RW':False})
nginx=self.config_dir/'nginx.conf'; request=build_recreate_request(clone,plan['image'],str(nginx))
if declaration.get('hostname_policy') or declaration.get('http_policy'):
request['Config']['Healthcheck']=_policy_healthcheck(source,declaration)
@@ -707,6 +815,8 @@ def adopt(self,settings,identity,*,environment=None):
config['Labels'].update({'com.centurylinklabs.watchtower.enable':'false','io.cryptolabs.update.origin':plan['origin'],'io.cryptolabs.update.reference':plan['image']})
for label in declaration['obsolete_owner_labels']: config['Labels'].pop(label,None)
config['Env']=[e for e in config.get('Env',[]) if not e.startswith('RELEASE_SELECTION_REQUIRED=') and e.split('=',1)[0] not in (environment or {})]+['RELEASE_SELECTION_REQUIRED=true']+[k+'='+v for k,v in (environment or {}).items()]
+ if not any(e.startswith('FLEET_PUBLIC_ORIGIN=') for e in config['Env']):
+ config['Env'].append('FLEET_PUBLIC_ORIGIN=https://'+declaration['domain'])
engine=self.engine; auth_copy=None; candidate_copy=None; candidate_started=False; recovered_id=None; recovery_expected=None; recovery_verified=False
def restart_source(name):
nonlocal recovered_id,recovery_expected,recovery_verified
@@ -716,7 +826,9 @@ def restart_source(name):
# Immutable source environment cannot be edited by Docker.
# Prove its real startup, then recreate the original image
# with only obsolete installation authority disabled.
+ phase('source-startup-proof-requested')
engine.prove_recovery_auth(source,expected,identity)
+ phase('source-startup-proven')
guard()
clone=deepcopy(source)
nginx_mount=next((m for m in clone['Mounts'] if m['Destination']=='/etc/nginx/nginx.conf'),None)
@@ -727,17 +839,21 @@ def restart_source(name):
from .migration import _mount_request
payload['HostConfig']['Mounts'][-1]=_mount_request(nginx_mount)
payload['Config']['Env']=[e for e in source['Config'].get('Env',[]) if not e.startswith('FLEET_ADMIN_PASS=')]+['FLEET_ADMIN_PASS=']
- for network in engine.inspect(name).get('NetworkSettings',{}).get('Networks',{}): engine.disconnect(name,network)
- engine.rename(name,'cryptolabs-proxy.original-'+identity)
- engine.create(name,payload)
+ for network in engine.inspect(name).get('NetworkSettings',{}).get('Networks',{}): mutation('disconnect',name,lambda network=network:engine.disconnect(name,network))
+ mutation('rename',name,lambda:engine.rename(name,'cryptolabs-proxy.original-'+identity))
+ mutation('source-create',name,lambda:engine.create(name,payload))
recovered_id=engine.inspect(name)['Id']
- engine.start(name)
+ mutation('source-start',name,lambda:engine.start(name))
healthy=wait_for_container_healthy(engine,name,PROXY_READY_TIMEOUT)
if auth_state(engine.archive_auth(name),allow_active_jobs=True)!=expected:
- engine.stop(name)
+ mutation('stop',name,lambda:engine.stop(name))
raise AdoptionError('source_recovery_unverified; forward_repair_required')
recovery_verified=healthy
- if not healthy: engine.stop(name,ignore_missing=True)
+ if not healthy: mutation('stop',name,lambda:engine.stop(name,ignore_missing=True))
+ if healthy:
+ evidence=_restoration_evidence(engine,source,plan,expected,self.config_dir)
+ write(backup/'restoration.json',_canonical(evidence))
+ phase('source-restored',restoration_auth=expected,restoration_sha256=sha256(_canonical(evidence)))
guard()
class StateSafeEngine:
def __getattr__(self,name): return getattr(engine,name)
@@ -745,7 +861,7 @@ def stop(self,name,ignore_missing=False):
nonlocal auth_copy,candidate_copy
guard()
stopping=engine.inspect(name)
- engine.stop(name,ignore_missing=ignore_missing)
+ mutation('stop',name,lambda:engine.stop(name,ignore_missing=ignore_missing))
if stopping and stopping['Id'] not in (source['Id'],recovered_id) and candidate_started:
candidate_copy=engine.archive_auth(name); write(backup/'candidate-auth.tar',candidate_copy)
if name==plan['container_id'] and auth_copy is None:
@@ -753,14 +869,22 @@ def stop(self,name,ignore_missing=False):
auth_copy=engine.archive_auth(name)
if auth_state(auth_copy)!=prestop_auth: raise AdoptionError('auth_store_changed_requires_quiescence')
write(backup/'auth.tar',auth_copy)
+ phase('source-quiescent')
write(nginx,candidate.read_bytes())
+ phase('source-config-ready')
except Exception:
_restore_registry_files(previous)
restart_source('cryptolabs-proxy')
raise AdoptionError('auth_snapshot_failed; '+('sanitized_source_recovered' if recovery_verified else 'forward_repair_required')) from None
def create(self,name,payload):
guard()
- engine.create(name,payload)
+ mutation('create',name,lambda:engine.create(name,payload))
+ def rename(self,name,target):
+ guard(); mutation('rename',name,lambda:engine.rename(name,target),rename_to=target)
+ def disconnect(self,name,network):
+ guard(); mutation('disconnect',name,lambda:engine.disconnect(name,network))
+ def remove(self,name,ignore_missing=False):
+ guard(); mutation('remove',name,lambda:engine.remove(name,ignore_missing=ignore_missing))
def connect(self,name,network,endpoint):
guard()
# The generic switch requests source reconnect before
@@ -768,7 +892,7 @@ def connect(self,name,network,endpoint):
# the sanitized source is recreated with its original
# endpoints only after owned restore and startup proof.
if engine.inspect(name)['Id'] in (source['Id'],recovered_id): return
- engine.connect(name,network,endpoint)
+ mutation('connect',name,lambda:engine.connect(name,network,endpoint))
def start(self,name):
nonlocal candidate_started
guard()
@@ -785,12 +909,14 @@ def start(self,name):
else:
if candidate_copy is not None: engine.restore_auth(name,candidate_copy,plan['image'],identity)
write(nginx,candidate.read_bytes()); candidate_started=True
- engine.start(name)
+ phase('candidate-start-requested',candidate_may_have_started=True)
+ mutation('start',name,lambda:engine.start(name))
guard()
def candidate_health(name):
guard()
ready=wait_for_proxy_ready(engine,name,lambda:engine.anonymous_auth(name,declaration['tls'],declaration['domain']) if declaration.get('hostname_policy') or declaration.get('http_policy') else engine.anonymous_auth(name,declaration['tls']),PROXY_READY_TIMEOUT)
- if not ready: engine.stop(name,ignore_missing=True)
+ if not ready: mutation('stop',name,lambda:engine.stop(name,ignore_missing=True))
+ if ready: phase('candidate-verified',candidate_verified=True,serving_runtime_sha256=_runtime_identity(engine.inspect(name)))
guard()
return ready
def rollback_health(name):
@@ -809,6 +935,7 @@ def rollback_health(name):
for name in ('hostname_policy','http_policy'):
if name in declaration: registry.config[name]=declaration[name]
else: registry.config.pop(name,None)
+ registry.config['release_broker_enabled']=declaration.get('release_broker_enabled',False)
registry.config['standard_adoption']={'plan_id':identity,'serving_sha256':_runtime_identity(engine.inspect('cryptolabs-proxy')),'declaration_sha256':plan['declaration_sha256'],'selection_sha256':plan['selection_sha256'],'auth_mount':{k:auth_mount[k] for k in ('Type','Name','Source','Destination') if k in auth_mount},'declaration':declaration}
if declaration.get('hostname_policy') or declaration.get('http_policy'):
registry.config['standard_adoption']['healthcheck']=deepcopy(request['Config']['Healthcheck'])
@@ -817,7 +944,9 @@ def rollback_health(name):
write(backup/'outcome.json',_canonical({'state':'candidate_retained','reason':'registry_forward_repair_required'}))
return {'state':'candidate_retained','reason':'registry_forward_repair_required','plan_id':identity}
result={'state':'adopted' if outcome.state=='applied' else 'candidate_retained','plan_id':identity,'positive_login_verified':False}
- try: write(backup/'outcome.json',_canonical(result))
+ try:
+ write(backup/'outcome.json',_canonical(result))
+ phase('completed')
except OSError: return {'state':'candidate_retained','reason':'outcome_record_unavailable','plan_id':identity}
return result
except (SelectionError,MigrationError,OSError,ValueError,KeyError) as error:
@@ -833,12 +962,174 @@ def rollback_health(name):
except Exception: states[name]={'unknown':True}
serving=states.get('cryptolabs-proxy',{})
restored=(serving.get('id')==locals().get('source',{}).get('Id') or serving.get('id')==locals().get('recovered_id') and locals().get('recovery_verified',False)) and serving.get('running')
- try: CustomConfigMigrator._write_private(backup/'outcome.json',_canonical({'state':'rolled_back' if restored else 'forward_repair_required','runtime':states,'candidate_auth_snapshot_retained':(backup/'candidate-auth.tar').exists()}))
+ try: _durable_private(backup/'outcome.json',_canonical({'state':'rolled_back' if restored else 'forward_repair_required','runtime':states,'candidate_auth_snapshot_retained':(backup/'candidate-auth.tar').exists()}))
except OSError: pass
reason=str(error) if isinstance(error,AdoptionError) else 'adoption_failed; inspect private recovery outcome for actual state'
raise AdoptionError(reason) from None
+# TRACEWEAVER: entrypoint=recover_adoption; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001
+ def recover_adoption(self, plan_id):
+ """Reconcile a fixed journal and restore only proven pre-candidate phases."""
+ if not isinstance(plan_id,str) or not re.fullmatch('[0-9a-f]{24}',plan_id):
+ raise AdoptionError('invalid_plan_id')
+ result={'state':'recovery-required','plan_id':plan_id}
+ if os.geteuid()!=0: raise AdoptionError('standard_adoption_requires_root')
+ from .cli import registry_lock
+ try:
+ with registry_lock(self.config_dir):
+ backup=self.backup_root/plan_id
+ def read(name):
+ path=backup/name
+ for parent in (self.backup_root,backup,path):
+ stat=parent.lstat()
+ if parent.is_symlink() or stat.st_uid!=os.getuid() or stat.st_mode & 0o077:
+ raise AdoptionError('unsafe_recovery_journal')
+ raw=path.read_bytes()
+ if len(raw)>MAX_CONFIG: raise AdoptionError('invalid_recovery_journal')
+ return json.loads(raw)
+ plan=read('plan.json'); declaration=read('declaration.json')
+ if plan.get('plan_id')!=plan_id or sha256(_canonical({k:v for k,v in plan.items() if k!='plan_id'}))[:24]!=plan_id:
+ return result
+ if sha256(_canonical(declaration))!=plan['declaration_sha256']: return result
+ try: outcome=read('outcome.json')
+ except FileNotFoundError: outcome={}
+ try: journal=read('phase.json')
+ except FileNotFoundError: journal={}
+ completed=outcome.get('state') in ('adopted','already_adopted') and self._already(declaration,plan_id)
+ source=read('inspect.json')
+ if _runtime_identity(source)!=plan['runtime_sha256']: return result
+ expected=journal.get('restoration_auth',journal.get('prestop_auth'))
+ if not isinstance(expected,dict): return result
+ serving=self.engine.inspect('cryptolabs-proxy')
+ restored=False
+ evidence=None
+ if journal.get('restoration_sha256'):
+ evidence=read('restoration.json')
+ observed=_restoration_evidence(self.engine,source,plan,expected,self.config_dir)
+ restored=(evidence==observed and journal['restoration_sha256']==sha256(_canonical(evidence)))
+ # Preserve exclusions on uncertain candidate/recovery phases.
+ restore_allowed=(journal.get('plan_id')==plan_id and journal.get('candidate_may_have_started') is False
+ and not journal.get('phase','').startswith('recovery-')
+ and serving and not serving.get('State',{}).get('Running')
+ and _runtime_identity(serving)==plan['runtime_sha256'])
+ if not completed and not restored and not restore_allowed: return result
+ # A killed owner can leave its fixed exclusion helper. The registry
+ # lock proves that owner is no longer executing this adoption.
+ helper_name='cryptolabs-adoption-lock-'+plan_id
+ try: helper=self.engine.inspect(helper_name)
+ except MigrationError as error:
+ if not str(error).endswith('HTTP 404'): return result
+ helper=None
+ if helper:
+ auth=next(m for m in source['Mounts'] if m['Destination']=='/data/auth')
+ mounts=[m for m in helper.get('Mounts',[]) if m.get('Destination')=='/data/auth']
+ def mount_identity(m):
+ return (m.get('Type'),m.get('Name') if m.get('Type')=='volume' else m.get('Source'),m.get('Destination'),m.get('RW'))
+ command=helper.get('Config',{}).get('Cmd',[])
+ if (not journal.get('guard_id') or helper.get('Id')!=journal['guard_id']
+ or helper.get('Image')!=journal.get('candidate_image_id')
+ or helper.get('Config',{}).get('Labels',{}).get('io.cryptolabs.adoption.guard')!=plan_id
+ or helper.get('Config',{}).get('Entrypoint')!=['python3']
+ or len(command)!=4 or command[0]!='-c' or command[-2:]!=['/data/auth/update-jobs','/tmp/updater-held']
+ or len(mounts)!=1 or mount_identity(mounts[0])!=mount_identity(auth)):
+ return result
+ journal['phase']='recovery-guard-remove-requested'
+ _durable_private(backup/'phase.json',_canonical(journal))
+ self.engine.remove(helper['Id'])
+ if completed: return {'state':'completed','plan_id':plan_id}
+ if restored:
+ with self.engine.updater_guard(source,plan['image'],plan_id) as guard:
+ guard()
+ if load(declaration['release_selection_dir'])['selection_sha256']!=plan['selection_sha256']: return result
+ observed=_restoration_evidence(self.engine,source,plan,expected,self.config_dir)
+ if evidence!=observed or journal.get('restoration_sha256')!=sha256(_canonical(evidence)): return result
+ guard()
+ return {'state':'failed-restored','plan_id':plan_id}
+ with self.engine.updater_guard(source,plan['image'],plan_id) as guard:
+ guard()
+ if load(declaration['release_selection_dir'])['selection_sha256']!=plan['selection_sha256']: return result
+ serving=self.engine.inspect('cryptolabs-proxy')
+ if serving.get('State',{}).get('Running') or _runtime_identity(serving)!=plan['runtime_sha256']: return result
+ archive=self.engine.archive_auth(source['Id'])
+ quiescent_auth=auth_state(archive)
+ prestop_auth_match=quiescent_auth==journal['prestop_auth']
+ expected=quiescent_auth
+ # This is the missing quiescent post-stop capture. It is kept
+ # for evidence, never restored over the current shared store.
+ _durable_private(backup/'auth.tar',archive)
+ previous=read('registry-snapshot.json')
+ allowed={str(self.config_dir/name) for name in ('services.yaml','config.yaml','nginx.conf')}
+ if set(previous)!=allowed: return result
+ old={Path(path):value.encode('latin1') if value is not None else None for path,value in previous.items()}
+ saved_sha=sha256(_canonical({p.name:sha256(data) if data is not None else None for p,data in old.items()}))
+ if saved_sha!=plan['registry_sha256']: return result
+ current=_snapshot_registry_files(ServiceRegistry(self.config_dir))
+ for path,data in current.items():
+ if path.name=='nginx.conf':
+ if sha256(data or b'') not in (sha256(old[path] or b''),plan['candidate_sha256']): return result
+ elif data!=old[path]: return result
+ if sha256(self.engine.read_config('cryptolabs-proxy')) not in (plan['config_sha256'],plan['candidate_sha256']): return result
+ def phase(name,**details):
+ journal.update(details); journal['phase']=name
+ _durable_private(backup/'phase.json',_canonical(journal))
+ def inspect_optional(name):
+ try: return self.engine.inspect(name)
+ except MigrationError as error:
+ if not str(error).endswith('HTTP 404'): raise
+ return None
+ def mutate(action,name,invoke,**details):
+ before=inspect_optional(name)
+ phase('recovery-'+action+'-requested',target=name,prior_id=before.get('Id') if before else None,**details)
+ invoke()
+ after=inspect_optional(name)
+ phase('recovery-'+action+'-done',target=name,observed_id=after.get('Id') if after else None,**details)
+ guard()
+ phase('recovery-startup-proof-requested',restoration_auth=expected,prestop_auth_match=prestop_auth_match)
+ self.engine.prove_recovery_auth(source,expected,plan_id)
+ phase('recovery-startup-proven',recovery_startup_proven=True)
+ if auth_state(self.engine.archive_auth(source['Id']))!=expected: return result
+ _restore_registry_files(old)
+ for path,data in old.items():
+ if data is not None: _durable_private(path,data)
+ clone=deepcopy(source)
+ nginx_mount=next((m for m in clone['Mounts'] if m['Destination']=='/etc/nginx/nginx.conf'),None)
+ nginx_path=nginx_mount['Source'] if nginx_mount and nginx_mount.get('Type')=='bind' else str(backup/'source.nginx.conf')
+ clone['Mounts']=[m for m in clone['Mounts'] if m['Destination'] not in ('/etc/nginx/nginx.conf',AUTH_OVERLAY)]
+ payload=build_recreate_request(clone,source['Image'],nginx_path)
+ if nginx_mount:
+ from .migration import _mount_request
+ payload['HostConfig']['Mounts'][-1]=_mount_request(nginx_mount)
+ payload['Config']['Env']=[e for e in source['Config'].get('Env',[]) if not e.startswith('FLEET_ADMIN_PASS=')]+['FLEET_ADMIN_PASS=']
+ # Preserve the exact original name/image, with installation
+ # authority disabled and its real initializer already proven.
+ retained='cryptolabs-proxy.original-'+plan_id
+ try: prior=self.engine.inspect(retained)
+ except MigrationError as error:
+ if not str(error).endswith('HTTP 404'): return result
+ prior=None
+ if prior: return result
+ for network in serving.get('NetworkSettings',{}).get('Networks',{}):
+ mutate('disconnect',source['Id'],lambda network=network:self.engine.disconnect(source['Id'],network))
+ mutate('rename',source['Id'],lambda:self.engine.rename(source['Id'],retained),rename_to=retained)
+ mutate('create','cryptolabs-proxy',lambda:self.engine.create('cryptolabs-proxy',payload))
+ recovered=self.engine.inspect('cryptolabs-proxy')
+ phase('recovery-source-created',recovery_container_id=recovered['Id'])
+ mutate('start',recovered['Id'],lambda:self.engine.start('cryptolabs-proxy'))
+ healthy=wait_for_container_healthy(self.engine,'cryptolabs-proxy',PROXY_READY_TIMEOUT)
+ if not healthy or auth_state(self.engine.archive_auth('cryptolabs-proxy'),allow_active_jobs=True)!=expected:
+ mutate('stop',recovered['Id'],lambda:self.engine.stop('cryptolabs-proxy'))
+ return result
+ evidence=_restoration_evidence(self.engine,source,plan,expected,self.config_dir)
+ _durable_private(backup/'restoration.json',_canonical(evidence))
+ phase('failed-restored',restoration_sha256=sha256(_canonical(evidence)))
+ _durable_private(backup/'outcome.json',_canonical({'state':'rolled_back','restoration_sha256':sha256(_canonical(evidence))}))
+ guard()
+ return {'state':'failed-restored','plan_id':plan_id}
+ except (OSError,ValueError,KeyError,MigrationError,TypeError,StopIteration):
+ pass
+ return result
+
def reconfigure(self,config):
"""Ordinary setup after adoption keeps inspected state and session key."""
try:
diff --git a/src/cryptolabs_proxy/templates/docker-compose.yml.j2 b/src/cryptolabs_proxy/templates/docker-compose.yml.j2
index d15f997..46d8020 100644
--- a/src/cryptolabs_proxy/templates/docker-compose.yml.j2
+++ b/src/cryptolabs_proxy/templates/docker-compose.yml.j2
@@ -14,6 +14,9 @@ services:
- ./ssl:/etc/nginx/ssl:ro
- ./services.yaml:/app/services.yaml:ro
- fleet-auth-data:/data/auth
+ {% if release_broker_enabled | default(false) %}
+ - /run/dc-overview-release-broker.sock:/run/dc-overview-release-broker.sock:ro
+ {% endif %}
{% if release_selection_dir %}
- {{ (release_selection_dir + ':/etc/cryptolabs/releases:ro') | tojson }}
{% endif %}
@@ -25,6 +28,9 @@ services:
{% endif %}
environment:
- DOCKER_HOST=unix:///var/run/docker.sock
+ {% if domain %}
+ - {{ ('FLEET_PUBLIC_ORIGIN=https://' + domain) | tojson }}
+ {% endif %}
{% if release_selection_dir %}
- RELEASE_SELECTION_REQUIRED=true
{% endif %}
diff --git a/src/cryptolabs_proxy/templates/nginx.conf.j2 b/src/cryptolabs_proxy/templates/nginx.conf.j2
index d1dd2d8..8e69164 100644
--- a/src/cryptolabs_proxy/templates/nginx.conf.j2
+++ b/src/cryptolabs_proxy/templates/nginx.conf.j2
@@ -191,6 +191,43 @@ http {
proxy_set_header Host $host;
}
+ # Fleet release writes and status use live session authority in Flask.
+ location ^~ /api/releases/ {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+ location = /api/update {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+ location = /api/pull {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+ location = /api/update-settings {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+ location = /api/update-status {
+ client_max_body_size 64k;
+ proxy_pass http://127.0.0.1:8081;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_read_timeout 20s;
+ }
+
{% include "fleet-api-routes.conf.j2" %}
# Static assets (no auth)
diff --git a/src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2 b/src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2
new file mode 100644
index 0000000..5a77d27
--- /dev/null
+++ b/src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2
@@ -0,0 +1,332 @@
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+ worker_connections 1024;
+}
+
+http {
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+
+ log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ '$status $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+
+ access_log /var/log/nginx/access.log main;
+ sendfile on;
+ keepalive_timeout 65;
+ client_max_body_size 100M;
+
+ # Gzip compression
+ gzip on;
+ gzip_vary on;
+ gzip_proxied any;
+ gzip_types text/plain text/css text/xml text/javascript application/javascript application/json application/xml;
+
+ # Rate limiting
+ limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;
+
+ {% if letsencrypt or http_policy %}
+ # Redirect HTTP to HTTPS
+ server {
+ listen 80{% if hostname_policy %} default_server{% endif %};
+ server_name {{ domain }};
+{% if hostname_policy %}
+ if ($host != {{ domain }}) {
+ return 444;
+ }
+{% endif %}
+
+{% if http_policy %}
+ location ^~ /.well-known/acme-challenge/ {
+ root /var/www/certbot;
+ default_type text/plain;
+ try_files $uri =404;
+ }
+ location / {
+ return 308 https://{{ domain }}$request_uri;
+ }
+ {% include "vast-price-manager-auth-blocks.conf.j2" %}
+{% else %}
+ location /.well-known/acme-challenge/ {
+ root /var/www/certbot;
+ }
+
+ location / {
+ return 301 https://$host$request_uri;
+ }
+{% endif %}
+ }
+
+ server {
+ listen 443 ssl http2{% if hostname_policy %} default_server{% endif %};
+ server_name {{ domain }};
+{% if hostname_policy %}
+ if ($host != {{ domain }}) {
+ return 444;
+ }
+{% endif %}
+
+ {% if letsencrypt %}
+ ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem;
+ ssl_protocols TLSv1.2 TLSv1.3;
+ ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256;
+ ssl_prefer_server_ciphers off;
+ ssl_session_cache shared:SSL:10m;
+ {% else %}
+ ssl_certificate /etc/nginx/ssl/server.crt;
+ ssl_certificate_key /etc/nginx/ssl/server.key;
+ ssl_protocols TLSv1.2 TLSv1.3;
+ ssl_ciphers HIGH:!aNULL:!MD5;
+ {% endif %}
+ {% else %}
+ server {
+ listen 80{% if hostname_policy %} default_server{% endif %};
+ listen 443 ssl http2{% if hostname_policy %} default_server{% endif %};
+ server_name {{ domain }};
+{% if hostname_policy %}
+ if ($host != {{ domain }}) {
+ return 444;
+ }
+{% endif %}
+
+ ssl_certificate /etc/nginx/ssl/server.crt;
+ ssl_certificate_key /etc/nginx/ssl/server.key;
+ ssl_protocols TLSv1.2 TLSv1.3;
+ {% endif %}
+
+ # DNS resolver for Docker networks - CRITICAL for runtime resolution
+ # This allows nginx to start even if backend services don't exist yet
+ resolver 127.0.0.11 valid=10s ipv6=off;
+ resolver_timeout 5s;
+
+ # Security headers
+ add_header X-Frame-Options "SAMEORIGIN" always;
+ add_header X-Content-Type-Options "nosniff" always;
+ add_header X-XSS-Protection "1; mode=block" always;
+
+{% include 'acme-challenge.conf.j2' %}
+ # =====================================================
+ # ERROR HANDLERS
+ # =====================================================
+
+ location @service_unavailable {
+ default_type application/json;
+ return 503 '{"error":"Service unavailable","message":"The requested service is not running. Install it or wait for it to start."}';
+ }
+
+ # =====================================================
+ # AUTHENTICATION
+ # =====================================================
+
+ # Auth server endpoints (public)
+ location /auth/ {
+ proxy_pass http://127.0.0.1:8081/auth/;
+{% if http_policy %}
+ proxy_hide_header X-Mgmt-Token;
+ proxy_cookie_flags fleet_session secure httponly samesite=lax;
+{% endif %}
+ proxy_http_version 1.1;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ }
+
+{% if http_policy %}
+ location = /auth/headers { return 404; }
+ location = /auth/headers/ { return 404; }
+ location = /auth/exporters/authorize { return 404; }
+ location = /auth/exporters/authorize/ { return 404; }
+{% endif %}
+ # Internal auth verification endpoint (for auth_request)
+ location = /_auth_check {
+ internal;
+ proxy_pass http://127.0.0.1:8081/auth/headers;
+ proxy_pass_request_body off;
+ proxy_set_header Content-Length "";
+ proxy_set_header X-Original-URI $request_uri;
+ proxy_set_header Cookie $http_cookie;
+ }
+
+ # =====================================================
+ # Internal API (blocked from public access)
+ # Only reachable via direct Docker network (172.30.0.10:8080)
+ # =====================================================
+ location /internal/ {
+ return 403;
+ }
+
+ # =====================================================
+ # PUBLIC ENDPOINTS (no auth required)
+ # =====================================================
+
+ # Health check API (always public)
+ location /api/health {
+ default_type application/json;
+ return 200 '{"status":"ok","proxy":"running"}';
+ }
+
+ # Service health check (internal, no auth)
+ location /api/services {
+ proxy_pass http://127.0.0.1:8080/api/services;
+ proxy_set_header Host $host;
+ }
+
+ {% include "fleet-api-routes.conf.j2" %}
+
+ # Static assets (no auth)
+ location /static/ {
+ root /usr/share/nginx/html;
+ }
+
+ # =====================================================
+ # PROTECTED ENDPOINTS (auth required)
+ # =====================================================
+
+ # Fleet Management Landing Page (requires auth)
+ location = / {
+ auth_request /_auth_check;
+ auth_request_set $auth_user $upstream_http_x_fleet_auth_user;
+ auth_request_set $auth_role $upstream_http_x_fleet_auth_role;
+ auth_request_set $auth_token $upstream_http_x_fleet_auth_token;
+
+ # Redirect to login if not authenticated
+ error_page 401 = @login_redirect;
+
+ root /usr/share/nginx/html;
+ index index.html;
+ }
+
+ location @login_redirect {
+ return 302 /auth/login?next=$request_uri;
+ }
+
+ {% include "vast-price-manager-auth-blocks.conf.j2" %}
+
+ {% if 'vast-price-manager' in services %}
+ {% include "vast-price-manager.conf.j2" %}
+ {% endif %}
+
+ {% include "standard-service-routes.conf.j2" %}
+
+ {% for name, service in services.items() %}
+ {% if name in ['ipmi-monitor','dc-overview','grafana','prometheus','vastai-exporter','runpod-exporter','vast-price-manager'] %}
+ # Vast Price Manager uses its dedicated admin-only route above.
+ {% elif not service.path %}
+ # {{ service.get('display_name', name) }} - external service, no local proxy route
+ {% else %}
+ # {{ service.get('display_name', name) }} - uses variable for runtime DNS resolution
+ location ~ ^{{ service.path }}(?.*)$ {
+ # Require authentication
+ auth_request /_auth_check;
+ auth_request_set $auth_user $upstream_http_x_fleet_auth_user;
+ auth_request_set $auth_role $upstream_http_x_fleet_auth_role;
+ auth_request_set $auth_token $upstream_http_x_fleet_auth_token;
+
+ # Redirect to login if not authenticated
+ error_page 401 = @login_redirect;
+
+ {% if name == 'grafana' or service.container_name == 'grafana' %}
+ # Map Fleet roles to Grafana roles (Admin, Editor, Viewer)
+ set $grafana_role "Viewer";
+ if ($auth_role = "admin") {
+ set $grafana_role "Admin";
+ }
+ if ($auth_role = "readwrite") {
+ set $grafana_role "Editor";
+ }
+ {% endif %}
+
+ # Variable allows nginx to start even if service doesn't exist yet
+ set $upstream_{{ name | replace('-', '_') }} http://{{ service.container_name }}:{{ service.port }};
+ proxy_pass $upstream_{{ name | replace('-', '_') }}/$custom_{{ name | replace('-', '_') }}$is_args$args;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header X-Script-Name {{ service.path.rstrip('/') }};
+
+ # Forward authentication headers to backend services
+ proxy_set_header X-Fleet-Auth-User $auth_user;
+ proxy_set_header X-Fleet-Auth-Role $auth_role;
+ proxy_set_header X-Fleet-Auth-Token $auth_token;
+ proxy_set_header X-Fleet-Authenticated "true";
+
+ {% if name == 'grafana' or service.container_name == 'grafana' %}
+ # Grafana-specific: pass user and role for auto-login with correct permissions
+ proxy_set_header X-WEBAUTH-USER $auth_user;
+ proxy_set_header X-WEBAUTH-ROLE $grafana_role;
+ {% endif %}
+
+ proxy_read_timeout 300;
+ proxy_connect_timeout 300;
+ proxy_send_timeout 300;
+
+ {% if name == 'grafana' or service.container_name == 'grafana' %}
+ # Inject Fleet Management Home button into Grafana HTML
+ # target="_top" prevents Grafana's SPA router from intercepting the click
+ sub_filter '' '';
+ sub_filter '