From 3b1ff8e4a71da0b0392c976c1fe8c5aa5bfbc002 Mon Sep 17 00:00:00 2001 From: Hannes Zietsman Date: Tue, 27 Oct 2026 05:08:12 +0200 Subject: [PATCH] Add admitted component releases and durable Fleet update recovery --- .traceweaver/intent-contract.yml | 11 + docs/release-selection.md | 62 ++++ landing-page/index.html | 272 ++++++++++++-- nginx/nginx.conf | 37 ++ requirements.md | 19 + scripts/health-api.py | 23 +- src/cryptolabs_proxy/auth.py | 142 ++++++++ src/cryptolabs_proxy/config.py | 5 + src/cryptolabs_proxy/release_client.py | 103 ++++++ src/cryptolabs_proxy/release_selection.py | 11 +- src/cryptolabs_proxy/standard_adoption.py | 330 +++++++++++++++-- .../templates/docker-compose.yml.j2 | 6 + src/cryptolabs_proxy/templates/nginx.conf.j2 | 37 ++ .../previous-standard-nginx-444d7c0.conf.j2 | 332 +++++++++++++++++ src/cryptolabs_proxy/updates.py | 344 +++++++++++++----- tests/test_auth.py | 161 ++++++++ tests/test_proxy_customer_route_repairs.py | 6 +- tests/test_release_selection.py | 24 +- tests/test_standard_adoption.py | 215 +++++++++++ tests/test_updates.py | 173 ++++++++- traceability-matrix.md | 59 +++ 21 files changed, 2200 insertions(+), 172 deletions(-) create mode 100644 .traceweaver/intent-contract.yml create mode 100644 requirements.md create mode 100644 src/cryptolabs_proxy/release_client.py create mode 100644 src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2 create mode 100644 traceability-matrix.md diff --git a/.traceweaver/intent-contract.yml b/.traceweaver/intent-contract.yml new file mode 100644 index 0000000..55b3651 --- /dev/null +++ b/.traceweaver/intent-contract.yml @@ -0,0 +1,11 @@ +schema_version: owner-approved-work-item-pointer/1 +work_item: fleet-manager-build-20261027 +canonical_contract: ../../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-20261027/intent-contract.json +requirement: REQ-FLEETUPDATE-001 +trace: TRACE-FLEETUPDATE-001 +verification: VER-FLEETUPDATE-001 +validation: VAL-FLEETUPDATE-001 +approved_plan_sha256: ec14802b47a1078ad6b1d051ba2943d43c77160e5bfe119c366eb4209caec516 +mode: implementation-no-publication +production_writes: disabled-until-approved-integration-evidence +must_not_change: unrelated worktrees, versions, secrets, customer settings, shared runner authority diff --git a/docs/release-selection.md b/docs/release-selection.md index a5d0f79..020269a 100644 --- a/docs/release-selection.md +++ b/docs/release-selection.md @@ -225,3 +225,65 @@ scope. `suite_complete` remains false, and external VPM/CLI/worker owners have explicit required outcomes. The proxy does not execute their Compose, package or SSH operations. Final published digests, independently released compatible consumers and real customer install/upgrade evidence remain release gates. + +## Fleet Manager release owner + +The administrator UI discovers admitted stable (`main`) and development (`dev`) +component releases through the installed DC release broker. Choosing a channel +filters discovery; it does not rewrite installed component channels or pins. +Preview names the exact components and target manifest. Fleet Manager itself is +an explicit `proxy-adopt` operation through the installed host adoption owner. +Service updates use one exact component group and a durable operation UUID; a +lost reply or reload returns the same job, including its terminal result. + +The fixed socket is `/run/dc-overview-release-broker.sock`. It is mounted only +when the operator's owner declaration sets `release_broker_enabled: true` and +its installed socket is root owned with mode `0600`. Browser requests cannot +supply socket paths, declarations, commands, Docker settings or trust keys. +Every release and legacy updater write requires the current Fleet administrator, +a release-purpose session token and the exact installed `FLEET_PUBLIC_ORIGIN`. +The raw health handler refuses updater/settings writes. An absent installed +origin or broker leaves the release controls unavailable. + +The proxy advertises legacy `selection_contract: 1` together with +`supported_selection_contracts: [1, 2]`. Contract 2 adds only the manifest-level +`component_channels` map. Omitted entries mean `main`; component artifact objects +and the schema-1 selection record retain their existing shape. The live source +image, as well as the host package, must support the selected contract and scoped +idempotent dispatch before a new broker job can run. + +`StandardAdoption.plan(..., proposed_selection=snapshot)` validates the exact +proposed manifest and selection bytes without publishing them. Actual adoption +revalidates the installed selection, source and existing recovery locks. +`recover_adoption(plan_id)` inspects only its retained private owner journal and +current runtime; uncertain state remains `recovery-required` and never triggers +an automatic second adoption. + +Production writes default to disabled in the DC broker policy. Real publisher +trust, fresh admission, installed bootstrap, packaged Linux checks, retained +rollback and browser disconnect/reconnect evidence are required before enablement. +Synthetic fixtures and local tests cannot activate releases. Without a real +compatible admitted dev candidate, dev remains unavailable. Restore an exact +compatible contract-1 selection before downgrading either reader. + +Selected installations refuse legacy HTTP update, pull and settings mutations with +`release_owner_required` (409). Even a current administrator and matching pin cannot +bypass fresh admission after a release is revoked. Use the admitted release preview +and apply flow; nonselected authenticated legacy operation remains supported. + +The host updater keeps its journal path separate from the helper's container path. +It derives both from exactly one inspected persistent writable volume or bind mount; +ambiguous mappings and nested shadows are refused. Status reconciliation cannot mark +a queued helper interrupted while dispatch still owns its launch transaction. + +Owner adoption records fsynced private phases around runtime mutations. Fixed restart +recovery can restore a positively identified stopped original before candidate startup, +after taking the same updater/selection exclusion and proving the source initializer +preserves every latest quiescent auth file with bootstrap authority disabled. A legitimate +write between the pre-stop capture and stop is retained as separate restoration evidence; +the pre-stop comparison remains recorded. Failed startup proof holds recovery. It never restores +an old archive over ambiguous newer candidate auth. Other uncertain phases remain +`recovery-required`. Verified in-process rollback records the actual sanitized runtime +and auth/config/selection evidence, so its changed container ID does not prevent later +`failed-restored` reconciliation. An orphaned exclusion helper is reclaimed only by its +recorded ID, image, mount and fixed owner identity under the registry lock. diff --git a/landing-page/index.html b/landing-page/index.html index a12c487..4f0b239 100644 --- a/landing-page/index.html +++ b/landing-page/index.html @@ -782,17 +782,25 @@

⚙️ System Updates

- +
- - + +
+
+

Checking installed release owner…

+
+

+                
+                
+                

+
@@ -1336,15 +1344,15 @@

${service.displayName}

const response = await fetch('/api/build-info'); if (response.ok) { const data = await response.json(); - const version = data.version || 'dev'; + const version = data.version || 'unknown'; const buildDate = data.build_date || ''; const dateStr = buildDate ? `, ${buildDate}` : ''; const vPrefix = version.startsWith('v') ? '' : 'v'; document.getElementById('buildInfo').textContent = - `Fleet Management ${vPrefix}${version}${dateStr}`; + `Fleet Manager image build: ${version === 'unknown' ? 'unavailable' : vPrefix + version}${dateStr}`; } } catch (e) { - document.getElementById('buildInfo').textContent = 'Fleet Management (dev build)'; + document.getElementById('buildInfo').textContent = 'Fleet Manager image build: unavailable'; } } @@ -1354,19 +1362,25 @@

${service.displayName}

}[char])); } + let versionsEpoch = 0; + let legacyUpdateBranch = 'main'; + let legacyServiceChannels = {}; // Load versions async function loadVersions() { const grid = document.getElementById('versionsGrid'); + const epoch = ++versionsEpoch; try { const response = await fetch('/api/versions'); if (!response.ok) throw new Error('Failed to fetch versions'); const data = await response.json(); + if (epoch !== versionsEpoch) return; const settings = data._settings || {}; + legacyUpdateBranch = settings.target_branch || 'main'; // Update branch selector const branchSelect = document.getElementById('targetBranch'); - branchSelect.value = settings.target_branch || 'main'; + // Candidate discovery filter never mutates installed component channels. // Render version cards let html = ''; @@ -1375,8 +1389,9 @@

${service.displayName}

for (const [name, info] of Object.entries(data)) { if (name === '_settings') continue; + legacyServiceChannels[name] = info.selected_channel || legacyUpdateBranch; const statusClass = info.running ? 'running' : 'stopped'; - const displayName = name.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase()); + const displayName = name === 'cryptolabs-proxy' ? 'Fleet Manager (proxy)' : name.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase()); let versionText = ''; let detailsText = ''; @@ -1387,8 +1402,8 @@

${service.displayName}

const readable = installed && !['latest', 'main', 'dev', 'Pinned'].includes(installed) ? escapeVersionText(displayVersion) : ''; if (info.latest_status === 'latest') { - versionText = '✓ Latest'; - if (readable) detailsText = `${readable}`; + versionText = readable || (info.pinned ? 'Pinned image' : 'Installed image'); + detailsText = `✓ Latest verified image comparison (${escapeVersionText(legacyUpdateBranch)})`; } else { versionText = readable || (info.pinned ? 'Pinned image' : 'Installed'); } @@ -1402,6 +1417,8 @@

${service.displayName}

versionText = 'Not running'; } + detailsText = `Observed image: ${versionText}` + detailsText; + versionText = info.selected_release ? `Selected: ${escapeVersionText(info.selected_release)} (${escapeVersionText(info.selected_channel || 'main')})` : 'No selected release'; let updateBtn = ''; if (info.running && info.latest_status !== 'latest') { updateBtn = `${info.latest_status === 'update_available' ? 'Update available' : 'Unable to verify · Check Updates'}`; @@ -1412,6 +1429,8 @@

${service.displayName}

updateBtn += 'Pinned image · managed separately'; } else if (!info.running) { updateBtn = 'Use service setup to start'; + } else if (info.selected_release) { + updateBtn += 'Use admitted releases above'; } else if (info.update_available === true) { updateBtn += ``; updatableCount++; @@ -1451,7 +1470,7 @@

${service.displayName}

} // Enable/disable Update Running button based on whether any services need updating - const updateAllBtn = document.querySelector('.update-buttons .btn-primary'); + const updateAllBtn = document.querySelector('.legacy-update-all'); if (updateAllBtn) { if (updatableCount > 0) { updateAllBtn.disabled = false; @@ -1470,25 +1489,14 @@

${service.displayName}

loadWatchdogStatus(); } catch (e) { + if (epoch !== versionsEpoch) return; console.error('Failed to load versions:', e); grid.innerHTML = `

Failed to load versions: ${e.message}

`; } } async function saveBranchSetting() { - const branch = document.getElementById('targetBranch').value; - try { - const response = await fetch('/api/update-settings', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ branch }) - }); - if (!response.ok) throw new Error('Failed to save'); - showUpdateStatus('info', `Target branch set to: ${branch}`); - loadVersions(); - } catch (e) { - showUpdateStatus('error', `Failed to save branch setting: ${e.message}`); - } + return loadReleaseCandidates(); } let activeUpdateJob = null; @@ -1502,13 +1510,16 @@

${service.displayName}

} catch (e) { /* Storage may be disabled; this page can still poll. */ } } - async function pollUpdateJob(id, failures = 0) { + async function pollUpdateJob(id, failures = 0, expected = id) { + if (activeUpdateJob && activeUpdateJob !== expected) return; activeUpdateJob = id; clearTimeout(updatePollTimer); try { const response = await fetch(`/api/update-status?id=${encodeURIComponent(id)}`); if (!response.ok) throw new Error('Update status unavailable'); const job = await response.json(); + if (activeUpdateJob !== expected) return; + if (!id && job.id) { id = job.id; expected = job.id; activeUpdateJob = id; rememberUpdateJob(id); } if (job.state === 'queued' || job.state === 'running') { showUpdateStatus('info', 'Update operation in progress. Services restart only after their images are ready.'); updatePollTimer = setTimeout(() => pollUpdateJob(id), 2000); @@ -1523,6 +1534,7 @@

${service.displayName}

loadVersions(); loadServices(); } catch (e) { + if (activeUpdateJob !== expected) return; if (failures >= 60) { showUpdateStatus('error', 'Update status is unavailable. Reload to reconnect before starting another update.'); return; @@ -1537,12 +1549,12 @@

${service.displayName}

showUpdateStatus('info', 'An update operation is already in progress.'); return; } - const branch = document.getElementById('targetBranch').value; + const branch = legacyServiceChannels[service] || legacyUpdateBranch; showUpdateStatus('info', action === 'pull' ? 'Checking images…' : 'Preparing update…'); try { const response = await fetch(action === 'pull' ? '/api/pull' : '/api/update', { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': await releaseToken() }, body: JSON.stringify({ service, branch }) }); const data = await response.json(); @@ -1590,6 +1602,206 @@

${service.displayName}

} } + const RELEASE_OPERATION_KEY = 'fleet-release-operation'; + const RELEASE_APPLY_KEY = 'fleet-release-apply'; + let releaseSession = null; + let releaseListing = null; + let releasePreview = null; + let releaseExpected = null; + let releaseEpoch = 0; + let releaseOperationId = null; + let releasePollTimer = null; + let releaseBusy = false; + + async function releaseToken() { + const response = await fetch('/api/releases/session', {cache:'no-store'}); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || 'Current administrator access required.'); + releaseSession = data; + return data.csrf_token; + } + + async function releaseRequest(path, body) { + const options = {cache:'no-store'}; + if (body !== undefined) { + options.method = 'POST'; + options.headers = {'Content-Type':'application/json', 'X-CSRF-Token':await releaseToken()}; + options.body = JSON.stringify(body); + } + const response = await fetch(path, options); + const data = await response.json(); + if (!response.ok) { + const error = new Error(data.error || 'Release owner unavailable.'); + error.status = response.status; + throw error; + } + return data; + } + + async function loadReleaseCandidates() { + const epoch = ++releaseEpoch; + releasePreview = null; + document.getElementById('releaseApplyButton').disabled = true; + document.getElementById('releasePreviewButton').disabled = true; + const message = document.getElementById('releaseAvailability'); + const container = document.getElementById('releaseCandidates'); + container.replaceChildren(); + document.getElementById('releasePreview').textContent = ''; + try { + await releaseToken(); + const channel = document.getElementById('targetBranch').value; + const listing = await releaseRequest('/api/releases/candidates?channel=' + encodeURIComponent(channel)); + if (epoch !== releaseEpoch) return; + releaseListing = listing; + releaseExpected = listing.selected; + const choices = new Map(); + for (const candidate of listing.candidates || []) { + for (const component of candidate.components || []) { + if (component.channel !== channel) continue; + if (!choices.has(component.component)) choices.set(component.component, []); + choices.get(component.component).push({candidate, component}); + } + } + for (const [name, available] of choices) { + const row = document.createElement('div'); + row.style.cssText = 'display:flex;flex-wrap:wrap;gap:.75rem;align-items:center;margin:.75rem 0'; + const label = document.createElement('label'); + const check = document.createElement('input'); + check.type = 'checkbox'; check.dataset.component = name; + check.disabled = releaseBusy; + label.append(check, document.createTextNode(' ' + (name === 'cryptolabs-proxy' ? 'Fleet Manager (proxy)' : name))); + const select = document.createElement('select'); + select.dataset.releaseComponent = name; + select.setAttribute('aria-label', name + ' admitted available release'); + for (const {candidate, component} of available) { + const option = document.createElement('option'); + option.value = candidate.candidate_id; + option.textContent = `${component.release || component.identity} (${component.channel})`; + select.append(option); + } + select.disabled = check.disabled; + const current = document.createElement('span'); + const selected = listing.selected && listing.selected.components && listing.selected.components[name]; + current.textContent = selected ? `Selected: ${selected.release || selected.identity} (${selected.channel})` : 'Selected release unavailable'; + row.append(label, current, select); + container.append(row); + } + const admitted = choices.size > 0; + message.textContent = !admitted ? `No admitted ${channel === 'main' ? 'stable' : 'dev'} releases are available.` + : listing.writes_enabled ? 'Choose components, then review the exact update scope.' + : 'Admitted releases found. Updates remain disabled until the installed owner and trust checks pass.'; + document.getElementById('releasePreviewButton').disabled = !admitted || releaseBusy; + // Dev is available only when the installed owner actually discovers an admitted dev release. + const dev = await releaseRequest('/api/releases/candidates?channel=dev').catch(() => ({candidates:[]})); + if (epoch !== releaseEpoch) return; + const option = document.querySelector('#targetBranch option[value="dev"]'); + const hasDev = (dev.candidates || []).some(c => (c.components || []).some(p => p.channel === 'dev')); + option.disabled = !hasDev; + option.textContent = hasDev ? 'dev (admitted development)' : 'dev (unavailable until admitted)'; + } catch (error) { + if (epoch !== releaseEpoch) return; + message.textContent = error.message + ' No update started.'; + } + } + + async function previewRelease() { + if (releaseBusy || !releaseListing || !releaseExpected) return; + const components = [], candidates = new Set(); + for (const check of document.querySelectorAll('#releaseCandidates input:checked')) { + const name = check.dataset.component; + components.push(name); + const select = Array.from(document.querySelectorAll('#releaseCandidates select')).find(s => s.dataset.releaseComponent === name); + candidates.add(select.value); + } + const previewText = document.getElementById('releasePreview'); + if (!components.length) { previewText.textContent = 'Choose at least one component.'; return; } + if (components.includes('cryptolabs-proxy') && components.length !== 1) { + previewText.textContent = 'Review Fleet Manager separately; mixed replacement groups require an admitted owner sequence.'; return; + } + const epoch = releaseEpoch; + try { + const result = await releaseRequest('/api/releases/preview', { + candidate_ids:Array.from(candidates), components, + expected_selection_id:releaseExpected.selection_id, + expected_selection_sha256:releaseExpected.selection_sha256, + operation_kind:components[0] === 'cryptolabs-proxy' ? 'proxy-adopt' : 'services-update' + }); + if (epoch !== releaseEpoch || releaseBusy) return; + releasePreview = result; + previewText.textContent = `Update scope: ${result.components.join(', ')}\nTarget manifest: ${result.manifest_sha256}\n` + + (result.owner_plan ? `Fleet Manager target: ${result.owner_plan.image}\n` : '') + + (result.disconnect_expected ? 'Fleet Manager will briefly disconnect. Reconnect to this same operation. Retained rollback is owned by the host.\n' : '') + + `Preview: ${result.preview_id}`; + document.getElementById('releaseApplyButton').disabled = !releaseListing.writes_enabled; + } catch (error) { if (epoch === releaseEpoch) previewText.textContent = error.message; } + } + + async function applyRelease() { + if (releaseBusy) return; + let body = null; + try { body = JSON.parse(localStorage.getItem(RELEASE_APPLY_KEY)); } catch (e) {} + if (!body) { + if (!releasePreview || !releaseExpected || !releaseListing.writes_enabled) return; + body = {operation_id:crypto.randomUUID(),preview_id:releasePreview.preview_id,preview_sha256:releasePreview.preview_sha256, + expected_selection_id:releaseExpected.selection_id,expected_selection_sha256:releaseExpected.selection_sha256}; + } + releaseOperationId = body.operation_id; + try { + localStorage.setItem(RELEASE_OPERATION_KEY,releaseOperationId); + localStorage.setItem(RELEASE_APPLY_KEY,JSON.stringify(body)); + } catch (e) { + document.getElementById('releaseOperation').textContent = 'Browser storage is required to retain this operation through a disconnect.'; return; + } + releaseBusy = true; + document.getElementById('releaseApplyButton').disabled = true; + document.getElementById('releasePreviewButton').disabled = true; + document.getElementById('releaseOperation').textContent = 'Submitting operation ' + releaseOperationId; + try { await releaseRequest('/api/releases/apply',body); } + catch (error) { document.getElementById('releaseOperation').textContent = 'Reply unavailable. Reconnecting to the retained operation; retry uses the same identity.'; } + return pollReleaseOperation(releaseOperationId); + } + + async function pollReleaseOperation(id, failures = 0) { + if (releaseOperationId !== id) return; + clearTimeout(releasePollTimer); + try { + const operation = await releaseRequest('/api/releases/operations/' + encodeURIComponent(id)); + if (releaseOperationId !== id) return; + const terminal = ['completed','failed-restored','failed','selection-conflict','recovery-required'].includes(operation.state); + document.getElementById('releaseOperation').textContent = `Operation ${id}: ${operation.state}` + (operation.reason ? ` · ${operation.reason}` : ''); + document.getElementById('releaseRecoverButton').hidden = operation.state !== 'recovery-required'; + if (!terminal) { + releaseBusy = true; + releasePollTimer = setTimeout(() => pollReleaseOperation(id),2000); + return; + } + releaseBusy = operation.state === 'recovery-required'; + if (!releaseBusy) { + localStorage.removeItem(RELEASE_APPLY_KEY); + loadVersions(); loadServices(); loadReleaseCandidates(); + } + } catch (error) { + if (releaseOperationId !== id) return; + document.getElementById('releaseOperation').textContent = error.status === 404 + ? `Operation ${id} has no confirmed receipt. Retry Apply uses the same request.` + : `Reconnecting to operation ${id}. The installed owner retains progress through a disconnect.`; + if (error.status === 404) { + releaseBusy = false; + document.getElementById('releaseApplyButton').disabled = false; + return; + } + if (failures < 60) releasePollTimer = setTimeout(() => pollReleaseOperation(id,failures+1),3000); + } + } + + async function recoverRelease() { + if (!releaseOperationId) return; + try { + await releaseRequest('/api/releases/operations/' + encodeURIComponent(releaseOperationId) + '/recover', {operation_id:releaseOperationId}); + await pollReleaseOperation(releaseOperationId); + } catch (error) { document.getElementById('releaseOperation').textContent = error.message; } + } + // DC Watchdog agent status - integrated into versions grid (single card only) let watchdogStatusLoading = false; async function loadWatchdogStatus() { @@ -1843,9 +2055,13 @@

${service.displayName}

loadServices(); loadBuildInfo(); loadVersions(); // This calls loadWatchdogStatus() at the end + loadReleaseCandidates(); try { const pendingUpdate = localStorage.getItem(UPDATE_JOB_KEY); if (pendingUpdate) pollUpdateJob(pendingUpdate); + else pollUpdateJob('').catch(() => {}); + const operation = localStorage.getItem(RELEASE_OPERATION_KEY); + if (operation) { releaseOperationId = operation; releaseBusy = true; pollReleaseOperation(operation); } } catch (e) { /* Browser storage is optional. */ } // Auto-refresh every 30 seconds diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 42065e2..5178da4 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -122,6 +122,43 @@ http { proxy_pass http://health_api/api/build-info; } + # Fleet release writes and status use live session authority in Flask. + location ^~ /api/releases/ { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + location = /api/update { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + location = /api/pull { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + location = /api/update-settings { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + location = /api/update-status { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + # Version management and update endpoints (protected) location /api/versions { auth_request /_auth_check; diff --git a/requirements.md b/requirements.md new file mode 100644 index 0000000..74a829c --- /dev/null +++ b/requirements.md @@ -0,0 +1,19 @@ +# Fleet Manager bounded work-item authority + +REQ-FLEETUPDATE-001 is an administrative label for the exact five-step revision 2 +Fleet Manager plan approved by the owner on 27 October 2026. It adds no requirement +semantics or approval. + +Canonical authority: [owner receipt](../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-20261027/owner-approval.json), +[approved plan](../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-plan/plan.json), +and [shared work-item requirements](../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-20261027/requirements.md). +Approved plan SHA256: `ec14802b47a1078ad6b1d051ba2943d43c77160e5bfe119c366eb4209caec516`. + +Repair truthful version/job status and let a Fleet administrator preview and +update admitted managed components, including Fleet Manager itself through the +installed host adoption owner. First installation remains operator-owned; +routine later updates use the UI or equivalent CLI. + +Production trust/admission, bootstrap, packaged Linux and browser acceptance, +retained rollback and final owner acceptance remain held until independently +verified. No feature version bump or publication is part of this worktree change. diff --git a/scripts/health-api.py b/scripts/health-api.py index 8d87886..a34ca12 100644 --- a/scripts/health-api.py +++ b/scripts/health-api.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 """ Health API server for CryptoLabs Proxy. Detects running Docker containers, reports their status, and manages updates. @@ -444,6 +445,7 @@ def get_vpm_prerequisite_for_display(): return value +# TRACEWEAVER: entrypoint=get_all_versions; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 def get_all_versions(): """Get version info for all services.""" versions = {} @@ -486,6 +488,18 @@ def get_all_versions(): if running: version_info.update(update_status(name, settings.get('branch', 'main'))) + from cryptolabs_proxy.updates import installed_selection + try: + snapshot=installed_selection() + except UpdateError: + snapshot=None + for name,info in versions.items(): + info['component']=name + info['observed_source']='running-image-metadata' + selected=snapshot['manifest']['components'].get(name) if snapshot else None + info['selected_release']=selected['release'] if selected else None + info['selected_channel']=snapshot['manifest'].get('component_channels',{}).get(name,'main') if selected else None + info['admitted_available_release']=None # Add configured branch versions['_settings'] = { 'target_branch': settings.get('branch', 'main'), @@ -506,10 +520,13 @@ def get_build_info(): if '=' in line: key, value = line.split('=', 1) info[key.lower()] = value + info.update(component='cryptolabs-proxy',metadata_source='image-build-info') return info except FileNotFoundError: return { - 'version': 'dev', + 'component': 'cryptolabs-proxy', + 'metadata_source': 'unavailable', + 'version': 'unknown', 'branch': 'unknown', 'commit': 'unknown', 'build_date': 'unknown', @@ -613,6 +630,10 @@ def do_GET(self): self.end_headers() def do_POST(self): + # Browser writes belong exclusively to live-admin Flask authority. + if urlparse(self.path).path in ('/api/update','/api/pull','/api/update-settings'): + self.send_json({'error':'Use the authenticated release API.'},403) + return parsed = urlparse(self.path) path = parsed.path diff --git a/src/cryptolabs_proxy/auth.py b/src/cryptolabs_proxy/auth.py index 6bf3316..ca96fe0 100644 --- a/src/cryptolabs_proxy/auth.py +++ b/src/cryptolabs_proxy/auth.py @@ -1,3 +1,4 @@ +# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 """ CryptoLabs Proxy - Unified Authentication Module @@ -593,6 +594,7 @@ def verify_proxy_auth(headers: dict) -> Optional[dict]: # FLASK INTEGRATION HELPERS # ============================================================================= +# TRACEWEAVER: entrypoint=create_flask_auth_app; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 def create_flask_auth_app(): """ Create a Flask app for the proxy authentication API. @@ -1280,6 +1282,144 @@ def derive(purpose: bytes) -> str: 'csrf_token': derive(b'cryptolabs/vpm/csrf/v1:'), }, 200 + def release_authority(mutation=False): + from .release_client import ReleaseBrokerError + user,status=current_admin_user() + if status!=200: raise ReleaseBrokerError('admin_required','Current administrator access required.',status) + cookie=request.cookies.get(app.config['SESSION_COOKIE_NAME']) + if not cookie: raise ReleaseBrokerError('admin_required','Current administrator access required.',401) + binding=session.get('_release_session') + valid=(isinstance(binding,dict) and binding.get('principal')==user['username'] + and isinstance(binding.get('nonce'),str) and len(binding['nonce'])==64 + and all(c in '0123456789abcdef' for c in binding['nonce'])) + if not valid: + if mutation: + raise ReleaseBrokerError('csrf_or_origin','Release session token must be issued before mutation.',403) + # This integrity-protected identifier is visible in Flask's signed + # cookie. Authority still depends on the server HMAC and live user. + binding={'principal':user['username'],'nonce':secrets.token_hex(32)} + session['_release_session']=binding + material=json.dumps([user['username'],binding['nonce']],separators=(',',':')).encode('utf-8') + def derive(purpose): + return hmac.new(AUTH_SECRET_KEY.encode(),purpose+material,hashlib.sha256).hexdigest() + token=derive(b'cryptolabs/releases/csrf/v1:') + if mutation: + # Only an installed canonical origin grants browser mutation authority. + origin=app.config.get('RELEASE_PUBLIC_ORIGIN') or os.environ.get('FLEET_PUBLIC_ORIGIN','') + supplied=request.headers.get('X-CSRF-Token','') + if not origin or request.headers.get('Origin')!=origin or not hmac.compare_digest(supplied,token): + raise ReleaseBrokerError('csrf_or_origin','Release origin and session token required.',403) + return {'kind':'fleet-admin','username':user['username'],'subject':derive(b'cryptolabs/releases/subject/v1:')},token + + def release_error(error): + return jsonify({'error':error.message,'code':error.code}),error.status + + def release_dispatch(action,params): + from .release_client import ReleaseBrokerClient,ReleaseBrokerError,validate_params + try: + actor,_=release_authority(request.method=='POST') + validate_params(action,params) + return jsonify(ReleaseBrokerClient().request(action,params,actor=actor)) + except ReleaseBrokerError as error: return release_error(error) + + def release_body(): + from .release_client import strict_json,ReleaseBrokerError,MAX_REQUEST + if request.content_length is None or request.content_length>MAX_REQUEST: + raise ReleaseBrokerError('invalid_request','Bounded JSON body required.',400) + try: + value=strict_json(request.get_data(cache=False)) + if not isinstance(value,dict): raise ValueError() + return value + except (ValueError,UnicodeError): raise ReleaseBrokerError('invalid_request','Invalid release JSON.',400) from None + + @app.route('/api/releases/session') + def release_session(): + from .release_client import ReleaseBrokerError,SOCKET_PATH + try: + actor,token=release_authority() + return jsonify({'authenticated':True,'username':actor['username'],'csrf_token':token, + 'broker_available':Path(SOCKET_PATH).exists(), + 'origin_configured':bool(app.config.get('RELEASE_PUBLIC_ORIGIN') or os.environ.get('FLEET_PUBLIC_ORIGIN'))}) + except ReleaseBrokerError as error: return release_error(error) + + @app.route('/api/releases/candidates') + def release_candidates(): + if set(request.args)-{'channel'}: return jsonify({'error':'Invalid discovery query.'}),400 + return release_dispatch('list',{'channel':request.args.get('channel','main')}) + + @app.route('/api/releases/preview',methods=['POST']) + @app.route('/api/releases/apply',methods=['POST']) + def release_mutation(): + from .release_client import ReleaseBrokerError + try: + release_authority(True) + return release_dispatch(request.path.rsplit('/',1)[-1],release_body()) + except ReleaseBrokerError as error: return release_error(error) + + @app.route('/api/releases/operations/') + def release_operation_status(operation_id): + return release_dispatch('status',{'operation_id':operation_id}) + + @app.route('/api/releases/operations//recover',methods=['POST']) + def release_operation_recover(operation_id): + from .release_client import ReleaseBrokerError + try: + release_authority(True) + body=release_body() + if body not in ({},{'operation_id':operation_id}): + raise ReleaseBrokerError('invalid_request','Recovery identifier must match operation.',400) + return release_dispatch('recover',{'operation_id':operation_id}) + except ReleaseBrokerError as error: return release_error(error) + + @app.route('/api/update-status') + def legacy_update_status(): + from .release_client import ReleaseBrokerError + from .updates import job_status,UpdateError + try: + release_authority() + return jsonify(job_status(request.args.get('id',''))) + except ReleaseBrokerError as error: return release_error(error) + except UpdateError as error: return jsonify({'error':str(error)}),404 + + @app.route('/api/update',methods=['POST']) + @app.route('/api/pull',methods=['POST']) + @app.route('/api/update-settings',methods=['GET','POST']) + def legacy_release_mutation(): + from .release_client import ReleaseBrokerError + from . import updates + try: + release_authority(request.method=='POST') + settings_path=DATA_DIR/'update-settings.json' + settings={'branch':'main','auto_update':True,'update_schedule':'daily'} + try: settings.update(json.loads(settings_path.read_text())) + except (OSError,ValueError,TypeError): pass + if request.method=='GET': return jsonify(settings) + body=release_body() + snapshot=updates.installed_selection() + if snapshot: + raise ReleaseBrokerError('release_owner_required','Selected releases require the Fleet Manager admitted release owner. Preview updates through /api/releases.',409) + if request.path=='/api/update-settings': + if set(body)-{'branch','auto_update','update_schedule'}: + raise ReleaseBrokerError('invalid_request','Invalid settings fields.',400) + if ('branch' in body and body['branch'] not in ('main','dev') or + 'auto_update' in body and type(body['auto_update']) is not bool or + 'update_schedule' in body and body['update_schedule'] not in ('daily','weekly','manual')): + raise ReleaseBrokerError('invalid_request','Invalid update settings.',400) + settings.update(body) + updates._write_json(settings_path,settings) + return jsonify({'success':True,'settings':settings}) + if set(body)-{'service','branch','selection_id','selection_sha256'}: + raise ReleaseBrokerError('invalid_request','Invalid updater fields.',400) + branch=body.get('branch',settings['branch']) + if 'selection_id' in body or 'selection_sha256' in body: + raise ReleaseBrokerError('selection_conflict','Installed selection unavailable.',409) + job=updates.submit_job(body.get('service','all'),branch,'pull' if request.path=='/api/pull' else 'update') + return jsonify({'success':True,'job':job}),202 + except ReleaseBrokerError as error: return release_error(error) + except updates.UpdateBusy as error: return jsonify({'error':str(error),'job':error.job}),409 + except updates.UpdateError as error: return jsonify({'error':str(error)}),400 + except (OSError,ValueError,TypeError): return jsonify({'error':'Updater unavailable; no update confirmed.'}),503 + # ========================================================================= # ROUTES # ========================================================================= @@ -1303,6 +1443,7 @@ def login(): error = 'Passwords do not match' else: create_user(username, password, 'admin') + session.pop('_release_session', None) session['logged_in'] = True session['username'] = username session['role'] = 'admin' @@ -1316,6 +1457,7 @@ def login(): # Normal login user = verify_user(username, password) if user: + session.pop('_release_session', None) session['logged_in'] = True session['username'] = user['username'] session['role'] = user['role'] diff --git a/src/cryptolabs_proxy/config.py b/src/cryptolabs_proxy/config.py index 5d4de6b..0f1e42a 100644 --- a/src/cryptolabs_proxy/config.py +++ b/src/cryptolabs_proxy/config.py @@ -1,3 +1,4 @@ +# TRACEWEAVER: file-role=fleet-release-compose-consumer; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 """Configuration management for CryptoLabs Proxy.""" import os @@ -59,6 +60,7 @@ def render_vpm_internal_auth_blocks() -> str: return get_jinja_env().get_template("vast-price-manager-auth-blocks.conf.j2").render() +# TRACEWEAVER: entrypoint=generate_docker_compose; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 def generate_docker_compose(config_dir: Path, domain: str = None, use_letsencrypt: bool = False, *, release_selection_dir=None, platform=None): """Generate docker-compose.yml from template.""" @@ -81,7 +83,10 @@ def generate_docker_compose(config_dir: Path, domain: str = None, use_letsencryp proxy_image = selected['cryptolabs-proxy']['reference'] proxy_origin = selected['cryptolabs-proxy']['origin'] + broker_enabled=registry.config.get('release_broker_enabled',False) + if type(broker_enabled) is not bool: raise ValueError('release_broker_enabled must be boolean') content = template.render( + release_broker_enabled=broker_enabled, domain=domain, use_letsencrypt=use_letsencrypt, release_selection_dir=str(Path(release_selection_dir).resolve()) if release_selection_dir else None, diff --git a/src/cryptolabs_proxy/release_client.py b/src/cryptolabs_proxy/release_client.py new file mode 100644 index 0000000..6fe64db --- /dev/null +++ b/src/cryptolabs_proxy/release_client.py @@ -0,0 +1,103 @@ +# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 +"""Bounded client for the installed Fleet release owner, with no remote paths.""" +import json +import os +from pathlib import Path +import re +import socket +import stat +import uuid + +from .release_selection import COMPONENTS + +SOCKET_PATH = '/run/dc-overview-release-broker.sock' +MAX_REQUEST = 65536 +MAX_RESPONSE = 1048576 + + +class ReleaseBrokerError(ValueError): + def __init__(self, code, message='Release owner unavailable.', status=503): + self.code, self.message, self.status = code, message, status + super().__init__(message) + + +def strict_json(raw): + if isinstance(raw,bytes): raw=raw.decode('utf-8') + def unique(pairs): + value = {} + for key, item in pairs: + if key in value: raise ValueError('duplicate_field') + value[key] = item + return value + return json.loads(raw, object_pairs_hook=unique, parse_constant=lambda _: (_ for _ in ()).throw(ValueError('invalid_number'))) + + +def validate_params(action, params): + fields = {'list': {'channel'}, 'preview': {'candidate_ids','components','expected_selection_id','expected_selection_sha256','operation_kind'}, + 'apply': {'operation_id','preview_id','preview_sha256','expected_selection_id','expected_selection_sha256'}, + 'status': {'operation_id'}, 'recover': {'operation_id'}} + if action not in fields or not isinstance(params, dict) or set(params) != fields[action]: + raise ReleaseBrokerError('invalid_request', 'Invalid release request fields.', 400) + for key, value in params.items(): + if key in ('candidate_ids','components'): + if (not isinstance(value, list) or not 1 <= len(value) <= 32 + or any(not isinstance(x,str) or not 1 <= len(x) <= 128 for x in value) + or len(set(value)) != len(value) or key == 'components' and any(x not in COMPONENTS for x in value)): + raise ReleaseBrokerError('invalid_request','Invalid release component scope.',400) + elif not isinstance(value,str) or not 1 <= len(value) <= 128: + raise ReleaseBrokerError('invalid_request','Invalid release request value.',400) + elif key.endswith('_sha256') and not re.fullmatch('[0-9a-f]{64}',value): + raise ReleaseBrokerError('invalid_request','Invalid release digest.',400) + elif key == 'operation_id': + try: + if str(uuid.UUID(value)) != value: raise ValueError() + except ValueError: raise ReleaseBrokerError('invalid_request','Invalid operation identifier.',400) from None + if 'channel' in params and params['channel'] not in ('main','dev'): + raise ReleaseBrokerError('invalid_channel','Use main or dev.',400) + if 'operation_kind' in params and params['operation_kind'] not in ('services-update','proxy-adopt'): + raise ReleaseBrokerError('invalid_request','Invalid release operation.',400) + return params + + +class ReleaseBrokerClient: +# TRACEWEAVER: entrypoint=request; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 + def request(self, action, params, *, actor): + validate_params(action,params) + if (not isinstance(actor,dict) or set(actor) != {'kind','username','subject'} + or actor.get('kind') != 'fleet-admin' or any(not isinstance(actor[k],str) or not 1<=len(actor[k])<=128 for k in ('username','subject'))): + raise ReleaseBrokerError('invalid_actor','Release administrator identity required.',403) + raw=json.dumps({'schema_version':1,'action':action,'params':params,'actor':actor},separators=(',',':'),allow_nan=False).encode()+b'\n' + if len(raw)>MAX_REQUEST: raise ReleaseBrokerError('request_too_large','Release request too large.',413) + try: + metadata=Path(SOCKET_PATH).lstat() + if not stat.S_ISSOCK(metadata.st_mode) or metadata.st_uid!=0 or metadata.st_mode & 0o077: + raise OSError('unsafe_owner_socket') + with socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) as connection: + connection.settimeout(3) + connection.connect(SOCKET_PATH) + connection.settimeout(15) + connection.sendall(raw) + connection.shutdown(socket.SHUT_WR) + response=bytearray() + while True: + chunk=connection.recv(min(65536,MAX_RESPONSE+1-len(response))) + if not chunk: break + response.extend(chunk) + if len(response)>MAX_RESPONSE: raise ValueError('response_too_large') + value=strict_json(bytes(response)) + if (not isinstance(value,dict) or type(value.get('schema_version')) is not int or value['schema_version']!=1 + or type(value.get('ok')) is not bool): raise ValueError('invalid_response') + if value['ok']: + if set(value)!={'schema_version','ok','result'} or not isinstance(value['result'],dict): raise ValueError('invalid_response') + return value['result'] + if set(value)!={'schema_version','ok','error'}: raise ValueError('invalid_response') + error=value['error'] + if (not isinstance(error,dict) or set(error)!={'code','message','status'} + or type(error['status']) is not int or not 400<=error['status']<=599 + or not isinstance(error['code'],str) or not re.fullmatch('[a-z][a-z0-9_-]{0,79}',error['code']) + or not isinstance(error['message'],str) or len(error['message'])>512): raise ValueError('invalid_response') + raise ReleaseBrokerError(error['code'],error['message'],error['status']) + except ReleaseBrokerError: + raise + except (OSError,ValueError,UnicodeError,TypeError): + raise ReleaseBrokerError('owner_unavailable','Release owner unavailable; no update confirmed.',503) from None diff --git a/src/cryptolabs_proxy/release_selection.py b/src/cryptolabs_proxy/release_selection.py index ab3be7b..4b54065 100644 --- a/src/cryptolabs_proxy/release_selection.py +++ b/src/cryptolabs_proxy/release_selection.py @@ -1,3 +1,4 @@ +# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 """Pure, bounded contract1 consumer; approval remains with the host operator. Hashes identify exact imported bytes, not publisher trust. Never interpret data @@ -71,13 +72,14 @@ def unique(pairs): raise SelectionError('invalid_json') from None +# TRACEWEAVER: entrypoint=validate; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 def validate(selection_bytes, manifest_bytes, *, selection_id=None, selection_sha256=None): """Validate exact bytes, all declared artifacts and every enabled role.""" selection = _json(selection_bytes) manifest = _json(manifest_bytes) _fields(selection, ('schema_version', 'selection_id', 'manifest_sha256', 'components')) - _fields(manifest, ('schema_version', 'selection_contract', 'baseline_id', 'components')) - if type(selection['schema_version']) is not int or selection['schema_version'] != 1 or type(manifest['schema_version']) is not int or manifest['schema_version'] != 1 or type(manifest['selection_contract']) is not int or manifest['selection_contract'] != 1: + _fields(manifest, ('schema_version', 'selection_contract', 'baseline_id', 'components'), ('component_channels',)) + if type(selection['schema_version']) is not int or selection['schema_version'] != 1 or type(manifest['schema_version']) is not int or manifest['schema_version'] not in (1, 2) or type(manifest['selection_contract']) is not int or manifest['selection_contract'] != manifest['schema_version']: raise SelectionError('unsupported_contract') _identifier(selection['selection_id']); _identifier(manifest['baseline_id']) _hash(selection['manifest_sha256']) @@ -91,6 +93,11 @@ def validate(selection_bytes, manifest_bytes, *, selection_id=None, selection_sh raise SelectionError('selection_changed') if not isinstance(manifest['components'], dict) or not isinstance(selection['components'], dict) or not selection['components']: raise SelectionError('invalid_components') + channels = manifest.get('component_channels', {}) + if (not isinstance(channels, dict) or set(channels) - set(manifest['components']) + or any(type(v) is not str or v not in ('main', 'dev') for v in channels.values()) + or (manifest['schema_version'] == 1 and 'component_channels' in manifest)): + raise SelectionError('invalid_component_channels') for name, component in manifest['components'].items(): if name not in COMPONENTS: raise SelectionError('unknown_component') diff --git a/src/cryptolabs_proxy/standard_adoption.py b/src/cryptolabs_proxy/standard_adoption.py index 0d67389..4846ef5 100644 --- a/src/cryptolabs_proxy/standard_adoption.py +++ b/src/cryptolabs_proxy/standard_adoption.py @@ -1,3 +1,4 @@ +# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 """Explicit standard configuration adoption, without Nginx passthrough. Public manifests contain identities and declarative fields only. Inspected @@ -22,7 +23,7 @@ build_recreate_request, endpoint_configurations, wait_for_container_healthy, wait_for_proxy_ready, PROXY_READY_TIMEOUT, _snapshot_registry_files, _write_registry_backup, _restore_registry_files) -from .release_selection import load, targets, sha256, SelectionError +from .release_selection import load, targets, sha256, SelectionError, validate as validate_selection from .services import ServiceRegistry, DEFAULT_SERVICES @@ -78,6 +79,42 @@ def _canonical(value): return json.dumps(value,sort_keys=True,separators=(',',':')).encode() +def _durable_private(path, content): + """Atomic, fsynced owner journal writes; never expose inspected values.""" + import os as filesystem + import uuid + path=Path(path) + temporary=path.with_name('.'+path.name+'-'+uuid.uuid4().hex) + descriptor=filesystem.open(temporary,filesystem.O_WRONLY|filesystem.O_CREAT|filesystem.O_EXCL|filesystem.O_NOFOLLOW,0o600) + try: + with filesystem.fdopen(descriptor,'wb') as stream: + stream.write(content); stream.flush(); filesystem.fsync(stream.fileno()) + filesystem.replace(temporary,path) + descriptor=filesystem.open(path.parent,filesystem.O_RDONLY|filesystem.O_DIRECTORY) + try: filesystem.fsync(descriptor) + finally: filesystem.close(descriptor) + finally: + try: temporary.unlink() + except FileNotFoundError: pass + + +def _restoration_evidence(engine, source, plan, expected, config_dir): + serving=engine.inspect('cryptolabs-proxy') + sanitized=[e for e in source['Config'].get('Env',[]) if not e.startswith('FLEET_ADMIN_PASS=')]+['FLEET_ADMIN_PASS='] + if (not serving.get('State',{}).get('Running') or serving['Image']!=source['Image'] + or serving['Config'].get('Env',[])!=sanitized + or sha256(engine.read_config('cryptolabs-proxy'))!=plan['config_sha256'] + or auth_state(engine.archive_auth('cryptolabs-proxy'),allow_active_jobs=True)!=expected): + raise AdoptionError('source_recovery_unverified') + registry=_snapshot_registry_files(ServiceRegistry(config_dir)) + registry_sha=sha256(_canonical({p.name:sha256(data) if data is not None else None for p,data in registry.items()})) + if registry_sha!=plan['registry_sha256']: raise AdoptionError('source_registry_recovery_unverified') + return {'schema_version':1,'plan_id':plan['plan_id'],'runtime_sha256':_runtime_identity(serving), + 'container_id':serving['Id'],'image_id':serving['Image'],'config_sha256':plan['config_sha256'], + 'selection_sha256':plan['selection_sha256'],'registry_sha256':registry_sha, + 'auth_state_sha256':sha256(_canonical(expected)),'startup_verified':True,'health_verified':True} + + def parse_nginx(data): """Bounded structural comparison; unknown directives are never executed.""" if not isinstance(data,bytes) or len(data)>MAX_CONFIG: @@ -142,8 +179,10 @@ def _safe_upstream(words): def _declaration(value): required={'schema_version','domain','tls','release_selection_dir','services','obsolete_overlays','obsolete_owner_labels'} - if not isinstance(value,dict) or set(value)-required-{'host_forward_groups','acme_challenge','hostname_policy','http_policy'} or not required<=set(value) or type(value['schema_version']) is not int or value['schema_version']!=1: + if not isinstance(value,dict) or set(value)-required-{'host_forward_groups','acme_challenge','hostname_policy','http_policy','release_broker_enabled'} or not required<=set(value) or type(value['schema_version']) is not int or value['schema_version']!=1: raise AdoptionError('declaration_schema_invalid') + if 'release_broker_enabled' in value and type(value['release_broker_enabled']) is not bool: + raise AdoptionError('release_broker_policy_invalid') domain=value['domain'] if not isinstance(domain,str) or not DOMAIN.fullmatch(domain) or '..' in domain or domain.endswith('.') or value['tls'] not in ('self-signed','letsencrypt'): raise AdoptionError('domain_or_tls_invalid') @@ -187,6 +226,22 @@ def render_standard(declaration): return get_jinja_env().get_template('nginx.conf.j2').render(domain=declaration['domain'],letsencrypt=declaration['tls']=='letsencrypt',services=declaration['services'],host_forward_groups=declaration.get('host_forward_groups',[]),acme_challenge=declaration.get('acme_challenge'),hostname_policy=validate_hostname_policy(declaration.get('hostname_policy'),declaration['domain']),http_policy=validate_http_policy(declaration.get('http_policy'),declaration['domain'],declaration.get('acme_challenge'))).encode() +PREVIOUS_STANDARD_SOURCE = '444d7c0e06e0f9935971c51872e9f37bd8869282' +PREVIOUS_STANDARD_TEMPLATES = {'previous-standard-nginx-444d7c0.conf.j2': 'f12e2b9800cbaa0e49e685506cfaad38eb7e568fa6d239fa15277af59073d8de', 'vast-price-manager-auth-blocks.conf.j2': 'b9d7013434557492887b0600bc2afc92227ee17f9efebe55b0cccc4b16bd3f81', 'acme-challenge.conf.j2': '7e594162700825757c9d402971094448aa1834d1f7121ac1c02f152f345e149a', 'fleet-api-routes.conf.j2': '6991ca6bf1dd4b258e8ee7851fc69ef50b19dfe2353570e4911a0ebadc68ed1b', 'vast-price-manager.conf.j2': '32483d169a09c42653396e66c5c7efe1995bd599f44f11ee64fa2f0731a0a2f8', 'standard-service-routes.conf.j2': 'ef0f55aae48e35127d288781e68281a371505ba27384c8fd04dff9200dce3963', 'exporter-auth-routes.conf.j2': '450f4c418c09474ac45b6dd314c47cc090fe7d230a0206b30e48c7975023133b', 'host-forwarding.conf.j2': '35c4a1fe1d3001865a77d7653350b7a61968953b94172dc7063bc31a42c0f9f7'} + + +def render_previous_standard(declaration): + env=get_jinja_env() + for name,expected in PREVIOUS_STANDARD_TEMPLATES.items(): + raw=env.loader.get_source(env,name)[0].encode() + if sha256(raw)!=expected: raise AdoptionError('previous_standard_profile_unavailable') + return env.get_template('previous-standard-nginx-444d7c0.conf.j2').render( + domain=declaration['domain'],letsencrypt=declaration['tls']=='letsencrypt',services=declaration['services'], + host_forward_groups=declaration.get('host_forward_groups',[]),acme_challenge=declaration.get('acme_challenge'), + hostname_policy=validate_hostname_policy(declaration.get('hostname_policy'),declaration['domain']), + http_policy=validate_http_policy(declaration.get('http_policy'),declaration['domain'],declaration.get('acme_challenge'))).encode() + + def _policy_healthcheck(source, declaration): try: return policy_healthcheck(source.get('Config',{}).get('Healthcheck'),declaration['domain'],declaration.get('hostname_policy'),declaration.get('http_policy'),letsencrypt=declaration['tls']=='letsencrypt') @@ -248,7 +303,7 @@ class AdoptionEngine(DockerEngine): def prove_recovery_auth(self,source,expected,identity): """Execute the source image's auth startup with no network or ports. - The restored store remains private until this actual old initializer + The stopped source store stays quiescent while this actual old initializer proves it preserves every file with the obsolete bootstrap disabled. """ from .migration import _mount_request @@ -581,6 +636,7 @@ def visit(items): visit(nodes); return nodes actual=normalize(tree); expected=normalize(parse_nginx(render_standard(declaration))) if actual==expected: return 'standard' + if actual==normalize(parse_nginx(render_previous_standard(declaration))): return 'standard-444d7c0' legacy=get_jinja_env().loader.get_source(get_jinja_env(),'legacy-bootstrap-nginx.conf')[0].encode() profiles=[legacy] for enabled in (False,True): profiles.append(render_managed_vpm_config(legacy,enabled)) @@ -618,7 +674,8 @@ def visit(items): context=_safe(' '.join(mismatch) if mismatch[0]=='location' else mismatch[0]) raise AdoptionError('unsupported_source_behavior: '+context+' requires a supported declarative product profile') - def plan(self,settings,*,environment=None): +# TRACEWEAVER: entrypoint=plan; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 + def plan(self,settings,*,environment=None,proposed_selection=None): declaration=_declaration(settings); source=self.engine.inspect('cryptolabs-proxy') if not source.get('State',{}).get('Running'): raise AdoptionError('source_not_running') healthcheck=_policy_healthcheck(source,declaration) @@ -642,7 +699,13 @@ def plan(self,settings,*,environment=None): if len(matching)!=1 or matching[0].get('Type')!='bind' or matching[0].get('RW') is not False: raise AdoptionError('overlay_mount_unproven') if sha256(self.engine.overlay_bytes(destination))!=overlay['sha256']: raise AdoptionError('overlay_source_changed') if overlay['replacement_sha256']!=sha256(Path(__file__).with_name('auth.py').read_bytes()): raise AdoptionError('overlay_replacement_module_unproven') - snapshot=load(declaration['release_selection_dir']); image=self.engine.image(source['Image']); platform=image.get('Os','')+'/'+image.get('Architecture','') + if proposed_selection is None: + snapshot=load(declaration['release_selection_dir']) + else: + if not isinstance(proposed_selection,dict): raise SelectionError('invalid_snapshot') + snapshot=validate_selection(proposed_selection['selection_bytes'].encode(), proposed_selection['manifest_bytes'].encode(), + selection_id=proposed_selection['selection_id'],selection_sha256=proposed_selection['selection_sha256']) + image=self.engine.image(source['Image']); platform=image.get('Os','')+'/'+image.get('Architecture','') selected=targets(snapshot,platform); artifact=selected.get('cryptolabs-proxy') if not artifact or artifact['owner']!='proxy-updater': raise AdoptionError('approved_proxy_required') candidate=render_standard(declaration) @@ -680,26 +743,68 @@ def adopt(self,settings,identity,*,environment=None): backup=self.backup_root/identity if backup.exists(): raise AdoptionError('private_recovery_snapshot_exists; inspect prior outcome before retry') backup.mkdir(parents=True,mode=0o700); self.backup_root.chmod(0o700); backup.chmod(0o700) - write=CustomConfigMigrator._write_private - write(backup/'inspect.json',_canonical(source)); write(backup/'source.nginx.conf',self.engine.read_config('cryptolabs-proxy')); _write_registry_backup(backup,previous) + import os as filesystem + for parent in (self.backup_root,self.config_dir): + descriptor=filesystem.open(parent,filesystem.O_RDONLY|filesystem.O_DIRECTORY) + try: filesystem.fsync(descriptor) + finally: filesystem.close(descriptor) + write=_durable_private + write(backup/'declaration.json',_canonical(declaration)); write(backup/'inspect.json',_canonical(source)); write(backup/'source.nginx.conf',self.engine.read_config('cryptolabs-proxy')); _write_registry_backup(backup,previous) candidate=backup/'candidate.nginx.conf'; write(candidate,render_standard(declaration)); write(backup/'plan.json',_canonical(plan)) + write(backup/'registry-snapshot.json',(backup/'registry-snapshot.json').read_bytes()) + journal={'schema_version':1,'plan_id':identity,'source_id':source['Id'], + 'source_image_id':source['Image'],'candidate_may_have_started':False} + def phase(value,**details): + journal.update(details); journal['phase']=value + write(backup/'phase.json',_canonical(journal)) + def mutation(action,name,invoke,**details): + try: before=engine.inspect(name) + except Exception: before=None + phase(action+'-requested',target=name,prior_id=before.get('Id') if before else None,**details) + result=invoke() + try: after=engine.inspect(name) + except Exception: after=None + if action=='create' and after and after.get('Image')==journal.get('candidate_image_id'): + journal['candidate_container_id']=after['Id'] + phase(action+'-done',target=name,observed_id=after.get('Id') if after else None,**details) + return result + engine=self.engine + phase('candidate-pull-requested') self.engine.pull(plan['image']); artifact=self.engine.image(plan['image']) + phase('candidate-pulled',candidate_image_id=artifact['Id']) if plan['image'] not in artifact.get('RepoDigests',[]) or artifact.get('Os','')+'/'+artifact.get('Architecture','')!=plan['platform']: raise AdoptionError('candidate_artifact_mismatch') if self.engine.artifact_file_sha256(plan['image'],AUTH_OVERLAY,identity)!=sha256(Path(__file__).with_name('auth.py').read_bytes()): raise AdoptionError('candidate_auth_module_mismatch') for overlay in declaration['obsolete_overlays']: if self.engine.artifact_file_sha256(plan['image'],overlay['destination'],identity)!=overlay['replacement_sha256']: raise AdoptionError('overlay_artifact_module_mismatch') write(backup/'retired-auth.py',self.engine.overlay_bytes(overlay['destination'])) + phase('candidate-validate-requested') self.engine.validate(plan['image'],candidate,source,identity) + phase('candidate-validated') # Recheck source and all policy/file identities after pull/test. if self.plan(settings,environment=environment)!=plan: raise AdoptionError('preflight_changed') + phase('updater-guard-requested') guard=exclusion.enter_context(self.engine.updater_guard(source,plan['image'],identity)) + try: held=self.engine.inspect('cryptolabs-adoption-lock-'+identity) + except Exception: held=None + phase('updater-held',guard_id=held.get('Id') if held else None) # Reject queued/running jobs and take an identity while the # dispatch and operation authorities are both excluded. - prestop_auth=auth_state(self.engine.archive_auth(plan['container_id'])) + prestop_archive=self.engine.archive_auth(plan['container_id']) + prestop_auth=auth_state(prestop_archive) + write(backup/'prestop-auth.tar',prestop_archive) + phase('source-ready-to-stop',prestop_auth=prestop_auth) if self.plan(settings,environment=environment)!=plan: raise AdoptionError('preflight_changed') clone=deepcopy(source); clone['Mounts']=[m for m in source['Mounts'] if m.get('Destination') not in ('/etc/nginx/nginx.conf','/etc/cryptolabs/releases') and m.get('Destination') not in {o['destination'] for o in declaration['obsolete_overlays']}] auth_mount=next(m for m in source['Mounts'] if m['Destination']=='/data/auth') clone['Mounts'].append({'Type':'bind','Source':declaration['release_selection_dir'],'Destination':'/etc/cryptolabs/releases','RW':False}) + if declaration.get('release_broker_enabled'): + socket_path='/run/dc-overview-release-broker.sock' + metadata=Path(socket_path).lstat() + import stat + if not stat.S_ISSOCK(metadata.st_mode) or metadata.st_uid!=0 or metadata.st_mode & 0o077: + raise AdoptionError('release_broker_owner_unavailable') + clone['Mounts']=[m for m in clone['Mounts'] if m.get('Destination')!=socket_path] + clone['Mounts'].append({'Type':'bind','Source':socket_path,'Destination':socket_path,'RW':False}) nginx=self.config_dir/'nginx.conf'; request=build_recreate_request(clone,plan['image'],str(nginx)) if declaration.get('hostname_policy') or declaration.get('http_policy'): request['Config']['Healthcheck']=_policy_healthcheck(source,declaration) @@ -707,6 +812,8 @@ def adopt(self,settings,identity,*,environment=None): config['Labels'].update({'com.centurylinklabs.watchtower.enable':'false','io.cryptolabs.update.origin':plan['origin'],'io.cryptolabs.update.reference':plan['image']}) for label in declaration['obsolete_owner_labels']: config['Labels'].pop(label,None) config['Env']=[e for e in config.get('Env',[]) if not e.startswith('RELEASE_SELECTION_REQUIRED=') and e.split('=',1)[0] not in (environment or {})]+['RELEASE_SELECTION_REQUIRED=true']+[k+'='+v for k,v in (environment or {}).items()] + if not any(e.startswith('FLEET_PUBLIC_ORIGIN=') for e in config['Env']): + config['Env'].append('FLEET_PUBLIC_ORIGIN=https://'+declaration['domain']) engine=self.engine; auth_copy=None; candidate_copy=None; candidate_started=False; recovered_id=None; recovery_expected=None; recovery_verified=False def restart_source(name): nonlocal recovered_id,recovery_expected,recovery_verified @@ -716,7 +823,9 @@ def restart_source(name): # Immutable source environment cannot be edited by Docker. # Prove its real startup, then recreate the original image # with only obsolete installation authority disabled. + phase('source-startup-proof-requested') engine.prove_recovery_auth(source,expected,identity) + phase('source-startup-proven') guard() clone=deepcopy(source) nginx_mount=next((m for m in clone['Mounts'] if m['Destination']=='/etc/nginx/nginx.conf'),None) @@ -727,17 +836,21 @@ def restart_source(name): from .migration import _mount_request payload['HostConfig']['Mounts'][-1]=_mount_request(nginx_mount) payload['Config']['Env']=[e for e in source['Config'].get('Env',[]) if not e.startswith('FLEET_ADMIN_PASS=')]+['FLEET_ADMIN_PASS='] - for network in engine.inspect(name).get('NetworkSettings',{}).get('Networks',{}): engine.disconnect(name,network) - engine.rename(name,'cryptolabs-proxy.original-'+identity) - engine.create(name,payload) + for network in engine.inspect(name).get('NetworkSettings',{}).get('Networks',{}): mutation('disconnect',name,lambda network=network:engine.disconnect(name,network)) + mutation('rename',name,lambda:engine.rename(name,'cryptolabs-proxy.original-'+identity)) + mutation('source-create',name,lambda:engine.create(name,payload)) recovered_id=engine.inspect(name)['Id'] - engine.start(name) + mutation('source-start',name,lambda:engine.start(name)) healthy=wait_for_container_healthy(engine,name,PROXY_READY_TIMEOUT) if auth_state(engine.archive_auth(name),allow_active_jobs=True)!=expected: - engine.stop(name) + mutation('stop',name,lambda:engine.stop(name)) raise AdoptionError('source_recovery_unverified; forward_repair_required') recovery_verified=healthy - if not healthy: engine.stop(name,ignore_missing=True) + if not healthy: mutation('stop',name,lambda:engine.stop(name,ignore_missing=True)) + if healthy: + evidence=_restoration_evidence(engine,source,plan,expected,self.config_dir) + write(backup/'restoration.json',_canonical(evidence)) + phase('source-restored',restoration_auth=expected,restoration_sha256=sha256(_canonical(evidence))) guard() class StateSafeEngine: def __getattr__(self,name): return getattr(engine,name) @@ -745,7 +858,7 @@ def stop(self,name,ignore_missing=False): nonlocal auth_copy,candidate_copy guard() stopping=engine.inspect(name) - engine.stop(name,ignore_missing=ignore_missing) + mutation('stop',name,lambda:engine.stop(name,ignore_missing=ignore_missing)) if stopping and stopping['Id'] not in (source['Id'],recovered_id) and candidate_started: candidate_copy=engine.archive_auth(name); write(backup/'candidate-auth.tar',candidate_copy) if name==plan['container_id'] and auth_copy is None: @@ -753,14 +866,22 @@ def stop(self,name,ignore_missing=False): auth_copy=engine.archive_auth(name) if auth_state(auth_copy)!=prestop_auth: raise AdoptionError('auth_store_changed_requires_quiescence') write(backup/'auth.tar',auth_copy) + phase('source-quiescent') write(nginx,candidate.read_bytes()) + phase('source-config-ready') except Exception: _restore_registry_files(previous) restart_source('cryptolabs-proxy') raise AdoptionError('auth_snapshot_failed; '+('sanitized_source_recovered' if recovery_verified else 'forward_repair_required')) from None def create(self,name,payload): guard() - engine.create(name,payload) + mutation('create',name,lambda:engine.create(name,payload)) + def rename(self,name,target): + guard(); mutation('rename',name,lambda:engine.rename(name,target),rename_to=target) + def disconnect(self,name,network): + guard(); mutation('disconnect',name,lambda:engine.disconnect(name,network)) + def remove(self,name,ignore_missing=False): + guard(); mutation('remove',name,lambda:engine.remove(name,ignore_missing=ignore_missing)) def connect(self,name,network,endpoint): guard() # The generic switch requests source reconnect before @@ -768,7 +889,7 @@ def connect(self,name,network,endpoint): # the sanitized source is recreated with its original # endpoints only after owned restore and startup proof. if engine.inspect(name)['Id'] in (source['Id'],recovered_id): return - engine.connect(name,network,endpoint) + mutation('connect',name,lambda:engine.connect(name,network,endpoint)) def start(self,name): nonlocal candidate_started guard() @@ -785,12 +906,14 @@ def start(self,name): else: if candidate_copy is not None: engine.restore_auth(name,candidate_copy,plan['image'],identity) write(nginx,candidate.read_bytes()); candidate_started=True - engine.start(name) + phase('candidate-start-requested',candidate_may_have_started=True) + mutation('start',name,lambda:engine.start(name)) guard() def candidate_health(name): guard() ready=wait_for_proxy_ready(engine,name,lambda:engine.anonymous_auth(name,declaration['tls'],declaration['domain']) if declaration.get('hostname_policy') or declaration.get('http_policy') else engine.anonymous_auth(name,declaration['tls']),PROXY_READY_TIMEOUT) - if not ready: engine.stop(name,ignore_missing=True) + if not ready: mutation('stop',name,lambda:engine.stop(name,ignore_missing=True)) + if ready: phase('candidate-verified',candidate_verified=True,serving_runtime_sha256=_runtime_identity(engine.inspect(name))) guard() return ready def rollback_health(name): @@ -809,6 +932,7 @@ def rollback_health(name): for name in ('hostname_policy','http_policy'): if name in declaration: registry.config[name]=declaration[name] else: registry.config.pop(name,None) + registry.config['release_broker_enabled']=declaration.get('release_broker_enabled',False) registry.config['standard_adoption']={'plan_id':identity,'serving_sha256':_runtime_identity(engine.inspect('cryptolabs-proxy')),'declaration_sha256':plan['declaration_sha256'],'selection_sha256':plan['selection_sha256'],'auth_mount':{k:auth_mount[k] for k in ('Type','Name','Source','Destination') if k in auth_mount},'declaration':declaration} if declaration.get('hostname_policy') or declaration.get('http_policy'): registry.config['standard_adoption']['healthcheck']=deepcopy(request['Config']['Healthcheck']) @@ -817,7 +941,9 @@ def rollback_health(name): write(backup/'outcome.json',_canonical({'state':'candidate_retained','reason':'registry_forward_repair_required'})) return {'state':'candidate_retained','reason':'registry_forward_repair_required','plan_id':identity} result={'state':'adopted' if outcome.state=='applied' else 'candidate_retained','plan_id':identity,'positive_login_verified':False} - try: write(backup/'outcome.json',_canonical(result)) + try: + write(backup/'outcome.json',_canonical(result)) + phase('completed') except OSError: return {'state':'candidate_retained','reason':'outcome_record_unavailable','plan_id':identity} return result except (SelectionError,MigrationError,OSError,ValueError,KeyError) as error: @@ -833,12 +959,174 @@ def rollback_health(name): except Exception: states[name]={'unknown':True} serving=states.get('cryptolabs-proxy',{}) restored=(serving.get('id')==locals().get('source',{}).get('Id') or serving.get('id')==locals().get('recovered_id') and locals().get('recovery_verified',False)) and serving.get('running') - try: CustomConfigMigrator._write_private(backup/'outcome.json',_canonical({'state':'rolled_back' if restored else 'forward_repair_required','runtime':states,'candidate_auth_snapshot_retained':(backup/'candidate-auth.tar').exists()})) + try: _durable_private(backup/'outcome.json',_canonical({'state':'rolled_back' if restored else 'forward_repair_required','runtime':states,'candidate_auth_snapshot_retained':(backup/'candidate-auth.tar').exists()})) except OSError: pass reason=str(error) if isinstance(error,AdoptionError) else 'adoption_failed; inspect private recovery outcome for actual state' raise AdoptionError(reason) from None +# TRACEWEAVER: entrypoint=recover_adoption; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 + def recover_adoption(self, plan_id): + """Reconcile a fixed journal and restore only proven pre-candidate phases.""" + if not isinstance(plan_id,str) or not re.fullmatch('[0-9a-f]{24}',plan_id): + raise AdoptionError('invalid_plan_id') + result={'state':'recovery-required','plan_id':plan_id} + if os.geteuid()!=0: raise AdoptionError('standard_adoption_requires_root') + from .cli import registry_lock + try: + with registry_lock(self.config_dir): + backup=self.backup_root/plan_id + def read(name): + path=backup/name + for parent in (self.backup_root,backup,path): + stat=parent.lstat() + if parent.is_symlink() or stat.st_uid!=os.getuid() or stat.st_mode & 0o077: + raise AdoptionError('unsafe_recovery_journal') + raw=path.read_bytes() + if len(raw)>MAX_CONFIG: raise AdoptionError('invalid_recovery_journal') + return json.loads(raw) + plan=read('plan.json'); declaration=read('declaration.json') + if plan.get('plan_id')!=plan_id or sha256(_canonical({k:v for k,v in plan.items() if k!='plan_id'}))[:24]!=plan_id: + return result + if sha256(_canonical(declaration))!=plan['declaration_sha256']: return result + try: outcome=read('outcome.json') + except FileNotFoundError: outcome={} + try: journal=read('phase.json') + except FileNotFoundError: journal={} + completed=outcome.get('state') in ('adopted','already_adopted') and self._already(declaration,plan_id) + source=read('inspect.json') + if _runtime_identity(source)!=plan['runtime_sha256']: return result + expected=journal.get('restoration_auth',journal.get('prestop_auth')) + if not isinstance(expected,dict): return result + serving=self.engine.inspect('cryptolabs-proxy') + restored=False + evidence=None + if journal.get('restoration_sha256'): + evidence=read('restoration.json') + observed=_restoration_evidence(self.engine,source,plan,expected,self.config_dir) + restored=(evidence==observed and journal['restoration_sha256']==sha256(_canonical(evidence))) + # Preserve exclusions on uncertain candidate/recovery phases. + restore_allowed=(journal.get('plan_id')==plan_id and journal.get('candidate_may_have_started') is False + and not journal.get('phase','').startswith('recovery-') + and serving and not serving.get('State',{}).get('Running') + and _runtime_identity(serving)==plan['runtime_sha256']) + if not completed and not restored and not restore_allowed: return result + # A killed owner can leave its fixed exclusion helper. The registry + # lock proves that owner is no longer executing this adoption. + helper_name='cryptolabs-adoption-lock-'+plan_id + try: helper=self.engine.inspect(helper_name) + except MigrationError as error: + if not str(error).endswith('HTTP 404'): return result + helper=None + if helper: + auth=next(m for m in source['Mounts'] if m['Destination']=='/data/auth') + mounts=[m for m in helper.get('Mounts',[]) if m.get('Destination')=='/data/auth'] + def mount_identity(m): + return (m.get('Type'),m.get('Name') if m.get('Type')=='volume' else m.get('Source'),m.get('Destination'),m.get('RW')) + command=helper.get('Config',{}).get('Cmd',[]) + if (not journal.get('guard_id') or helper.get('Id')!=journal['guard_id'] + or helper.get('Image')!=journal.get('candidate_image_id') + or helper.get('Config',{}).get('Labels',{}).get('io.cryptolabs.adoption.guard')!=plan_id + or helper.get('Config',{}).get('Entrypoint')!=['python3'] + or len(command)!=4 or command[0]!='-c' or command[-2:]!=['/data/auth/update-jobs','/tmp/updater-held'] + or len(mounts)!=1 or mount_identity(mounts[0])!=mount_identity(auth)): + return result + journal['phase']='recovery-guard-remove-requested' + _durable_private(backup/'phase.json',_canonical(journal)) + self.engine.remove(helper['Id']) + if completed: return {'state':'completed','plan_id':plan_id} + if restored: + with self.engine.updater_guard(source,plan['image'],plan_id) as guard: + guard() + if load(declaration['release_selection_dir'])['selection_sha256']!=plan['selection_sha256']: return result + observed=_restoration_evidence(self.engine,source,plan,expected,self.config_dir) + if evidence!=observed or journal.get('restoration_sha256')!=sha256(_canonical(evidence)): return result + guard() + return {'state':'failed-restored','plan_id':plan_id} + with self.engine.updater_guard(source,plan['image'],plan_id) as guard: + guard() + if load(declaration['release_selection_dir'])['selection_sha256']!=plan['selection_sha256']: return result + serving=self.engine.inspect('cryptolabs-proxy') + if serving.get('State',{}).get('Running') or _runtime_identity(serving)!=plan['runtime_sha256']: return result + archive=self.engine.archive_auth(source['Id']) + quiescent_auth=auth_state(archive) + prestop_auth_match=quiescent_auth==journal['prestop_auth'] + expected=quiescent_auth + # This is the missing quiescent post-stop capture. It is kept + # for evidence, never restored over the current shared store. + _durable_private(backup/'auth.tar',archive) + previous=read('registry-snapshot.json') + allowed={str(self.config_dir/name) for name in ('services.yaml','config.yaml','nginx.conf')} + if set(previous)!=allowed: return result + old={Path(path):value.encode('latin1') if value is not None else None for path,value in previous.items()} + saved_sha=sha256(_canonical({p.name:sha256(data) if data is not None else None for p,data in old.items()})) + if saved_sha!=plan['registry_sha256']: return result + current=_snapshot_registry_files(ServiceRegistry(self.config_dir)) + for path,data in current.items(): + if path.name=='nginx.conf': + if sha256(data or b'') not in (sha256(old[path] or b''),plan['candidate_sha256']): return result + elif data!=old[path]: return result + if sha256(self.engine.read_config('cryptolabs-proxy')) not in (plan['config_sha256'],plan['candidate_sha256']): return result + def phase(name,**details): + journal.update(details); journal['phase']=name + _durable_private(backup/'phase.json',_canonical(journal)) + def inspect_optional(name): + try: return self.engine.inspect(name) + except MigrationError as error: + if not str(error).endswith('HTTP 404'): raise + return None + def mutate(action,name,invoke,**details): + before=inspect_optional(name) + phase('recovery-'+action+'-requested',target=name,prior_id=before.get('Id') if before else None,**details) + invoke() + after=inspect_optional(name) + phase('recovery-'+action+'-done',target=name,observed_id=after.get('Id') if after else None,**details) + guard() + phase('recovery-startup-proof-requested',restoration_auth=expected,prestop_auth_match=prestop_auth_match) + self.engine.prove_recovery_auth(source,expected,plan_id) + phase('recovery-startup-proven',recovery_startup_proven=True) + if auth_state(self.engine.archive_auth(source['Id']))!=expected: return result + _restore_registry_files(old) + for path,data in old.items(): + if data is not None: _durable_private(path,data) + clone=deepcopy(source) + nginx_mount=next((m for m in clone['Mounts'] if m['Destination']=='/etc/nginx/nginx.conf'),None) + nginx_path=nginx_mount['Source'] if nginx_mount and nginx_mount.get('Type')=='bind' else str(backup/'source.nginx.conf') + clone['Mounts']=[m for m in clone['Mounts'] if m['Destination'] not in ('/etc/nginx/nginx.conf',AUTH_OVERLAY)] + payload=build_recreate_request(clone,source['Image'],nginx_path) + if nginx_mount: + from .migration import _mount_request + payload['HostConfig']['Mounts'][-1]=_mount_request(nginx_mount) + payload['Config']['Env']=[e for e in source['Config'].get('Env',[]) if not e.startswith('FLEET_ADMIN_PASS=')]+['FLEET_ADMIN_PASS='] + # Preserve the exact original name/image, with installation + # authority disabled and its real initializer already proven. + retained='cryptolabs-proxy.original-'+plan_id + try: prior=self.engine.inspect(retained) + except MigrationError as error: + if not str(error).endswith('HTTP 404'): return result + prior=None + if prior: return result + for network in serving.get('NetworkSettings',{}).get('Networks',{}): + mutate('disconnect',source['Id'],lambda network=network:self.engine.disconnect(source['Id'],network)) + mutate('rename',source['Id'],lambda:self.engine.rename(source['Id'],retained),rename_to=retained) + mutate('create','cryptolabs-proxy',lambda:self.engine.create('cryptolabs-proxy',payload)) + recovered=self.engine.inspect('cryptolabs-proxy') + phase('recovery-source-created',recovery_container_id=recovered['Id']) + mutate('start',recovered['Id'],lambda:self.engine.start('cryptolabs-proxy')) + healthy=wait_for_container_healthy(self.engine,'cryptolabs-proxy',PROXY_READY_TIMEOUT) + if not healthy or auth_state(self.engine.archive_auth('cryptolabs-proxy'),allow_active_jobs=True)!=expected: + mutate('stop',recovered['Id'],lambda:self.engine.stop('cryptolabs-proxy')) + return result + evidence=_restoration_evidence(self.engine,source,plan,expected,self.config_dir) + _durable_private(backup/'restoration.json',_canonical(evidence)) + phase('failed-restored',restoration_sha256=sha256(_canonical(evidence))) + _durable_private(backup/'outcome.json',_canonical({'state':'rolled_back','restoration_sha256':sha256(_canonical(evidence))})) + guard() + return {'state':'failed-restored','plan_id':plan_id} + except (OSError,ValueError,KeyError,MigrationError,TypeError,StopIteration): + pass + return result + def reconfigure(self,config): """Ordinary setup after adoption keeps inspected state and session key.""" try: diff --git a/src/cryptolabs_proxy/templates/docker-compose.yml.j2 b/src/cryptolabs_proxy/templates/docker-compose.yml.j2 index d15f997..46d8020 100644 --- a/src/cryptolabs_proxy/templates/docker-compose.yml.j2 +++ b/src/cryptolabs_proxy/templates/docker-compose.yml.j2 @@ -14,6 +14,9 @@ services: - ./ssl:/etc/nginx/ssl:ro - ./services.yaml:/app/services.yaml:ro - fleet-auth-data:/data/auth + {% if release_broker_enabled | default(false) %} + - /run/dc-overview-release-broker.sock:/run/dc-overview-release-broker.sock:ro + {% endif %} {% if release_selection_dir %} - {{ (release_selection_dir + ':/etc/cryptolabs/releases:ro') | tojson }} {% endif %} @@ -25,6 +28,9 @@ services: {% endif %} environment: - DOCKER_HOST=unix:///var/run/docker.sock + {% if domain %} + - {{ ('FLEET_PUBLIC_ORIGIN=https://' + domain) | tojson }} + {% endif %} {% if release_selection_dir %} - RELEASE_SELECTION_REQUIRED=true {% endif %} diff --git a/src/cryptolabs_proxy/templates/nginx.conf.j2 b/src/cryptolabs_proxy/templates/nginx.conf.j2 index 5a77d27..2624d7f 100644 --- a/src/cryptolabs_proxy/templates/nginx.conf.j2 +++ b/src/cryptolabs_proxy/templates/nginx.conf.j2 @@ -176,6 +176,43 @@ http { proxy_set_header Host $host; } + # Fleet release writes and status use live session authority in Flask. + location ^~ /api/releases/ { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + location = /api/update { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + location = /api/pull { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + location = /api/update-settings { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + location = /api/update-status { + client_max_body_size 64k; + proxy_pass http://127.0.0.1:8081; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 20s; + } + {% include "fleet-api-routes.conf.j2" %} # Static assets (no auth) diff --git a/src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2 b/src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2 new file mode 100644 index 0000000..5a77d27 --- /dev/null +++ b/src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2 @@ -0,0 +1,332 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + client_max_body_size 100M; + + # Gzip compression + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_types text/plain text/css text/xml text/javascript application/javascript application/json application/xml; + + # Rate limiting + limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s; + + {% if letsencrypt or http_policy %} + # Redirect HTTP to HTTPS + server { + listen 80{% if hostname_policy %} default_server{% endif %}; + server_name {{ domain }}; +{% if hostname_policy %} + if ($host != {{ domain }}) { + return 444; + } +{% endif %} + +{% if http_policy %} + location ^~ /.well-known/acme-challenge/ { + root /var/www/certbot; + default_type text/plain; + try_files $uri =404; + } + location / { + return 308 https://{{ domain }}$request_uri; + } + {% include "vast-price-manager-auth-blocks.conf.j2" %} +{% else %} + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +{% endif %} + } + + server { + listen 443 ssl http2{% if hostname_policy %} default_server{% endif %}; + server_name {{ domain }}; +{% if hostname_policy %} + if ($host != {{ domain }}) { + return 444; + } +{% endif %} + + {% if letsencrypt %} + ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256; + ssl_prefer_server_ciphers off; + ssl_session_cache shared:SSL:10m; + {% else %} + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + {% endif %} + {% else %} + server { + listen 80{% if hostname_policy %} default_server{% endif %}; + listen 443 ssl http2{% if hostname_policy %} default_server{% endif %}; + server_name {{ domain }}; +{% if hostname_policy %} + if ($host != {{ domain }}) { + return 444; + } +{% endif %} + + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + {% endif %} + + # DNS resolver for Docker networks - CRITICAL for runtime resolution + # This allows nginx to start even if backend services don't exist yet + resolver 127.0.0.11 valid=10s ipv6=off; + resolver_timeout 5s; + + # Security headers + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; + +{% include 'acme-challenge.conf.j2' %} + # ===================================================== + # ERROR HANDLERS + # ===================================================== + + location @service_unavailable { + default_type application/json; + return 503 '{"error":"Service unavailable","message":"The requested service is not running. Install it or wait for it to start."}'; + } + + # ===================================================== + # AUTHENTICATION + # ===================================================== + + # Auth server endpoints (public) + location /auth/ { + proxy_pass http://127.0.0.1:8081/auth/; +{% if http_policy %} + proxy_hide_header X-Mgmt-Token; + proxy_cookie_flags fleet_session secure httponly samesite=lax; +{% endif %} + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + +{% if http_policy %} + location = /auth/headers { return 404; } + location = /auth/headers/ { return 404; } + location = /auth/exporters/authorize { return 404; } + location = /auth/exporters/authorize/ { return 404; } +{% endif %} + # Internal auth verification endpoint (for auth_request) + location = /_auth_check { + internal; + proxy_pass http://127.0.0.1:8081/auth/headers; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header Cookie $http_cookie; + } + + # ===================================================== + # Internal API (blocked from public access) + # Only reachable via direct Docker network (172.30.0.10:8080) + # ===================================================== + location /internal/ { + return 403; + } + + # ===================================================== + # PUBLIC ENDPOINTS (no auth required) + # ===================================================== + + # Health check API (always public) + location /api/health { + default_type application/json; + return 200 '{"status":"ok","proxy":"running"}'; + } + + # Service health check (internal, no auth) + location /api/services { + proxy_pass http://127.0.0.1:8080/api/services; + proxy_set_header Host $host; + } + + {% include "fleet-api-routes.conf.j2" %} + + # Static assets (no auth) + location /static/ { + root /usr/share/nginx/html; + } + + # ===================================================== + # PROTECTED ENDPOINTS (auth required) + # ===================================================== + + # Fleet Management Landing Page (requires auth) + location = / { + auth_request /_auth_check; + auth_request_set $auth_user $upstream_http_x_fleet_auth_user; + auth_request_set $auth_role $upstream_http_x_fleet_auth_role; + auth_request_set $auth_token $upstream_http_x_fleet_auth_token; + + # Redirect to login if not authenticated + error_page 401 = @login_redirect; + + root /usr/share/nginx/html; + index index.html; + } + + location @login_redirect { + return 302 /auth/login?next=$request_uri; + } + + {% include "vast-price-manager-auth-blocks.conf.j2" %} + + {% if 'vast-price-manager' in services %} + {% include "vast-price-manager.conf.j2" %} + {% endif %} + + {% include "standard-service-routes.conf.j2" %} + + {% for name, service in services.items() %} + {% if name in ['ipmi-monitor','dc-overview','grafana','prometheus','vastai-exporter','runpod-exporter','vast-price-manager'] %} + # Vast Price Manager uses its dedicated admin-only route above. + {% elif not service.path %} + # {{ service.get('display_name', name) }} - external service, no local proxy route + {% else %} + # {{ service.get('display_name', name) }} - uses variable for runtime DNS resolution + location ~ ^{{ service.path }}(?.*)$ { + # Require authentication + auth_request /_auth_check; + auth_request_set $auth_user $upstream_http_x_fleet_auth_user; + auth_request_set $auth_role $upstream_http_x_fleet_auth_role; + auth_request_set $auth_token $upstream_http_x_fleet_auth_token; + + # Redirect to login if not authenticated + error_page 401 = @login_redirect; + + {% if name == 'grafana' or service.container_name == 'grafana' %} + # Map Fleet roles to Grafana roles (Admin, Editor, Viewer) + set $grafana_role "Viewer"; + if ($auth_role = "admin") { + set $grafana_role "Admin"; + } + if ($auth_role = "readwrite") { + set $grafana_role "Editor"; + } + {% endif %} + + # Variable allows nginx to start even if service doesn't exist yet + set $upstream_{{ name | replace('-', '_') }} http://{{ service.container_name }}:{{ service.port }}; + proxy_pass $upstream_{{ name | replace('-', '_') }}/$custom_{{ name | replace('-', '_') }}$is_args$args; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Script-Name {{ service.path.rstrip('/') }}; + + # Forward authentication headers to backend services + proxy_set_header X-Fleet-Auth-User $auth_user; + proxy_set_header X-Fleet-Auth-Role $auth_role; + proxy_set_header X-Fleet-Auth-Token $auth_token; + proxy_set_header X-Fleet-Authenticated "true"; + + {% if name == 'grafana' or service.container_name == 'grafana' %} + # Grafana-specific: pass user and role for auto-login with correct permissions + proxy_set_header X-WEBAUTH-USER $auth_user; + proxy_set_header X-WEBAUTH-ROLE $grafana_role; + {% endif %} + + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_send_timeout 300; + + {% if name == 'grafana' or service.container_name == 'grafana' %} + # Inject Fleet Management Home button into Grafana HTML + # target="_top" prevents Grafana's SPA router from intercepting the click + sub_filter '' ''; + sub_filter '' '
Home|Grafana
'; + sub_filter_once on; + sub_filter_types text/html; + proxy_set_header Accept-Encoding ""; + {% elif name == 'prometheus' or service.container_name == 'prometheus' %} + # Inject Fleet Management Home button into Prometheus HTML + sub_filter '' ''; + sub_filter '' '
Home|Prometheus
'; + sub_filter_once on; + sub_filter_types text/html; + proxy_set_header Accept-Encoding ""; + {% endif %} + + # Graceful error handling when service is unavailable + proxy_intercept_errors on; + error_page 502 503 504 = @service_unavailable; + } + + {% endif %} + {% endfor %} + + # Exporter management pages - served by auth server + location /vastai/ { + auth_request /_auth_check; + auth_request_set $auth_user $upstream_http_x_fleet_auth_user; + auth_request_set $auth_role $upstream_http_x_fleet_auth_role; + error_page 401 = @login_redirect; + proxy_pass http://127.0.0.1:8081/vastai/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /runpod/ { + auth_request /_auth_check; + auth_request_set $auth_user $upstream_http_x_fleet_auth_user; + auth_request_set $auth_role $upstream_http_x_fleet_auth_role; + error_page 401 = @login_redirect; + proxy_pass http://127.0.0.1:8081/runpod/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + {% include "exporter-auth-routes.conf.j2" %} + + # Default: show landing page + location / { + root /usr/share/nginx/html; + try_files $uri $uri/ /index.html; + } + } + {% include "host-forwarding.conf.j2" %} + +} diff --git a/src/cryptolabs_proxy/updates.py b/src/cryptolabs_proxy/updates.py index afae181..8eb6dec 100644 --- a/src/cryptolabs_proxy/updates.py +++ b/src/cryptolabs_proxy/updates.py @@ -1,3 +1,4 @@ +# TRACEWEAVER: file-role=fleet-release-implementation; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 """Serialized fleet image updates executed outside the serving proxy container. Job files contain only operation state, never container environments or Docker @@ -58,17 +59,25 @@ def get_capabilities(): """Additive caller contract; proxy replacement requires host authority.""" return {'selection_contract': selection.SELECTION_CONTRACT, + 'supported_selection_contracts': [1, 2], + 'release_components': 1, 'release_idempotency': 1, + 'proxy_selection_preview': 1, 'proxy_adoption_recovery': 1, 'proxy_replacement_policy': PROXY_REPLACEMENT_POLICY, 'controller_adoption_guard': dict(controller_recovery.CAPABILITY)} -def _require_selected_updater(engine, source): +def _require_selected_updater(engine, source, contract=1, *, scoped=False): """The selected helper executes source-image code, not the host package.""" try: capabilities = engine.updater_capabilities(source['Id']) if not source.get('State', {}).get('Running') or not isinstance(capabilities, dict): raise ValueError() - if type(capabilities.get('selection_contract')) is not int or capabilities['selection_contract'] != 1: + if type(capabilities.get('selection_contract')) is not int or capabilities['selection_contract'] != 1: raise ValueError() + supported = capabilities.get('supported_selection_contracts', [capabilities.get('selection_contract')]) + if (not isinstance(supported, list) or any(type(v) is not int for v in supported) + or contract not in supported): + raise ValueError() + if scoped and any(type(capabilities.get(k)) is not int or capabilities.get(k) != 1 for k in ('release_components','release_idempotency')): raise ValueError() if type(capabilities.get('proxy_replacement_policy')) is not str or capabilities['proxy_replacement_policy'] != PROXY_REPLACEMENT_POLICY: raise ValueError() @@ -156,13 +165,13 @@ def _platform(image): return image.get('Os', '') + '/' + image.get('Architecture', '') -def _selection_plan(snapshot, engine): +def _selection_plan(snapshot, engine, components=None): """Every enabled proxy role must exist and be running before queuing work.""" enabled = snapshot['selection']['components'] sources = {} platform = None for name, entry in enabled.items(): - if entry['enabled'] and snapshot['manifest']['components'][name]['owner'] == 'proxy-updater': + if entry['enabled'] and (components is None or name in components) and snapshot['manifest']['components'][name]['owner'] == 'proxy-updater': if name not in SERVICES or SERVICES[name].get('update_supported') is False: raise UpdateError('unsupported_proxy_role') try: @@ -180,6 +189,11 @@ def _selection_plan(snapshot, engine): platform = _platform(engine.image(engine.inspect('cryptolabs-proxy')['Image'])) try: targets = selection.targets(snapshot, platform) + if components is not None: + if set(components) - set(targets): raise UpdateError('service_not_selected') + targets = {name: targets[name] for name in components} + if any(target['owner'] != 'proxy-updater' for target in targets.values()): + raise UpdateError('external_owner_required') except selection.SelectionError as error: raise UpdateError(str(error)) from None return targets, sources @@ -450,6 +464,9 @@ def _write_json(path, value): stream.flush() os.fsync(stream.fileno()) os.replace(temporary, path) + descriptor = os.open(path.parent, os.O_RDONLY) + try: os.fsync(descriptor) + finally: os.close(descriptor) def write_job(directory, job): @@ -464,24 +481,47 @@ def read_job(directory, job_id): def job_status(job_id, *, directory=JOBS_DIR, engine=None): - job = read_job(directory, job_id) - if job.get('state') in ACTIVE_STATES: - engine = engine or UpdateEngine() + try: return _status_snapshot(job_id,directory=directory,engine=engine) + except controller_recovery.RecoveryError: + raise UpdateError('controller_recovery_required') from None + + +def _status_snapshot(job_id, *, directory, engine): + # Dispatch owns queued -> created -> started. A concurrent reader cannot + # interpret that launch window as an exited worker, including nested replay. + with controller_recovery.open_directory(directory) as directory_fd: + descriptor = controller_recovery.open_lock(directory_fd, 'dispatch.lock') + acquired = False try: - running = engine.inspect(job['helper'])['State']['Running'] - except MigrationError as error: - if not str(error).endswith('HTTP 404'): + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + except BlockingIOError: + pass + if not job_id: + try: job_id = json.loads((Path(directory) / 'active.json').read_text())['id'] + except (OSError, ValueError, KeyError, TypeError): raise UpdateError('Update job not found.') from None + job = read_job(directory, job_id) + if not acquired or job.get('state') not in ACTIVE_STATES: return job - running = False - except Exception: + engine = engine or UpdateEngine() + try: + running = engine.inspect(job['helper'])['State']['Running'] + except MigrationError as error: + if not str(error).endswith('HTTP 404'): return job + running = False + except Exception: + return job + if not running: + job = read_job(directory, job_id) + if job.get('state') in ACTIVE_STATES: + job.update(state='interrupted', success=False, + message='Update helper stopped before completion. Inspect services before retrying.') + write_job(directory, job) return job - if not running: - job = read_job(directory, job_id) - if job.get('state') in ACTIVE_STATES: - job.update(state='interrupted', success=False, - message='Update helper stopped before completion. Inspect services before retrying.') - write_job(directory, job) - return job + finally: + if acquired: fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) @contextmanager @@ -494,15 +534,55 @@ def _lock(directory, name): def helper_request(source, directory, job_id): - """The helper uses the current immutable proxy image, outside its PID space.""" - parent = next((mount for mount in sorted(source.get('Mounts', []), key=lambda item: len(item['Destination']), reverse=True) - if str(directory).startswith(mount['Destination'].rstrip('/') + '/')), None) - if not parent or not parent.get('RW', True): - raise UpdateError('Updater needs a writable persistent /data/auth mount.') + """Map trusted host/container journal paths onto one inspected backing mount.""" + directory = Path(directory) + if not directory.is_absolute() or '..' in directory.parts: + raise UpdateError('invalid_worker_store') + matches = [] + for parent in source.get('Mounts', []): + if parent.get('Type') not in ('bind','volume') or parent.get('RW') is not True: + continue + destination = parent.get('Destination') + backing = parent.get('Source') + if (not isinstance(destination,str) or not Path(destination).is_absolute() + or '..' in Path(destination).parts or destination == '/' + or backing is not None and (not isinstance(backing,str) or not Path(backing).is_absolute() or '..' in Path(backing).parts) + or parent['Type']=='bind' and (not isinstance(backing,str) or not Path(backing).is_absolute() or '..' in Path(backing).parts) + or parent['Type']=='volume' and (not isinstance(parent.get('Name'),str) or not parent['Name'] + or not re.fullmatch('[A-Za-z0-9][A-Za-z0-9_.-]*',parent['Name']))): + continue + prefixes=[Path(destination)] + if isinstance(backing,str) and Path(backing).is_absolute() and '..' not in Path(backing).parts: + prefixes.append(Path(backing)) + candidates = [] + for prefix in prefixes: + try: relative = directory.relative_to(prefix) + except ValueError: continue + if not relative.parts: continue + candidates.append(Path(destination)/relative) + for worker_directory in set(candidates): + matches.append((parent,worker_directory)) + if len(matches)!=1: + raise UpdateError('Updater needs one unambiguous writable persistent auth mount.') + parent,worker_directory = matches[0] + # Nested mount shadowing would send worker state to another backing store. + for other in source.get('Mounts', []): + if other is parent: continue + destination=other.get('Destination') + if isinstance(destination,str): + other_destination=Path(destination) + selected_destination=Path(parent['Destination']) + # Dockerfile VOLUME /data coexists with the more-specific auth + # mount. Only mounts within the chosen mount can shadow its store. + if other_destination in selected_destination.parents: continue + if (other_destination==selected_destination or selected_destination in other_destination.parents): + if (other_destination==worker_directory or other_destination in worker_directory.parents + or worker_directory in other_destination.parents): + raise UpdateError('ambiguous_worker_store') mount = _mount_request(parent) return { 'Config': {'Image': source['Image'], 'Entrypoint': ['python3'], - 'Cmd': ['-m', 'cryptolabs_proxy.updates', 'worker', job_id, str(directory)], + 'Cmd': ['-m', 'cryptolabs_proxy.updates', 'worker', job_id, str(worker_directory)], 'Env': ['PYTHONPATH=/app/src'], 'Healthcheck': {'Test': ['NONE']}, 'Labels': {'io.cryptolabs.update.worker': job_id, 'com.centurylinklabs.watchtower.enable': 'false'}}, 'HostConfig': {'NetworkMode': 'host', 'RestartPolicy': {'Name': 'no'}, @@ -512,79 +592,155 @@ def helper_request(source, directory, job_id): } +def _mapped_job(mapping, directory, engine=None): + try: + job = read_job(directory, mapping['job_id']) + except UpdateError: + job = dict(mapping['initial_job']) + job.update(state='recovery-required', success=False, message='Dispatch journal requires reconciliation.') + write_job(directory, job) + if job.get('operation_id') != mapping['operation_id'] or job.get('request_fingerprint') != mapping['request_fingerprint']: + raise UpdateError('operation_conflict') + # A replay never creates/starts another helper, including after a lost reply. + if job.get('state') in ACTIVE_STATES: + job = job_status(job['id'], directory=directory, engine=engine) + return job + + +def operation_status(operation_id, *, directory=JOBS_DIR, engine=None): + try: + if str(uuid.UUID(operation_id)) != operation_id: raise ValueError() + mapping = json.loads((Path(directory) / 'operations' / (operation_id + '.json')).read_text()) + except (ValueError, OSError, TypeError): raise UpdateError('operation_not_found') from None + return _mapped_job(mapping, directory, engine) + + +# TRACEWEAVER: entrypoint=submit_job; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 def submit_job(service, branch, action, *, directory=JOBS_DIR, engine=None, - selection_id=None, selection_sha256=None, selection_directory=None): + selection_id=None, selection_sha256=None, selection_directory=None, + components=None, operation_id=None, request_fingerprint=None): if branch not in ('main', 'dev') or action not in ('pull', 'update'): raise UpdateError('Invalid update channel or action.') if service != 'all' and (service not in SERVICES or (action == 'update' and SERVICES[service].get('update_supported') is False)): raise UpdateError('Service is not managed by this updater.') + scoped = components is not None + if scoped or operation_id is not None or request_fingerprint is not None: + if (not isinstance(components, list) or not 1 <= len(components) <= 32 + or any(type(n) is not str or n not in SERVICES or n == 'cryptolabs-proxy' or SERVICES[n].get('update_supported') is False for n in components) + or len(set(components)) != len(components) or service not in components + or not isinstance(operation_id, str) or not isinstance(request_fingerprint, str) + or not re.fullmatch('[0-9a-f]{64}', request_fingerprint)): + raise UpdateError('invalid_release_operation') + try: + if str(uuid.UUID(operation_id)) != operation_id: raise ValueError() + except ValueError: raise UpdateError('invalid_release_operation') from None + if selection_id is None or selection_sha256 is None: raise UpdateError('selection_pair_required') + request_identity = {'service': service, 'branch': branch, 'action': action, + 'components': components, 'selection_id': selection_id, 'selection_sha256': selection_sha256} + with _lock(directory, 'dispatch.lock'): + if scoped: + mapping_path = Path(directory) / 'operations' / (operation_id + '.json') + if mapping_path.exists(): + mapping = json.loads(mapping_path.read_text()) + if mapping.get('request_fingerprint') != request_fingerprint or mapping.get('request') != request_identity: + raise UpdateError('operation_conflict') + return _mapped_job(mapping, directory, engine) + return _submit_job_locked(service, branch, action, directory=directory, engine=engine, + selection_id=selection_id, selection_sha256=selection_sha256, selection_directory=selection_directory, + components=components, operation_id=operation_id, request_fingerprint=request_fingerprint, request_identity=request_identity) + + +def _submit_job_locked(service, branch, action, *, directory, engine, selection_id, selection_sha256, + selection_directory, components, operation_id, request_fingerprint, request_identity): engine = engine or UpdateEngine() snapshot = installed_selection(selection_directory, selection_id=selection_id, selection_sha256=selection_sha256) - if snapshot and branch == 'dev': - raise UpdateError('selection_channel_conflict') - targets, sources = _selection_plan(snapshot, engine) if snapshot else ({}, {}) + if components is not None and snapshot is None: raise UpdateError('selection_required') + targets, sources = _selection_plan(snapshot, engine, components) if snapshot else ({}, {}) + if snapshot: + names = components if components is not None else ([service] if service != 'all' else list(targets)) + channels = {snapshot['manifest'].get('component_channels', {}).get(n, 'main') for n in names} + if channels != {branch}: raise UpdateError('selection_channel_conflict') if snapshot and service != 'all' and (service not in targets or targets[service]['owner'] != 'proxy-updater'): raise UpdateError('service_not_selected') - with _lock(directory, 'dispatch.lock'): - try: - controller_recovery.require_clear(directory) - except controller_recovery.RecoveryError: - raise UpdateError('controller_recovery_required') from None - active = Path(directory) / 'active.json' - if active.exists(): - previous = read_job(directory, json.loads(active.read_text())['id']) - if previous.get('state') in ACTIVE_STATES: - try: - running = engine.inspect(previous['helper'])['State']['Running'] - except MigrationError as error: - if not str(error).endswith('HTTP 404'): - # An inaccessible Engine is not proof that a worker stopped. - raise UpdateBusy(previous) from None - running = False - except Exception: + try: + controller_recovery.require_clear(directory) + except controller_recovery.RecoveryError: + raise UpdateError('controller_recovery_required') from None + active = Path(directory) / 'active.json' + if active.exists(): + previous = read_job(directory, json.loads(active.read_text())['id']) + if previous.get('state') == 'recovery-required': raise UpdateBusy(previous) + if previous.get('state') in ACTIVE_STATES: + try: + running = engine.inspect(previous['helper'])['State']['Running'] + except MigrationError as error: + if not str(error).endswith('HTTP 404'): # An inaccessible Engine is not proof that a worker stopped. raise UpdateBusy(previous) from None - if running: - raise UpdateBusy(previous) - previous.update(state='interrupted', success=False, - message='Update helper exited before recording completion. Inspect service status before retrying.') - write_job(directory, previous) - source = engine.inspect('cryptolabs-proxy') - if action == 'update' and service in ('all', 'cryptolabs-proxy'): - proxy_target = targets.get('cryptolabs-proxy') if snapshot else None - if snapshot and proxy_target and proxy_target['owner'] == 'proxy-updater': - try: noop = not _proxy_replacement_required(source, engine.image(proxy_target['reference']), proxy_target) - except Exception: noop = False - if not noop: raise UpdateError(PROXY_ADOPTION_MESSAGE) - elif not snapshot and source.get('State', {}).get('Running') and target_for('cryptolabs-proxy',source,branch): - # Floating channels cannot prove a future old helper's pull is a - # no-op. A host dispatcher must not launch unpatched source code. - raise UpdateError(PROXY_ADOPTION_MESSAGE) - if snapshot: - _require_selected_updater(engine, source) - job_id = uuid.uuid4().hex - helper = 'cryptolabs-update-' + job_id - request = helper_request(source, Path(directory), job_id) - if action == 'pull': - _invalidate_checks(directory, SERVICES if service == 'all' else [service], branch) - job = {'id': job_id, 'helper': helper, 'service': service, 'branch': branch, - 'action': action, 'state': 'queued', 'results': {}, 'created_at': time.time(), - 'scope': 'proxy-updater', 'suite_complete': False, 'external_owners': []} - if snapshot: - job.update(selection_id=snapshot['selection_id'], selection_sha256=snapshot['selection_sha256'], - manifest_sha256=snapshot['manifest_sha256'], selection_snapshot=snapshot, targets=targets, - sources=sources, scope='proxy-updater', suite_complete=False, - external_owners=sorted({t['owner'] for t in targets.values() if t['owner'] != 'proxy-updater'})) + running = False + except Exception: + # An inaccessible Engine is not proof that a worker stopped. + raise UpdateBusy(previous) from None + if running: + raise UpdateBusy(previous) + previous.update(state='interrupted', success=False, + message='Update helper exited before recording completion. Inspect service status before retrying.') + write_job(directory, previous) + source = engine.inspect('cryptolabs-proxy') + if action == 'update' and service in ('all', 'cryptolabs-proxy'): + proxy_target = targets.get('cryptolabs-proxy') if snapshot else None + if snapshot and proxy_target and proxy_target['owner'] == 'proxy-updater': + try: noop = not _proxy_replacement_required(source, engine.image(proxy_target['reference']), proxy_target) + except Exception: noop = False + if not noop: raise UpdateError(PROXY_ADOPTION_MESSAGE) + elif not snapshot and source.get('State', {}).get('Running') and target_for('cryptolabs-proxy',source,branch): + # Floating channels cannot prove a future old helper's pull is a + # no-op. A host dispatcher must not launch unpatched source code. + raise UpdateError(PROXY_ADOPTION_MESSAGE) + if snapshot: + _require_selected_updater(engine, source, snapshot['manifest']['selection_contract'], scoped=components is not None) + if snapshot: + installed_selection(selection_directory,selection_id=snapshot['selection_id'],selection_sha256=snapshot['selection_sha256']) + job_id = uuid.uuid4().hex + helper = 'cryptolabs-update-' + job_id + request = helper_request(source, Path(directory), job_id) + if action == 'pull': + _invalidate_checks(directory, SERVICES if service == 'all' else [service], branch) + job = {'id': job_id, 'helper': helper, 'service': service, 'branch': branch, + 'action': action, 'state': 'queued', 'results': {}, 'created_at': time.time(), + 'scope': 'proxy-updater', 'suite_complete': False, 'external_owners': []} + if snapshot: + job.update(selection_id=snapshot['selection_id'], selection_sha256=snapshot['selection_sha256'], + manifest_sha256=snapshot['manifest_sha256'], selection_snapshot=snapshot, targets=targets, + sources=sources, scope='proxy-updater', suite_complete=False, + external_owners=sorted({t['owner'] for t in targets.values() if t['owner'] != 'proxy-updater'})) + if components is not None: + job.update(components=list(components), operation_id=operation_id, request_fingerprint=request_fingerprint) + _write_json(Path(directory) / 'operations' / (operation_id + '.json'), + {'operation_id': operation_id, 'request_fingerprint': request_fingerprint, + 'request': request_identity, 'job_id': job_id, 'initial_job': job}) + write_job(directory, job) + _write_json(active, {'id': job_id}) + try: + engine.create(helper, request) + engine.start(helper) + except Exception: + # A worker can finish before the Engine start reply is lost. Its + # durable outcome wins; dispatch must never overwrite it with a guess. + recorded = read_job(directory, job_id) + if recorded.get('state') != 'queued': return recorded + if components is not None: + try: + helper_state = engine.inspect(helper) + if helper_state.get('State', {}).get('Running'): + return recorded + except Exception: + pass + job.update(state='recovery-required' if components is not None else 'failed', success=False, + message='Helper launch outcome requires reconciliation.' if components is not None else 'Could not launch update helper.') write_job(directory, job) - _write_json(active, {'id': job_id}) - try: - engine.create(helper, request) - engine.start(helper) - except Exception: - job.update(state='failed', success=False, message='Could not launch update helper.') - write_job(directory, job) - raise UpdateError(job['message']) from None - return job + if components is None: raise UpdateError(job['message']) from None + return read_job(directory,job_id) def pull_target(target): @@ -605,11 +761,11 @@ def _run_selected_job(directory, job, engine): raise UpdateError('selection_changed') if selection.sha256(engine.manifest_bytes(job['manifest_sha256'])) != job['manifest_sha256']: raise UpdateError('manifest_changed') - targets, sources = _selection_plan(snapshot, engine) + targets, sources = _selection_plan(snapshot, engine, job.get('components')) if targets != job['targets'] or sources != job['sources']: raise UpdateError('runtime_changed') names = [name for name, target in targets.items() if target['owner'] == 'proxy-updater' - and job['service'] in ('all', name)] + and (name in job['components'] if 'components' in job else job['service'] in ('all', name))] names.sort(key=lambda name: name == 'cryptolabs-proxy') staged = {} for name in names: @@ -639,7 +795,7 @@ def _run_selected_job(directory, job, engine): pass # Optional channel discovery cannot block a verified selected artifact. # A policy edit or container replacement during pulls invalidates the # complete plan before the first stop, rather than allowing partial work. - if selection.sha256(engine.selection_bytes()) != job['selection_sha256'] or selection.sha256(engine.manifest_bytes(job['manifest_sha256'])) != job['manifest_sha256'] or _selection_plan(snapshot, engine)[1] != sources: + if selection.sha256(engine.selection_bytes()) != job['selection_sha256'] or selection.sha256(engine.manifest_bytes(job['manifest_sha256'])) != job['manifest_sha256'] or _selection_plan(snapshot, engine, job.get('components'))[1] != sources: raise UpdateError('selection_or_runtime_changed') for name, check in checked.items(): _write_json(_check_path(directory, name, job['branch']), check) @@ -787,17 +943,23 @@ def main(argv=None): submit.add_argument('--action', choices=('update', 'pull'), default='update') submit.add_argument('--selection-id') submit.add_argument('--selection-sha256') + submit.add_argument('--component',dest='components',action='append') + submit.add_argument('--operation-id') + submit.add_argument('--request-fingerprint') commands.add_parser('capabilities') status = commands.add_parser('status') - status.add_argument('job_id') + status.add_argument('job_id',nargs='?',default='') worker = commands.add_parser('worker') worker.add_argument('job_id') worker.add_argument('directory', type=Path) args = parser.parse_args(argv) try: if args.command == 'submit': + operation = {} + if args.components is not None or args.operation_id is not None or args.request_fingerprint is not None: + operation = dict(components=args.components,operation_id=args.operation_id,request_fingerprint=args.request_fingerprint) result = submit_job(args.service, args.branch, args.action, - selection_id=args.selection_id, selection_sha256=args.selection_sha256) + selection_id=args.selection_id, selection_sha256=args.selection_sha256,**operation) elif args.command == 'capabilities': result = get_capabilities() elif args.command == 'status': diff --git a/tests/test_auth.py b/tests/test_auth.py index 648416c..dea5590 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,3 +1,5 @@ +# TRACEWEAVER: file-role=verification; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 +# TRACEWEAVER: verifies=VER-FLEETUPDATE-001; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001 """Tests for token management, user CRUD, permissions, and account lockout.""" import time @@ -529,3 +531,162 @@ def test_mismatched_username(self): headers = auth.generate_proxy_headers('alice', 'admin') headers[auth.AUTH_HEADER_USER] = 'eve' assert auth.verify_proxy_auth(headers) is None + + +@pytest.mark.parametrize('change', ['demote','disable','delete','password-change']) +def test_release_live_authority_and_purpose_csrf(app, logged_in_admin, admin_user, monkeypatch, change): + import cryptolabs_proxy.auth as auth + from cryptolabs_proxy.release_client import ReleaseBrokerClient + app.config['RELEASE_PUBLIC_ORIGIN'] = 'http://localhost' + calls = [] + monkeypatch.setattr(ReleaseBrokerClient, 'request', lambda self, action, params, actor: calls.append((action,params,actor)) or {'candidates':[]}) + response = logged_in_admin.get('/api/releases/session') + assert response.status_code == 200 + token = response.json['csrf_token'] + body = {'operation_id':'12345678-1234-4234-8234-123456789abc','preview_id':'fixture','preview_sha256':'a'*64,'expected_selection_id':'fixture','expected_selection_sha256':'b'*64} + assert logged_in_admin.post('/api/releases/apply',json=body,headers={'Origin':'http://evil.test','X-CSRF-Token':token}).status_code == 403 + assert logged_in_admin.post('/api/releases/apply',json=body,headers={'Origin':'http://localhost','X-CSRF-Token':'bad'}).status_code == 403 + assert calls == [] + assert logged_in_admin.post('/api/releases/apply',json=body,headers={'Origin':'http://localhost','X-CSRF-Token':token}).status_code == 200 + assert calls[-1][2]['username'] == admin_user['username'] + if change == 'demote': auth.update_user(admin_user['username'], role='readonly') + elif change == 'disable': auth.update_user(admin_user['username'], enabled=False) + elif change == 'delete': + users=auth.load_users(); users.pop(admin_user['username']); auth.save_users(users) + else: + users=auth.load_users(); users[admin_user['username']]['require_password_change']=True; auth.save_users(users) + before=len(calls) + assert logged_in_admin.get('/api/releases/candidates?channel=main').status_code in (401,403) + assert logged_in_admin.post('/api/update',json={'service':'prometheus'},headers={'Origin':'http://localhost','X-CSRF-Token':token}).status_code in (401,403) + assert len(calls) == before + + +@pytest.mark.parametrize('reply,expected', [ + (b'{"schema_version":1,"ok":true,"result":{"candidates":[]}}\n', True), + (b'{"schema_version":1,"ok":true,"ok":false,"result":{}}\n', False), +]) +def test_release_client_real_unix_stream_framing(tmp_path,monkeypatch,reply,expected): + import socket + import threading + from types import SimpleNamespace + from cryptolabs_proxy import release_client + import tempfile + directory=tempfile.TemporaryDirectory(prefix='frc-',dir='/private/tmp') + path=directory.name+'/owner.sock' + server=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) + server.bind(path);server.listen(1) + received=[] + def serve(): + with server: + connection,_=server.accept() + with connection: + raw=b'' + while True: + chunk=connection.recv(65536) + if not chunk:break + raw+=chunk + received.append(raw) + connection.sendall(reply) + thread=threading.Thread(target=serve,daemon=True);thread.start() + metadata=__import__('pathlib').Path(path).lstat() + # Only root ownership is represented here; real socket I/O remains native. + monkeypatch.setattr(release_client,'SOCKET_PATH',path) + monkeypatch.setattr(release_client,'Path',lambda _:SimpleNamespace(lstat=lambda:SimpleNamespace(st_mode=(metadata.st_mode&~0o777)|0o600,st_uid=0))) + actor={'kind':'fleet-admin','username':'fixture','subject':'a'*64} + if expected: + assert release_client.ReleaseBrokerClient().request('list',{'channel':'main'},actor=actor)=={'candidates':[]} + else: + with pytest.raises(release_client.ReleaseBrokerError,match='Release owner unavailable'): + release_client.ReleaseBrokerClient().request('list',{'channel':'main'},actor=actor) + thread.join(timeout=2) + assert not thread.is_alive() + assert received and received[0].endswith(b'\n') + assert json.loads(received[0])=={'schema_version':1,'action':'list','params':{'channel':'main'},'actor':actor} + directory.cleanup() + + +def test_release_stream_json_requires_utf8(): + from cryptolabs_proxy.release_client import strict_json + with pytest.raises(ValueError): + strict_json('{"schema_version":1}'.encode('utf-16')) + + +@pytest.mark.parametrize('path,body',[('/api/update',{'service':'prometheus'}),('/api/pull',{'service':'prometheus'}),('/api/update-settings',{'branch':'main'})]) +def test_selected_legacy_mutations_require_admission_owner(app,logged_in_admin,monkeypatch,path,body): + from cryptolabs_proxy import updates + app.config['RELEASE_PUBLIC_ORIGIN']='http://localhost' + token=logged_in_admin.get('/api/releases/session').json['csrf_token'] + monkeypatch.setattr(updates,'installed_selection',lambda:{'selection_id':'fixture','selection_sha256':'a'*64}) + calls=[] + monkeypatch.setattr(updates,'submit_job',lambda *args,**kwargs:calls.append(args) or {'id':'job'}) + response=logged_in_admin.post(path,json=body,headers={'Origin':'http://localhost','X-CSRF-Token':token}) + assert response.status_code==409 + assert response.json['code']=='release_owner_required' + assert calls==[] + + +def test_release_authority_survives_real_permanent_cookie_refresh_and_replacement(app,admin_user,monkeypatch): + from itsdangerous import TimestampSigner + from cryptolabs_proxy import auth + from cryptolabs_proxy.release_client import ReleaseBrokerClient + timestamp=[int(time.time())] + monkeypatch.setattr(TimestampSigner,'get_timestamp',lambda self:timestamp[0]) + app.config['RELEASE_PUBLIC_ORIGIN']='http://localhost' + actors=[] + monkeypatch.setattr(ReleaseBrokerClient,'request',lambda self,action,params,actor:actors.append(actor) or {}) + client=app.test_client();client.post('/auth/login',data=admin_user) + token=client.get('/api/releases/session').json['csrf_token'] + assert client.get('/api/releases/candidates?channel=main').status_code==200 + actor=actors[-1] + assert token!=actor['subject'] + cookie_name=app.config['SESSION_COOKIE_NAME'];before=client.get_cookie(cookie_name).value + timestamp[0]+=2 + response=client.get('/api/releases/session') + refreshed=client.get_cookie(cookie_name).value + assert refreshed!=before and response.status_code==200 + body={'operation_id':'12345678-1234-4234-8234-123456789abc','preview_id':'fixture','preview_sha256':'a'*64,'expected_selection_id':'fixture','expected_selection_sha256':'b'*64} + assert client.post('/api/releases/apply',json=body,headers={'Origin':'http://localhost','X-CSRF-Token':token}).status_code==200 + assert response.json['csrf_token']==token and actors[-1]==actor + timestamp[0]+=2 + assert client.get('/api/releases/candidates?channel=main').status_code==200 + assert actors[-1]==actor and client.get_cookie(cookie_name).value!=refreshed + replacement=auth.create_flask_auth_app();replacement.config.update(TESTING=True,SESSION_COOKIE_SECURE=False,RELEASE_PUBLIC_ORIGIN='http://localhost') + other= replacement.test_client();other.set_cookie(cookie_name,client.get_cookie(cookie_name).value) + assert other.post('/api/releases/apply',json=body,headers={'Origin':'http://localhost','X-CSRF-Token':token}).status_code==200 + assert actors[-1]==actor + + +def test_release_authority_isolates_logins_and_rebinds_logout_and_principal(app,admin_user,monkeypatch): + from cryptolabs_proxy import auth + from cryptolabs_proxy.release_client import ReleaseBrokerClient + fixed=time.time();monkeypatch.setattr(time,'time',lambda:fixed) + app.config['RELEASE_PUBLIC_ORIGIN']='http://localhost' + actors=[] + monkeypatch.setattr(ReleaseBrokerClient,'request',lambda self,action,params,actor:actors.append(actor) or {}) + first=app.test_client();second=app.test_client() + for client in (first,second):client.post('/auth/login',data=admin_user) + body={'operation_id':'12345678-1234-4234-8234-123456789abc','preview_id':'fixture','preview_sha256':'a'*64,'expected_selection_id':'fixture','expected_selection_sha256':'b'*64} + def apply(client,token):return client.post('/api/releases/apply',json=body,headers={'Origin':'http://localhost','X-CSRF-Token':token}) + assert apply(first,'unissued').status_code==403 and actors==[] + one=first.get('/api/releases/session').json['csrf_token'];two=second.get('/api/releases/session').json['csrf_token'] + assert one!=two + assert apply(first,two).status_code==403 and actors==[] + assert apply(first,one).status_code==200;actor_one=actors[-1] + assert apply(second,two).status_code==200 and actors[-1]['subject']!=actor_one['subject'] + first.post('/auth/login',data=admin_user) + assert apply(first,one).status_code==403 + relogin=first.get('/api/releases/session').json['csrf_token'];assert relogin!=one + assert apply(first,one).status_code==403 and apply(first,relogin).status_code==200 + assert actors[-1]['subject']!=actor_one['subject'] + one=relogin + first.get('/auth/logout');assert apply(first,one).status_code==401 + first.post('/auth/login',data=admin_user) + assert apply(first,one).status_code==403 + again=first.get('/api/releases/session').json['csrf_token'];assert again!=one + assert apply(first,one).status_code==403 and apply(first,again).status_code==200 + auth.create_user('second-admin','different-pass',role='admin') + first.post('/auth/login',data={'username':'second-admin','password':'different-pass'}) + assert apply(first,again).status_code==403 + rebound=first.get('/api/releases/session').json['csrf_token'] + assert apply(first,rebound).status_code==200 and actors[-1]['username']=='second-admin' + assert actors[-1]['subject']!=actor_one['subject'] diff --git a/tests/test_proxy_customer_route_repairs.py b/tests/test_proxy_customer_route_repairs.py index 935af1f..2d8eb06 100644 --- a/tests/test_proxy_customer_route_repairs.py +++ b/tests/test_proxy_customer_route_repairs.py @@ -1,3 +1,5 @@ +# TRACEWEAVER: file-role=verification; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 +# TRACEWEAVER: verifies=VER-FLEETUPDATE-001; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001 """Existing-customer setup/adoption route regressions; Docker is a boundary.""" import json from pathlib import Path @@ -167,7 +169,9 @@ def test_new_updater_nonproxy_replacement_still_supported(): def test_additive_capabilities_getter_matches_cli_contract(capsys): from cryptolabs_proxy import updates capabilities=updates.get_capabilities() - assert capabilities=={'selection_contract':1,'proxy_replacement_policy':'host-adoption-required-v1', + assert capabilities=={'selection_contract':1,'supported_selection_contracts':[1,2], + 'release_components':1,'release_idempotency':1,'proxy_selection_preview':1,'proxy_adoption_recovery':1, + 'proxy_replacement_policy':'host-adoption-required-v1', 'controller_adoption_guard':{'schema_version':1,'api':'AdoptionEngine.controller_guard', 'durable_updater_exclusion':True}} assert type(capabilities['controller_adoption_guard']['schema_version']) is int diff --git a/tests/test_release_selection.py b/tests/test_release_selection.py index d61c34c..ea9781d 100644 --- a/tests/test_release_selection.py +++ b/tests/test_release_selection.py @@ -1,3 +1,5 @@ +# TRACEWEAVER: file-role=verification; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 +# TRACEWEAVER: verifies=VER-FLEETUPDATE-001; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001 """Synthetic contract1 selection/update authority regressions.""" import hashlib import importlib.util @@ -133,17 +135,15 @@ def test_configured_missing_selection_cannot_fall_back_to_channel(tmp_path,monke assert engine.calls==[] -def test_api_passes_selection_pair_and_keeps_202_envelope(monkeypatch): - path=Path(__file__).parents[1]/'scripts/health-api.py' - spec=importlib.util.spec_from_file_location('health_selection',path); api=importlib.util.module_from_spec(spec); spec.loader.exec_module(api) +def test_selected_api_rejects_direct_dispatch_even_with_current_selection_pair(monkeypatch, app, logged_in_admin): + app.config['RELEASE_PUBLIC_ORIGIN']='http://localhost' + token=logged_in_admin.get('/api/releases/session').json['csrf_token'] data={'service':'all','selection_id':'synthetic-selection','selection_sha256':'a'*64} - body=json.dumps(data).encode(); handler=object.__new__(api.HealthHandler) - handler.path='/api/update'; handler.headers={'Content-Length':str(len(body))}; handler.rfile=io.BytesIO(body) - response=[]; handler.send_json=lambda value,status=200: response.append((value,status)) - submit=Mock(return_value={'id':'job'}); monkeypatch.setattr(api,'submit_update_job',submit) - handler.do_POST() - assert submit.call_args.kwargs=={'selection_id':data['selection_id'],'selection_sha256':data['selection_sha256']} - assert response==[({'success':True,'job':{'id':'job'}},202)] + monkeypatch.setattr(updates,'installed_selection',lambda:{k:v for k,v in data.items() if k!='service'}) + submit=Mock(return_value={'id':'job'});monkeypatch.setattr(updates,'submit_job',submit) + response=logged_in_admin.post('/api/update',json=data,headers={'Origin':'http://localhost','X-CSRF-Token':token}) + submit.assert_not_called() + assert response.status_code==409 and response.json['code']=='release_owner_required' def test_setup_config_adds_supported_selection_directory(): @@ -251,7 +251,7 @@ def test_missing_required_service_and_explicit_dev_reject_before_queue(tmp_path) updates.submit_job('all','dev' if failure=='dev' else 'main','update',directory=tmp_path/'jobs',engine=engine, selection_id='synthetic-selection',selection_sha256=digest,selection_directory=directory) assert engine.calls==[] - assert not (tmp_path/'jobs').exists() + assert not list((tmp_path/'jobs').glob('*.json')), 'invalid selection must never queue a job' def test_customer_pin_exact_match_and_external_owner_outcomes(tmp_path,monkeypatch): @@ -307,7 +307,7 @@ def test_api_cannot_upload_policy_paths_or_images(monkeypatch): response=[]; handler.send_json=lambda value,status=200:response.append((value,status)) submit=Mock(); monkeypatch.setattr(api,'submit_update_job',submit) handler.do_POST(); submit.assert_not_called() - assert response[0][1]==400 + assert response[0][1]==403 def test_external_owner_is_reported_and_never_executed_by_proxy(tmp_path,monkeypatch): diff --git a/tests/test_standard_adoption.py b/tests/test_standard_adoption.py index d0410ef..966d3f4 100644 --- a/tests/test_standard_adoption.py +++ b/tests/test_standard_adoption.py @@ -1,3 +1,5 @@ +# TRACEWEAVER: file-role=verification; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 +# TRACEWEAVER: verifies=VER-FLEETUPDATE-001; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001 """Synthetic standard adoption: production render and compensated switch path.""" import hashlib import json @@ -474,3 +476,216 @@ def test_selected_image_must_have_normal_auth_module_even_without_overlay(tmp_pa monkeypatch.setattr(engine,'artifact_file_sha256',lambda *args:'0'*64,raising=False) with pytest.raises(AdoptionError): adoption.adopt(settings,plan['plan_id']) assert not [c for c in engine.calls if c[0]=='stop'] + + +def test_proposed_selection_plan_is_pure_and_revalidates_exact_bytes(tmp_path, monkeypatch): + from cryptolabs_proxy import release_selection + adoption, engine, settings, directory = system(tmp_path, monkeypatch) + releases = Path(settings['release_selection_dir']) + before = {str(p):p.read_bytes() for p in releases.rglob('*') if p.is_file()} + old = release_selection.load(releases) + manifest = json.loads(old['manifest_bytes']) + manifest['components']['cryptolabs-proxy']['artifacts']['linux/amd64']['reference'] = REF.replace('a'*64, 'b'*64) + manifest_raw = json.dumps(manifest).encode() + selected = json.loads(old['selection_bytes']) + selected['manifest_sha256'] = hashlib.sha256(manifest_raw).hexdigest() + proposed = release_selection.validate(json.dumps(selected).encode(), manifest_raw) + preview = adoption.plan(settings, proposed_selection=proposed) + assert preview['image'].endswith('b'*64) + assert preview['selection_sha256'] == proposed['selection_sha256'] + assert {str(p):p.read_bytes() for p in releases.rglob('*') if p.is_file()} == before + assert engine.calls == [] + proposed['selection_sha256'] = '0'*64 + with pytest.raises(ValueError): adoption.plan(settings, proposed_selection=proposed) + + +def test_recover_adoption_reports_retained_success_only_with_live_identity(tmp_path, monkeypatch): + adoption, engine, settings, directory = system(tmp_path, monkeypatch) + plan = adoption.plan(settings) + result = adoption.adopt(settings, plan['plan_id']) + assert result['state'] == 'adopted' + import cryptolabs_proxy.standard_adoption as owner + monkeypatch.setattr(owner.os, 'getuid', __import__('os').getuid, raising=False) + assert adoption.recover_adoption(plan['plan_id'])['state'] == 'completed' + engine.current['Config']['Env'].append('UNRELATED=drift') + # Installed serving object, not a stale source pointer. + engine.containers['cryptolabs-proxy']['Config']['Env'].append('DRIFT=true') + assert adoption.recover_adoption(plan['plan_id'])['state'] == 'recovery-required' + + +def test_previous_standard_profile_is_exact_and_adoptable(tmp_path,monkeypatch): + import subprocess + from cryptolabs_proxy.config import get_jinja_env + from cryptolabs_proxy.standard_adoption import AdoptionError + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + raw=(Path(__file__).parents[1]/'src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2').read_bytes() + assert hashlib.sha256(raw).hexdigest()=='f12e2b9800cbaa0e49e685506cfaad38eb7e568fa6d239fa15277af59073d8de' + template=raw.decode() + old=get_jinja_env().from_string(template).render(domain=settings['domain'],letsencrypt=False,services=settings['services'],host_forward_groups=[],acme_challenge=None,hostname_policy=None,http_policy=None).encode() + engine.config=old + plan=adoption.plan(settings) + assert plan['source_profile']=='standard-444d7c0' + engine.config=old.replace(b'keepalive_timeout 65;',b'keepalive_timeout 66;') + with pytest.raises(AdoptionError):adoption.plan(settings) + engine.config=old + assert adoption.adopt(settings,plan['plan_id'])['state']=='adopted' + + +@pytest.mark.parametrize('enabled',[True,False,'true']) +def test_real_compose_renderer_preserves_validated_broker_flag(tmp_path,enabled): + from cryptolabs_proxy.config import generate_docker_compose + registry=ServiceRegistry(tmp_path);registry.config={'domain':'fleet.example.test','release_broker_enabled':enabled};registry.save() + if type(enabled) is not bool: + with pytest.raises(ValueError):generate_docker_compose(tmp_path) + assert not (tmp_path/'docker-compose.yml').exists() + else: + generate_docker_compose(tmp_path) + assert ('/run/dc-overview-release-broker.sock:' in (tmp_path/'docker-compose.yml').read_text()) is enabled + + +def test_fixed_recovery_restores_stopped_source_before_auth_archive_gap(tmp_path,monkeypatch): + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + import cryptolabs_proxy.standard_adoption as owner + monkeypatch.setattr(owner.os,'getuid',__import__('os').getuid,raising=False) + plan=adoption.plan(settings);stop=engine.stop + def die(name,ignore_missing=False): + stop(name,ignore_missing) + raise KeyboardInterrupt('owner died immediately after stop') + engine.stop=die + with pytest.raises(KeyboardInterrupt):adoption.adopt(settings,plan['plan_id']) + assert not engine.inspect('cryptolabs-proxy')['State']['Running'] + engine.stop=stop + recovered=adoption.recover_adoption(plan['plan_id']) + assert recovered['state']=='failed-restored' + serving=engine.inspect('cryptolabs-proxy') + assert serving['State']['Running'] and serving['Image']=='sha256:old' + assert 'FLEET_ADMIN_PASS=' in serving['Config']['Env'] + assert engine.auth==auth_tar({'users.json':b'synthetic-auth-archive'}) + assert adoption.recover_adoption(plan['plan_id'])['state']=='failed-restored' + + +def test_recovery_recognizes_actual_sanitized_rollback_identity(tmp_path,monkeypatch): + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + import cryptolabs_proxy.standard_adoption as owner + monkeypatch.setattr(owner.os,'getuid',__import__('os').getuid,raising=False) + plan=adoption.plan(settings);engine.fail_health=True + with pytest.raises(owner.AdoptionError):adoption.adopt(settings,plan['plan_id']) + assert engine.inspect('cryptolabs-proxy')['Id']!=plan['container_id'] + assert adoption.recover_adoption(plan['plan_id'])['state']=='failed-restored' + engine.auth=auth_tar({'users.json':b'changed-authority'}) + assert adoption.recover_adoption(plan['plan_id'])['state']=='recovery-required' + + +def test_fixed_recovery_never_replays_candidate_start_or_overwrites_new_auth(tmp_path,monkeypatch): + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + import cryptolabs_proxy.standard_adoption as owner + monkeypatch.setattr(owner.os,'getuid',__import__('os').getuid,raising=False) + plan=adoption.plan(settings);start=engine.start + def die(name): + start(name) + if engine.inspect(name)['Image']=='sha256:new': + engine.auth=auth_tar({'users.json':b'new-authority'}) + raise KeyboardInterrupt('owner died after candidate start') + engine.start=die + with pytest.raises(KeyboardInterrupt):adoption.adopt(settings,plan['plan_id']) + archive=engine.auth;calls=list(engine.calls) + assert adoption.recover_adoption(plan['plan_id'])['state']=='recovery-required' + assert engine.auth==archive and engine.calls==calls + + +@pytest.mark.parametrize("startup_preserves_auth",[True,False]) +def test_fixed_recovery_preserves_latest_quiescent_auth_after_stop(tmp_path,monkeypatch,startup_preserves_auth): + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + import cryptolabs_proxy.standard_adoption as owner + monkeypatch.setattr(owner.os,'getuid',__import__('os').getuid,raising=False) + plan=adoption.plan(settings);stop=engine.stop + def die(name,ignore_missing=False): + stop(name,ignore_missing) + raise KeyboardInterrupt('after stop') + engine.stop=die + with pytest.raises(KeyboardInterrupt):adoption.adopt(settings,plan['plan_id']) + engine.stop=stop;engine.auth=auth_tar({'users.json':b'concurrent-authority'}) + latest=engine.auth + if not startup_preserves_auth: + def reject(*args): + raise owner.AdoptionError('source_auth_startup_unverified') + engine.prove_recovery_auth=reject + assert adoption.recover_adoption(plan['plan_id'])['state']==('failed-restored' if startup_preserves_auth else 'recovery-required') + assert engine.inspect('cryptolabs-proxy')['State']['Running'] is startup_preserves_auth + assert engine.auth==latest + assert not [call for call in engine.calls if call[0]=='restore-auth'] + if startup_preserves_auth: + assert adoption.recover_adoption(plan['plan_id'])['state']=='failed-restored' + phase=json.loads((adoption.backup_root/plan['plan_id']/'phase.json').read_text()) + assert phase['prestop_auth_match'] is False and phase['restoration_auth']!=phase['prestop_auth'] + + +def test_verified_restoration_receipt_survives_missing_final_outcome(tmp_path,monkeypatch): + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + import cryptolabs_proxy.standard_adoption as owner + monkeypatch.setattr(owner.os,'getuid',__import__('os').getuid,raising=False) + plan=adoption.plan(settings);engine.fail_health=True + with pytest.raises(owner.AdoptionError):adoption.adopt(settings,plan['plan_id']) + (adoption.backup_root/plan['plan_id']/'outcome.json').unlink() + assert adoption.recover_adoption(plan['plan_id'])['state']=='failed-restored' + + +@pytest.mark.parametrize('owned',[True,False]) +def test_finished_owner_reclaims_only_exact_orphan_guard(tmp_path,monkeypatch,owned): + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + import cryptolabs_proxy.standard_adoption as owner + monkeypatch.setattr(owner.os,'getuid',__import__('os').getuid,raising=False) + plan=adoption.plan(settings);adoption.adopt(settings,plan['plan_id']) + path=adoption.backup_root/plan['plan_id']/'phase.json';journal=json.loads(path.read_text()) + journal['guard_id']='guard-id';path.write_text(json.dumps(journal)) + name='cryptolabs-adoption-lock-'+plan['plan_id'] + auth=next(m for m in engine.current['Mounts'] if m['Destination']=='/data/auth') + helper={'Id':'guard-id','Image':journal['candidate_image_id'],'Config':{ + 'Entrypoint':['python3'],'Cmd':['-c','fixed-lock-script','/data/auth/update-jobs','/tmp/updater-held'], + 'Labels':{'io.cryptolabs.adoption.guard':plan['plan_id'] if owned else 'another-plan'}},'Mounts':[deepcopy(auth)]} + engine.containers[name]=helper;engine.by_id['guard-id']=helper + removed=[];remove=engine.remove + def remove_id(identifier,ignore_missing=False): + removed.append(identifier) + found=next((n for n,v in engine.containers.items() if v.get('Id')==identifier),identifier) + remove(found,ignore_missing) + engine.remove=remove_id + assert adoption.recover_adoption(plan['plan_id'])['state']==('completed' if owned else 'recovery-required') + assert removed==(['guard-id'] if owned else []) + + +def test_fixed_recovery_preserves_external_registry_change_after_owner_death(tmp_path,monkeypatch): + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + import cryptolabs_proxy.standard_adoption as owner + monkeypatch.setattr(owner.os,'getuid',__import__('os').getuid,raising=False) + plan=adoption.plan(settings);stop=engine.stop + def die(name,ignore_missing=False): + stop(name,ignore_missing);raise KeyboardInterrupt('after stop') + engine.stop=die + with pytest.raises(KeyboardInterrupt):adoption.adopt(settings,plan['plan_id']) + engine.stop=stop + path=directory/'services.yaml';changed=path.read_bytes()+b'\n# external owner change\n';path.write_bytes(changed) + assert adoption.recover_adoption(plan['plan_id'])['state']=='recovery-required' + assert path.read_bytes()==changed and not engine.inspect('cryptolabs-proxy')['State']['Running'] + assert not [call for call in engine.calls if call[0]=='prove-recovery-auth'] + + +def test_inprocess_latest_auth_restoration_reconciles_repeat_without_archive_restore(tmp_path,monkeypatch): + import cryptolabs_proxy.standard_adoption as owner + adoption,engine,settings,directory=system(tmp_path,monkeypatch) + monkeypatch.setattr(owner.os,'getuid',__import__('os').getuid,raising=False) + plan=adoption.plan(settings);stop=engine.stop + latest=auth_tar({'users.json':b'latest-authority','sessions.json':b'latest-session'}) + def changed_stop(name,ignore_missing=False): + stop(name,ignore_missing) + if name==plan['container_id']:engine.auth=latest + engine.stop=changed_stop + with pytest.raises(owner.AdoptionError):adoption.adopt(settings,plan['plan_id']) + assert engine.inspect('cryptolabs-proxy')['State']['Running'] + assert engine.auth==latest + for _ in range(2): + assert adoption.recover_adoption(plan['plan_id'])['state']=='failed-restored' + assert engine.auth==latest + assert not [call for call in engine.calls if call[0]=='restore-auth'] + journal=json.loads((adoption.backup_root/plan['plan_id']/'phase.json').read_text()) + assert journal['restoration_auth']!=journal['prestop_auth'] diff --git a/tests/test_updates.py b/tests/test_updates.py index b49cfb3..233c329 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -1,3 +1,5 @@ +# TRACEWEAVER: file-role=verification; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001; ver=VER-FLEETUPDATE-001 +# TRACEWEAVER: verifies=VER-FLEETUPDATE-001; req=REQ-FLEETUPDATE-001; trace=TRACE-FLEETUPDATE-001 """Updater failures must not destroy existing runtime state or conceal outcomes.""" import importlib.util import json @@ -193,22 +195,15 @@ def test_helper_runs_independently_with_persistent_state_and_no_copied_secrets() assert request['Config']['Healthcheck'] == {'Test': ['NONE']} -def test_update_api_queues_job_instead_of_mutating_docker(monkeypatch): - path = Path(__file__).parents[1] / 'scripts' / 'health-api.py' - spec = importlib.util.spec_from_file_location('health_api_updates', path) - api = importlib.util.module_from_spec(spec) - spec.loader.exec_module(api) - import io - body = json.dumps({'service': 'prometheus', 'branch': 'main'}).encode() - handler = object.__new__(api.HealthHandler) - handler.path = '/api/update' - handler.headers = {'Content-Length': str(len(body))} - handler.rfile = io.BytesIO(body) - replies = [] - handler.send_json = lambda result, status=200: replies.append((result, status)) - monkeypatch.setattr(api, 'submit_update_job', lambda *args, **kwargs: {'id': 'job', 'state': 'queued'}) - handler.do_POST() - assert replies == [({'success': True, 'job': {'id': 'job', 'state': 'queued'}}, 202)] +def test_update_api_queues_job_instead_of_mutating_docker(monkeypatch, app, logged_in_admin): + app.config['RELEASE_PUBLIC_ORIGIN']='http://localhost' + token=logged_in_admin.get('/api/releases/session').json['csrf_token'] + monkeypatch.setattr(updates,'installed_selection',lambda:None) + monkeypatch.setattr(updates,'submit_job',lambda *args,**kwargs:{'id':'job','state':'queued'}) + response=logged_in_admin.post('/api/update',json={'service':'prometheus','branch':'main'}, + headers={'Origin':'http://localhost','X-CSRF-Token':token}) + assert response.status_code==202 + assert response.json=={'success':True,'job':{'id':'job','state':'queued'}} def test_cli_submit_and_status_return_machine_readable_jobs(monkeypatch, capsys): @@ -356,3 +351,149 @@ def test_landing_renders_partial_failure_as_error(): result = json.loads(run.stdout) assert result['kind'] == 'error' assert 'Original restored' in result['message'] + + +def test_release_operation_replays_terminal_job_and_rejects_changed_fingerprint(tmp_path): + from tests.test_release_selection import selection_files, SelectedEngine + directory, digest = selection_files(tmp_path) + engine = SelectedEngine(tmp_path) + engine.updater_capabilities = lambda source_id: updates.get_capabilities() + operation = '12345678-1234-4234-8234-123456789abc' + options = dict(directory=tmp_path/'jobs', engine=engine, selection_id='synthetic-selection', + selection_sha256=digest, selection_directory=directory, + components=['prometheus'], operation_id=operation, request_fingerprint='a'*64) + job = updates.submit_job('prometheus', 'main', 'update', **options) + assert job['components'] == ['prometheus'] and set(job['targets']) == {'prometheus'} + job.update(state='completed', success=True) + updates.write_job(tmp_path/'jobs', job) + engine.calls.clear() + assert updates.submit_job('prometheus', 'main', 'update', **options) == job + assert engine.calls == [] + options['request_fingerprint'] = 'b'*64 + with pytest.raises(updates.UpdateError, match='operation_conflict'): + updates.submit_job('prometheus', 'main', 'update', **options) + + +def test_release_operation_lost_create_reply_keeps_same_helper_and_recovery_hold(tmp_path): + from tests.test_release_selection import selection_files, SelectedEngine + directory, digest = selection_files(tmp_path) + engine = SelectedEngine(tmp_path) + engine.updater_capabilities = lambda source_id: updates.get_capabilities() + def uncertain_create(name, request): + engine.calls.append(('create', name)) + raise OSError('reply lost after helper creation') + engine.create = uncertain_create + operation = '12345678-1234-4234-8234-123456789abc' + options = dict(directory=tmp_path/'jobs', engine=engine, selection_id='synthetic-selection', + selection_sha256=digest, selection_directory=directory, + components=['prometheus'], operation_id=operation, request_fingerprint='a'*64) + first = updates.submit_job('prometheus', 'main', 'update', **options) + second = updates.submit_job('prometheus', 'main', 'update', **options) + assert first['id'] == second['id'] + assert second['state'] == 'recovery-required' + assert len([call for call in engine.calls if call[0] == 'create']) == 1 + + +def test_landing_reload_ignores_stale_operation_response(): + script = landing_script() + ''' + let observed = []; + showUpdateStatus = (kind,message) => observed.push({kind,message}); + loadVersions = () => {}; loadServices = () => {}; + globalThis.localStorage = {setItem:()=>{},removeItem:()=>{}}; + globalThis.fetch = async () => ({ok:true,json:async()=>({id:'old',state:'completed',success:true})}); + activeUpdateJob = 'new'; + pollUpdateJob('old',0,'old').then(()=>console.log(JSON.stringify({active:activeUpdateJob,observed}))); + ''' + run = subprocess.run(['node'],input=script,capture_output=True,text=True) + assert run.returncode == 0,run.stderr + result=json.loads(run.stdout) + assert result == {'active':'new','observed':[]} + + +def test_lost_helper_start_reply_does_not_overwrite_worker_terminal_result(tmp_path): + from tests.test_release_selection import selection_files, SelectedEngine + directory, digest = selection_files(tmp_path) + engine = SelectedEngine(tmp_path) + engine.updater_capabilities = lambda source_id: updates.get_capabilities() + def completed_before_lost_reply(name): + active=json.loads((tmp_path/'jobs/active.json').read_text()) + job=updates.read_job(tmp_path/'jobs',active['id']) + job.update(state='completed',success=True) + updates.write_job(tmp_path/'jobs',job) + raise OSError('start response lost after worker completed') + engine.start=completed_before_lost_reply + result=updates.submit_job('prometheus','main','update',directory=tmp_path/'jobs',engine=engine, + selection_id='synthetic-selection',selection_sha256=digest,selection_directory=directory, + components=['prometheus'],operation_id='12345678-1234-4234-8234-123456789abc',request_fingerprint='a'*64) + assert result['state']=='completed' and result['success'] is True + + +def test_explicit_component_group_preserves_unscoped_missing_service(tmp_path,monkeypatch): + from tests.test_release_selection import selection_files, SelectedEngine, REF + directory,_=selection_files(tmp_path) + manifest=json.loads(next((directory/'manifests').iterdir()).read_text()) + for name,repository in [('grafana','grafana/grafana'),('ipmi-monitor','ghcr.io/cryptolabsza/ipmi-monitor')]: + manifest['components'][name]={'owner':'proxy-updater','release':'synthetic-1','artifacts':{'linux/amd64':{'kind':'oci','reference':repository+'@sha256:'+'a'*64}}} + raw=json.dumps(manifest).encode();digest=__import__('hashlib').sha256(raw).hexdigest() + (directory/'manifests'/(digest+'.json')).write_bytes(raw) + selection=json.loads((directory/'selection.json').read_text());selection['manifest_sha256']=digest + for name in ['grafana','ipmi-monitor']:selection['components'][name]={'enabled':True,'origin':'baseline'} + raw=json.dumps(selection).encode();(directory/'selection.json').write_bytes(raw) + engine=SelectedEngine(tmp_path);engine.selection=raw + engine.containers['grafana']=deepcopy(engine.current) + engine.containers['grafana']['Id']='c'*64 + engine.updater_capabilities=lambda source_id:updates.get_capabilities() + old_image=engine.image + def image(reference): + result=old_image(reference) + if '@sha256:' in reference:result['RepoDigests']=[reference] + return result + engine.image=image + job=updates.submit_job('prometheus','main','pull',directory=tmp_path/'jobs',engine=engine, + selection_id='synthetic-selection',selection_sha256=__import__('hashlib').sha256(raw).hexdigest(),selection_directory=directory, + components=['prometheus','grafana'],operation_id='12345678-1234-4234-8234-123456789abc',request_fingerprint='a'*64) + assert set(job['targets'])=={'prometheus','grafana'} + assert 'ipmi-monitor' not in job['sources'] + monkeypatch.setattr(updates,'pull_target',lambda reference:True) + result=updates.run_job(tmp_path/'jobs',job['id'],engine=engine) + assert result['success'] is True and set(result['results'])=={'prometheus','grafana'} + assert not [call for call in engine.calls if call[0]=='stop'] + + +@pytest.mark.parametrize('kind,name,source_path',[ + ('volume','fleet-auth-data','/var/lib/docker/volumes/fleet-auth-data/_data'), + ('bind',None,'/srv/fleet/auth'), +]) +def test_host_job_store_maps_to_same_backing_worker_namespace(kind,name,source_path): + source={'Image':'sha256:source','Mounts':[ + {'Type':'volume','Name':'anonymous-data','Source':'/var/lib/docker/volumes/anonymous-data/_data','Destination':'/data','RW':True}, + {'Type':kind,'Name':name,'Source':source_path,'Destination':'/data/auth','RW':True}]} + request=updates.helper_request(source,Path(source_path)/'update-jobs','a'*32) + assert request['Config']['Cmd'][-1]=='/data/auth/update-jobs' + assert request['HostConfig']['Mounts'][0]['Source']==(name if kind=='volume' else source_path) + assert request['HostConfig']['Mounts'][0]['Target']=='/data/auth' + if kind=='volume': + source['Mounts'].append({'Type':'bind','Source':'/shadow/operations','Destination':'/data/auth/update-jobs/operations','RW':True}) + with pytest.raises(updates.UpdateError):updates.helper_request(source,Path(source_path)/'update-jobs','a'*32) + source['Mounts'].pop() + source['Mounts'].append({'Type':'bind','Source':source_path,'Destination':'/other','RW':True}) + with pytest.raises(updates.UpdateError):updates.helper_request(source,Path(source_path)/'update-jobs','a'*32) + + +def test_status_cannot_interrupt_dispatch_before_helper_start(tmp_path,monkeypatch): + from tests.test_release_selection import selection_files,SelectedEngine + directory,digest=selection_files(tmp_path);engine=SelectedEngine(tmp_path) + engine.updater_capabilities=lambda _:updates.get_capabilities() + observations=[];operation='12345678-1234-4234-8234-123456789abc' + original_create=engine.create + def create(name,request): + observations.append(updates.operation_status(operation,directory=tmp_path/'jobs',engine=engine)['state']) + original_create(name,request) + engine.create=create + result=updates.submit_job('prometheus','main','pull',directory=tmp_path/'jobs',engine=engine, + selection_id='synthetic-selection',selection_sha256=digest,selection_directory=directory, + components=['prometheus'],operation_id=operation,request_fingerprint='a'*64) + assert observations==['queued'] and result['state']=='queued' + monkeypatch.setattr(updates,'pull_target',lambda _:True) + engine.selection=(directory/'selection.json').read_bytes() + assert updates.run_job(tmp_path/'jobs',result['id'],engine=engine)['state']=='completed' diff --git a/traceability-matrix.md b/traceability-matrix.md new file mode 100644 index 0000000..ab2ffdc --- /dev/null +++ b/traceability-matrix.md @@ -0,0 +1,59 @@ +# Fleet Manager proxy work-item trace evidence + +This is a thin product mapping to the [shared baseline](../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-20261027/traceability-matrix.md). +REQ/TRACE/VER/VAL-FLEETUPDATE-001 labels normalize exact approved source; +implementation and installed customer acceptance are separate. + +## Traceability Matrix + +| Trace | Requirement | Implementation | Verification | Validation | Acceptance status | Status | +|---|---|---|---|---|---|---| +| TRACE-FLEETUPDATE-001 | REQ-FLEETUPDATE-001 | `src/cryptolabs_proxy/auth.py`, `src/cryptolabs_proxy/config.py`, `src/cryptolabs_proxy/release_client.py`, `src/cryptolabs_proxy/release_selection.py`, `src/cryptolabs_proxy/standard_adoption.py`, `src/cryptolabs_proxy/updates.py`, `scripts/health-api.py` | VER-FLEETUPDATE-001: affected existing suites and shared preimplementation RED artifacts | VAL-FLEETUPDATE-001: shared validation-definition.json | Local implementation candidate; installed/native/browser/owner acceptance held | approved | + +The `approved` authority status above records only the exact owner-approved plan. +Implementation candidate; all installed/runtime/native/browser/requirement closure remains unaccepted and held. + +## Code Anchor Evidence + +| Artifact Path | Requirement | Trace | Verification | Anchor Type | Role | +| --- | --- | --- | --- | --- | --- | +| src/cryptolabs_proxy/auth.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | file-role | fleet-release-implementation | +| src/cryptolabs_proxy/release_client.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | file-role | fleet-release-implementation | +| src/cryptolabs_proxy/release_selection.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | file-role | fleet-release-implementation | +| src/cryptolabs_proxy/standard_adoption.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | file-role | fleet-release-implementation | +| src/cryptolabs_proxy/updates.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | file-role | fleet-release-implementation | +| scripts/health-api.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | file-role | fleet-release-implementation | +| tests/test_auth.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | verifies | verification | +| tests/test_updates.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | verifies | verification | +| tests/test_release_selection.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | verifies | verification | +| tests/test_standard_adoption.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | verifies | verification | +| src/cryptolabs_proxy/auth.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | entrypoint | fleet-release-entrypoint | +| src/cryptolabs_proxy/release_client.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | entrypoint | fleet-release-entrypoint | +| src/cryptolabs_proxy/release_selection.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | entrypoint | fleet-release-entrypoint | +| src/cryptolabs_proxy/standard_adoption.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | entrypoint | fleet-release-entrypoint | +| src/cryptolabs_proxy/updates.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | entrypoint | fleet-release-entrypoint | +| scripts/health-api.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | entrypoint | fleet-release-entrypoint | +| tests/test_proxy_customer_route_repairs.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | verifies | verification | +| src/cryptolabs_proxy/standard_adoption.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | entrypoint | fleet-release-recovery | +| src/cryptolabs_proxy/config.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | file-role | fleet-release-compose-consumer | +| src/cryptolabs_proxy/config.py | REQ-FLEETUPDATE-001 | TRACE-FLEETUPDATE-001 | VER-FLEETUPDATE-001 | entrypoint | fleet-release-compose-consumer | + +## Formats outside the Python anchor scanner + +These are manually mapped under the frozen profile; no unsupported-format scanner pass is claimed. + +| Artifact | SHA256 | Trace | Verification evidence | +|---|---|---|---| +| landing-page/index.html | 20061e8140c5132b68f0bdc5a7ebfd48023534cf6f1ea7f2921ea6d04530ff37 | TRACE-FLEETUPDATE-001 | proxy-repair-cycle1/format-verification.json + focused-final.log + full-suite.log; native/browser acceptance held | +| nginx/nginx.conf | 7993c9adc7b90f9c79cc569470008a124bb31763ecbc3cf12105cd906c930fbb | TRACE-FLEETUPDATE-001 | proxy-repair-cycle1/format-verification.json + focused-final.log + full-suite.log; native/browser acceptance held | +| src/cryptolabs_proxy/templates/nginx.conf.j2 | 6b40f203f6331d6610e558e7a42655f8917fa87ce8ada5edc6f844ffab274377 | TRACE-FLEETUPDATE-001 | proxy-repair-cycle1/format-verification.json + focused-final.log + full-suite.log; native/browser acceptance held | +| src/cryptolabs_proxy/templates/docker-compose.yml.j2 | ec923fb17e99c649bb0aacd95eb8aeee465cfc9dbc6c65b721461cbf094a8856 | TRACE-FLEETUPDATE-001 | proxy-repair-cycle1/format-verification.json + focused-final.log + full-suite.log; native/browser acceptance held | +| docs/release-selection.md | e7b27805bd35954c3f51cbeab2b28eff936ad3ac90c816e96906a0da8f9ce18e | TRACE-FLEETUPDATE-001 | proxy-repair-cycle1/format-verification.json + focused-final.log + full-suite.log; native/browser acceptance held | + +Evidence directory: `../artifacts/ipmi-readiness-20261026/release/fleet-manager-build-20261027/proxy-implementation/`. + +Immutable prior profile (data bytes preserved; no inserted Python anchor): + +| Artifact | SHA256 | Trace | Verification evidence | +|---|---|---|---| +| src/cryptolabs_proxy/templates/previous-standard-nginx-444d7c0.conf.j2 | f12e2b9800cbaa0e49e685506cfaad38eb7e568fa6d239fa15277af59073d8de | TRACE-FLEETUPDATE-001 | proxy-repair-cycle1/previous-standard-provenance.json; exact complete rendered tree and unknown directive rejection in test_standard_adoption.py |