From f71d67f5928a95ea34ea712870e4971ae91b42f8 Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Sat, 17 Oct 2026 22:24:27 +0100 Subject: [PATCH] otamaker: Manifest artefacts, tar payloads, signatures Manifests allow artefacts to be checked progressively, so that each payload is authenticated before it is used, rather than only once the whole download has arrived. A transitional artefact is a manifest artefact with a legacy _contents_.yaml as its first member, so that it can also be installed by OTA clients that predate manifests. It is validated by two hashes: the hash of the whole artefact for older applicators, and the manifest hash for newer ones. tar payloads are extracted at the given path (/ by default), using the ownership and permissions in the tar archive. Tar support is disabled if ExecuteScripts=false, and only available for manifest artefacts. Manifests can be signed by a list of private keys. The signatures are stored in _manifest_.sig, which immediately precedes _manifest_.yaml in the artefact archive. Signed-off-by: Phil Elwell --- otamaker/README.md | 115 ++++++++++++-- otamaker/otamaker | 379 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 462 insertions(+), 32 deletions(-) diff --git a/otamaker/README.md b/otamaker/README.md index f7fcc82..07c55ce 100644 --- a/otamaker/README.md +++ b/otamaker/README.md @@ -15,15 +15,46 @@ Or just download and run it - it's a Python3 script. **Usage** ``` -usage: otamaker [-h] [-w] contents [outfile] +usage: otamaker [-h] [-C DIR] [-w] [-t | -m | -l] [-c CHUNK_SIZE] + [-k SIGN_KEY] [-s SIGNATURE] [-f] [--signed-data FILE] + [--print-pubkey KEY] + [contents] [outfile] positional arguments: - contents the YAML contents file - outfile the artefact name (defaults to the 'name' in the contents) + contents the YAML contents file + outfile the artefact name (defaults to the 'name' in the + contents) options: - -h, --help show this help message and exit - -w, --writesum also write the sha256sum to .sha256sum + -h, --help show this help message and exit + -C DIR, --directory DIR + the directory payload paths are relative to (default: + the current directory) + -w, --writesum also write the sha256sum to .sha256sum (and + for a transitional artefact, the manifest hash to + .manifest.sha256sum) + -t, --transitional build a transitional artefact, with a manifest and a + legacy contents file for older applicators (the + default) + -m, --manifest build a manifest-only (format 2) artefact, published + by the hash of its manifest, which older applicators + can't install + -l, --legacy build a legacy artefact, without a manifest + -c CHUNK_SIZE, --chunk-size CHUNK_SIZE + chunk size for large payloads in a manifest artefact + (default 4M) + -k SIGN_KEY, --sign-key SIGN_KEY + sign the manifest with this ed25519 private key (PEM) + - may be repeated + -s SIGNATURE, --signature SIGNATURE + add the signature line(s) in this file to the manifest + signature - may be repeated + -f, --force allow an unsupported manifest format, or payload types + which need a manifest in a legacy or transitional + artefact, e.g. to test applicators + --signed-data FILE write the data to be signed to this file, and stop + --print-pubkey KEY print the trusted keys line (and key id) for this key, + and stop ``` **Enabling Rasberry Pi Connect Remote Updates** @@ -47,7 +78,9 @@ You will be prompted to enter your password to grant it the necessary permission **Update structure** -Raspberry Pi Connect Remote Update packages are `zstd`-compressed `tar` archives. The first element of the archive should be a YAML file describing the payloads of the update (*). otamaker reads such a YAML file, assembles the other elements into a temporary directory, renaming them as requested, then runs `tar` with the right options. +Raspberry Pi Connect Remote Update packages are `zstd`-compressed `tar` archives. The first element of the archive is a YAML file describing the payloads of the update (*). otamaker reads such a YAML file, assembles the other elements into a temporary directory, renaming them as requested, then runs `tar` with the right options. + +There are two generations of artefact. A _legacy_ artefact starts with the contents file, `_contents_.yaml`, and is published by the SHA256 of the whole archive, so nothing can be trusted until all of it has been downloaded. A _manifest_ (format 2) artefact starts with `_manifest_.yaml`, which otamaker generates from the contents file by adding the size and SHA256 of every payload (large payloads are hashed in chunks). It is published by the SHA256 of the manifest alone, so the applicator can check each payload as it arrives, and a manifest can be signed. By default otamaker builds a _transitional_ artefact which contains both files and can be installed either way - see "Artefact kinds" below. Here is an example of an artefact description file: @@ -73,13 +106,13 @@ artefact: type: script ``` -Any fields which aren't recognised are ignored. However, an error is returned if an update doesn't either write to at least one filesystem or include a script. +Any fields which aren't recognised are ignored. However, an error is returned if an update doesn't either write to at least one filesystem or include a script. `payloads` is shown here as a child of `artefact`, which is the preferred layout; a top-level `payloads` is also accepted. N.B. Although the contents file is structured as a YAML document, only a simple subset of the YAML standard is recognised. It is recommended that you stick to the structure above - anything clever is likely to fail. The artefact header is meant to provide a high-level view of the contents - what it is, and to which platforms it applies. The values here will appear in the logged output from the Remote Update service, but at present they have no other significance. -(*) If the first element in the archive is not called `_contents_.yaml`, every element is assumed to be a sparse image (see below). +(*) If the first element in the archive is not called `_contents_.yaml` or `_manifest_.yaml`, every element is assumed to be a sparse image (see below). **Payloads** @@ -88,9 +121,53 @@ The following payload types are recognised: * `eeprom-ab` - a Pi 5 A/B EEPROM update (requires that an AB-compatible EEPROM image is already installed, that the image includes `rpi-connect-ota` 1.3.14 or later and the `rpieepromab` package, and that `config.txt` includes `dtoverlay=nospi10`). * `script` - a program to execute on the remote device. As the name suggests, this is expected to be a script of some description, but it doesn't have to be. * `tmpfile` - some other file to be copied to the device for the duration of the update. These are intended to be used by scripts, which they must appear before in the list of payloads in order to be accessible. +* `tar` - a tar archive to be extracted on the device, at the directory given by an optional `path` field (`/` by default), with the ownership and permissions recorded in the archive. The applicator streams it straight through `tar` as each chunk is verified, so it is only available in manifest artefacts (and must be built with `-m` - see below), and it is disabled if the device has `ExecuteScripts=false`. +* `hashlist` - generated by otamaker for payloads which are hashed in chunks; don't write these yourself. + +Payload names must be unique, and may not start with `_`, which is reserved for the contents, manifest and signature files. Each payload is read from the file named by its `source` field, or from the file with the same name as the payload if there is no `source`. Paths are relative to the current directory, unless `-C ` (`--directory`) gives another, as for `tar`. `source` is a build-time field: otamaker removes it from the copy of the contents in the archive. An older form, a comment immediately after (and on the same line as) the `name` field, is still accepted as the path when there is no `source`; such comments are also removed. +**Artefact kinds** + +otamaker can build three kinds of artefact, chosen with one of `-t`, `-m` and `-l`: + +* `-t` (`--transitional`) - the current default. The archive starts with a legacy `_contents_.yaml` followed by `_manifest_.yaml`, so applicators from before manifests install it as a legacy artefact and newer ones as a manifest artefact. It therefore has two hashes: the SHA256 of the whole archive for older applicators, and the SHA256 of the manifest for newer ones. Give the kind explicitly in scripts which depend on it, because the default will become `-m` at some point. +* `-m` (`--manifest`) - a manifest-only artefact, published by the hash of its manifest. Older applicators can't install it. This is required for `tar` payloads, and for anything else older applicators don't understand. +* `-l` (`--legacy`) - a legacy artefact with just a `_contents_.yaml`, as otamaker has always built. + +With `-m` or `-t`, otamaker adds `size` and `sha256` fields to each payload in the manifest. Payloads larger than the chunk size (`-c`, default 4M, up to 64M) are instead hashed one chunk at a time, with the per-chunk digests in an extra `.hashes` payload placed before them, so that the applicator can check a large image as it is written. The manifest gets a `format` field, currently always 2; a `format` in the contents file asks for at least that version, and `-f` (`--force`) allows an unsupported one, e.g. to test applicators. Don't put `size`, `sha256` or `chunk_size` in the contents file yourself. + +`-w` (`--writesum`) writes the published hash to `.sha256sum`. For a transitional artefact it writes the whole-archive hash there, and the manifest hash to `.manifest.sha256sum`. + +Note that the manifest hash doesn't change between builds from the same contents and payloads, unlike the whole-archive hash, which depends on timestamps. + +**Signing** + +A manifest can be signed, which signs everything it describes. A device with a trusted keys file (`/etc/rpi-connect/rpi-ota-trusted-keys`, one public key per line) then refuses any artefact that isn't a manifest artefact signed by one of its keys, so it can only be updated by its owner, whichever server the artefact comes from. Devices without the file ignore signatures. + +Signing uses ed25519 keys, which `openssl` can make: +``` +$ openssl genpkey -algorithm ed25519 -out key.pem +$ otamaker --print-pubkey key.pem +``` +`--print-pubkey` prints the line to add to the device's trusted keys file, plus the key id. Then build with `-k`: +``` +$ otamaker -t -k key.pem contents.yaml +``` +`-k` can be repeated to sign with several keys, which is useful while rotating them - one valid signature from a trusted key is enough. The signature goes in a `_manifest_.sig` member just before the manifest; older applicators treat it as a `tmpfile`. + +If the private key isn't in a file otamaker can use, e.g. it lives in an HSM, sign in two steps. Run otamaker with `--signed-data` to write the bytes to be signed, sign them to produce a line of the form `ed25519 `, then run otamaker again with the same contents and payloads, adding `-s` with the file holding that line: +``` +$ otamaker -m --signed-data manifest.tbs contents.yaml +$ SIG=$(openssl pkeyutl -sign -rawin -inkey key.pem -in manifest.tbs | base64 -w 0) # or your HSM's equivalent +$ echo "ed25519 $SIG" > manifest.sig +$ otamaker -m -s manifest.sig contents.yaml +``` +where `` is as printed by `--print-pubkey`. + +Signed artefacts can also carry a `security_version` field in the artefact header. Once a device has installed and committed an artefact with a given `security_version`, it refuses anything with a lower one, which stops an old signed artefact with a known problem being installed again. Only raise it for releases that fix security problems. + **Scripts** It is expected that executable payloads will be written in one of the many scripting languages - Bourne shell, Perl, Python 3, etc. - because they are largely independent of the device architecture, but it could be any program that can be run on the target device. For interpreted languages this requires that the interpreter and any associated library modules are already installed on the target. It is also necessary to use a standard "shebang" (`#!/path/to/...`) header, so the target knows how to run it. Either way, executable payloads are referred to as scripts from now on. @@ -127,7 +204,7 @@ for i in {1..8}; do done echo $OLD_TRIGGER > $LED ``` -Put that in a file called `blinky`, then create `blink.yaml`: +Put that in a file called `blinky`, then create `blinky.yaml`: ``` artefact: name: blinky @@ -143,18 +220,20 @@ Now we can create the artefact: $ otamaker blinky.yaml Contents: _contents_.yaml + _manifest_.yaml blinky Artefact: blinky.tar.zst -SHA256: 1beab01defe22e7831fffd2ee42342bf85a50da2673628bbf742f6170b13c445 +SHA256: 9fbb02297d0229893b61a44559737e462c94ceeb109e263cca45704caf0064c2 (whole artefact) +SHA256: 637c1526b9df4200f960fdf31f14f3496e5205b0a0dc9a651c508fcc5fca1a58 (manifest) ``` -See how the YAML file, even though it was called `blink.yaml`, ended up being called `_contents_.yaml` as required. You will get a different SHA256 every time because of the changing timestamps in the temporary directory that `otamaker` creates. +See how the YAML file, even though it was called `blinky.yaml`, ended up being called `_contents_.yaml` as required, and that otamaker has added a `_manifest_.yaml` because it built a transitional artefact. There are two hashes: use the manifest hash for devices whose `rpi-connect-ota` understands manifests, and the whole-artefact hash for older ones. The whole-artefact hash is different every time because of the changing timestamps in the temporary directory that `otamaker` creates, but the manifest hash only changes when the contents or the payloads do. Artefacts can be hosted anywhere you like as long as it is accessible to the target. Find your host computer's IP address on your local network using `hostname -I` - on a home network it is likely to be of the form `192.168.x.y`, where `x` and `y` are numbers. Now run up an HTTP server on your host. This wouldn’t normally be the target device to update, but it will work if you only have one Pi. ``` $ python3 -m http.server 8000 --directory . ``` -When you create the artefact as part of adding a new deployment, the URI you will need to provide will be something like `http://192.168.x.y:8000/blink.tar.zst`, substituting your own values for `x` and `y`; the SHA256 checksum should be copied exactly from the output of `otamaker`. +When you create the artefact as part of adding a new deployment, the URI you will need to provide will be something like `http://192.168.x.y:8000/blinky.tar.zst`, substituting your own values for `x` and `y`; the SHA256 checksum should be copied exactly from the output of `otamaker` (the manifest hash, unless the device is too old to understand manifests). Before clicking the "Create and deploy" button (just "Deploy" if you have alread created the artefact), locate the green LED on your target Pi. Watch it as you click - it should flash for a second or two. @@ -178,15 +257,19 @@ artefact: ``` This artefact YAML file is designed to be used from root of the rpi-image-gen repository - you could also drop the paths, keeping `boot.sparse` and `system.sparse`, and either run `otamaker` from within the `work/image-myapp-1.0.3` directory (or your equivalent) or point it there with `-C`. Here's the output it generated: ``` -pi@philpi5:~/rpi-image-gen $ otamaker abupdate.yaml +pi@philpi5:~/rpi-image-gen $ otamaker abupdate.yaml Contents: _contents_.yaml + _manifest_.yaml + boot.hashes boot + system.hashes system Artefact: abupdate.tar.zst -SHA256: fa156308d8baeee47c7c610142a4d509a80b9c3b96ea21f0bbbb7740583eb8ba +SHA256: fa156308d8baeee47c7c610142a4d509a80b9c3b96ea21f0bbbb7740583eb8ba (whole artefact) +SHA256: 54fffd50f0b73544e321315541158e2ea202ad82337828ba2f0f21fa80be8a9a (manifest) pi@philpi5:~/rpi-image-gen $ ls -l abupdate.tar.zst --rw-rw-r-- 1 pi pi 248291841 Mar 13 16:19 abupdate.tar.zst +-rw-rw-r-- 1 pi pi 248315371 Nov 3 12:44 abupdate.tar.zst ``` -This is significantly larger than blinky, and may take a minute or more to install. At the end, the device will reboot into the new `boot` and `system` partitions, hopefully marking the deployment as successfully completed when it reconnects. +The images are larger than the 4M chunk size, so each gets a `.hashes` payload ahead of it holding the SHA256 of every chunk, and the applicator checks each chunk before writing it to the partition. This is significantly larger than blinky, and may take a minute or more to install. At the end, the device will reboot into the new `boot` and `system` partitions, hopefully marking the deployment as successfully completed when it reconnects. diff --git a/otamaker/otamaker b/otamaker/otamaker index 6e79b3d..5865317 100755 --- a/otamaker/otamaker +++ b/otamaker/otamaker @@ -7,6 +7,24 @@ Each payload is read from the file named by its 'source:' key, or by a comment on its 'name:' line (the older form), or failing both by its name. Paths are relative to the current directory, or to -C . 'source:' is dropped from the copy of the contents in the artefact. + +-t builds a transitional artefact, which applicators can install either as a +manifest (format 2) artefact or, if they predate manifests, as a legacy +artefact. -m builds a manifest-only artefact, and -l a legacy one. The default +is currently -t. + +Manifests can be signed with ed25519 keys (-k), e.g. made with: + openssl genpkey -algorithm ed25519 -out key.pem +A device which has a trusted keys file (/etc/rpi-connect/rpi-ota-trusted-keys) +only accepts artefacts signed by one of its keys. --print-pubkey prints the +line to add to that file for a key. + +To sign with a key which otamaker can't use directly (e.g. in an HSM), run +otamaker with --signed-data to write the data to be signed, sign it, and run +otamaker again with the same contents and payloads, adding -s with a file +holding the line: + ed25519 +where the key id is as printed by --print-pubkey. """ # Author: Phil Elwell @@ -41,6 +59,8 @@ import os import re import sys import argparse +import base64 +import hashlib import shutil import subprocess import tempfile @@ -50,19 +70,184 @@ BLOCK_HEADER = 1 BLOCK_PAYLOADS = 2 CONTENTS_FILE = '_contents_.yaml' +MANIFEST_FILE = '_manifest_.yaml' +SIGNATURE_FILE = '_manifest_.sig' +# Signatures are over this, followed by the manifest +SIGNATURE_DOMAIN = b'rpi-ota-manifest-v1\n' +MAX_SIGNATURE_FILE_SIZE = 4096 +# Format 1 is legacy. The format is raised for changes to the manifest which +# older applicators must not ignore, so a manifest has the lowest format which +# covers its contents. The contents file may ask for a higher one. +FIRST_MANIFEST_FORMAT = 2 +MANIFEST_FORMAT = 2 # The highest supported format +DEFAULT_CHUNK_SIZE = 4 * 1024 * 1024 +MAX_CHUNK_SIZE = 64 * 1024 * 1024 +# Keys which otamaker generates for a manifest +MANIFEST_KEYS = ('size', 'sha256', 'chunk_size') -# Must be kept up to date with new payload types -KNOWN_PAYLOAD_TYPES = ('image-sparse', 'script', 'tmpfile', 'eeprom-ab') +# The first artefact format which supports each payload type. Format 1 types +# can be installed by applicators which predate manifests. +PAYLOAD_TYPES = { + 'image-sparse': 1, + 'script': 1, + 'tmpfile': 1, + 'eeprom-ab': 1, + 'tar': 2, +} # The final artefact is already zstd-compressed as a whole, so an # individually-compressed payload (e.g. 'boot.sparse.zst' instead of # 'boot.sparse') is almost always a mistake. COMPRESSED_SUFFIXES = ('.zst', '.gz', '.bz2', '.xz', '.lz4', '.tgz', '.tzst') +def parse_size(text): + m = re.match(r'^(\d+)([KkMm]?)$', text) + if not m: + raise argparse.ArgumentTypeError(f"invalid size '{text}'") + size = int(m.group(1)) * {'': 1, 'k': 1024, 'm': 1024 * 1024}[m.group(2).lower()] + if size < 1 or size > MAX_CHUNK_SIZE: + raise argparse.ArgumentTypeError(f"size must be between 1 and {MAX_CHUNK_SIZE}") + return size + +def file_sha256(path): + h = hashlib.sha256() + with open(path, 'rb') as fh: + while True: + buf = fh.read(1024 * 1024) + if not buf: + break + h.update(buf) + return h.hexdigest() + +def chunk_digests(path, chunk_size): + digests = [] + with open(path, 'rb') as fh: + while True: + buf = fh.read(chunk_size) + if not buf: + break + digests.append(hashlib.sha256(buf).digest()) + return b''.join(digests) + +def openssl(args, input=None): + r = subprocess.run(['openssl'] + args, input=input, capture_output=True) + if r.returncode != 0: + print(f"* error: openssl {' '.join(args)}: {r.stderr.decode(errors='replace').strip()}") + sys.exit(1) + return r.stdout + +def public_key(keyfile): + """Return the raw ed25519 public key from a private or public key file""" + r = subprocess.run(['openssl', 'pkey', '-in', keyfile, '-pubout', '-outform', 'DER'], + capture_output=True) + if r.returncode != 0: + r = subprocess.run(['openssl', 'pkey', '-pubin', '-in', keyfile, '-outform', 'DER'], + capture_output=True) + # A DER ed25519 public key is a fixed 12 byte header, then the key + if r.returncode != 0 or len(r.stdout) != 44: + print(f"* error: '{keyfile}' is not an ed25519 key") + sys.exit(1) + return r.stdout[12:] + +def key_id(pubkey): + return hashlib.sha256(pubkey).hexdigest()[:16] + +def sign_manifest(keyfile, manifest_path): + """Return a signature line for the manifest""" + with open(manifest_path, 'rb') as fh: + data = SIGNATURE_DOMAIN + fh.read() + with tempfile.NamedTemporaryFile() as tbs: + tbs.write(data) + tbs.flush() + sig = openssl(['pkeyutl', '-sign', '-rawin', '-inkey', keyfile, '-in', tbs.name]) + if len(sig) != 64: + print(f"* error: '{keyfile}' did not make an ed25519 signature") + sys.exit(1) + return f'ed25519 {key_id(public_key(keyfile))} {base64.b64encode(sig).decode()}' + +def build_manifest(contents, payloads_idx, groups, tmpdir, chunk_size, manifest_format): + """Return the manifest lines and the archive member names (in order). + + Each payload gains a size and either a sha256, or (for payloads larger + than chunk_size) a chunk_size and a preceding hashlist payload + (.hashes) holding the SHA256 of each chunk.""" + header = [line for line in contents[:payloads_idx] if not re.match(r'^\s+format:', line)] + for i, line in enumerate(header): + if line.strip() == 'artefact:': + header.insert(i + 1, f' format: {manifest_format}') + break + out = header + [contents[payloads_idx]] + members = [MANIFEST_FILE] + names = {g['name'] for g in groups} + + for g in groups: + path = os.path.join(tmpdir, g['name']) + if not os.path.isfile(path): + print(f"* error: payload {g['name']} is not a regular file") + sys.exit(1) + size = os.path.getsize(path) + # Match the indentation of the payload list + m = re.match(r'^(\s*)-(\s*)', g['lines'][0]) + item = m.group(1) + indent = item + ' ' + m.group(2) + lines = [line for line in g['lines'] + if not re.match(r'^\s*-?\s*(' + '|'.join(MANIFEST_KEYS) + r'):', line)] + if size > chunk_size: + hashlist = g['name'] + '.hashes' + if hashlist in names: + print(f'* error: hashlist name {hashlist} clashes with a payload') + sys.exit(1) + digests = chunk_digests(path, chunk_size) + with open(os.path.join(tmpdir, hashlist), 'wb') as fh: + fh.write(digests) + out += [f'{item}- name: {hashlist}', + f'{indent}type: hashlist', + f'{indent}size: {len(digests)}', + f'{indent}sha256: {hashlib.sha256(digests).hexdigest()}'] + members.append(hashlist) + lines += [f'{indent}size: {size}', + f'{indent}chunk_size: {chunk_size}'] + else: + lines += [f'{indent}size: {size}', + f'{indent}sha256: {file_sha256(path)}'] + out += lines + members.append(g['name']) + + return out, members + +def build_legacy_contents(contents, payloads_idx, groups, members): + """Return the lines of a _contents_.yaml for a transitional artefact. + + This lets applicators which predate manifests install the artefact. The + manifest, hashlists and payloads of types they don't know are listed as + tmpfiles, because older applicators would try to flash any member which + isn't listed (or has an unknown type) to the slot of that name. payloads + is placed at the top level, which all versions accept.""" + out = list(contents[:payloads_idx]) + ['payloads:'] + by_name = {g['name']: g for g in groups} + + for name in members: + g = by_name.get(name) + if g is None: + out += [f'- name: {name}', + ' type: tmpfile'] + continue + m = re.match(r'^(\s*)-', g['lines'][0]) + item = m.group(1) + for line in g['lines']: + if line.startswith(item): + line = line[len(item):] + if PAYLOAD_TYPES.get(g['type']) != 1: + line = re.sub(r'^(\s*-?\s*type:\s*).*$', r'\1tmpfile', line) + out.append(line) + + return out + def main(): parser = argparse.ArgumentParser() parser.add_argument('contents', - help='the YAML contents file') + help='the YAML contents file', + nargs='?') parser.add_argument('outfile', help="the artefact name (defaults to the 'name' in the contents)", nargs='?') @@ -71,10 +256,56 @@ def main(): '(default: the current directory)', metavar='DIR') parser.add_argument('-w', '--writesum', - help='also write the sha256sum to .sha256sum', + help='also write the sha256sum to .sha256sum (and for a transitional ' + 'artefact, the manifest hash to .manifest.sha256sum)', + action='store_true') + kind = parser.add_mutually_exclusive_group() + kind.add_argument('-t', '--transitional', + help='build a transitional artefact, with a manifest and a legacy contents file ' + 'for older applicators (the default)', + action='store_true') + kind.add_argument('-m', '--manifest', + help="build a manifest-only (format 2) artefact, published by the hash of its " + "manifest, which older applicators can't install", + action='store_true') + kind.add_argument('-l', '--legacy', + help='build a legacy artefact, without a manifest', + action='store_true') + parser.add_argument('-c', '--chunk-size', + help=f'chunk size for large payloads in a manifest artefact (default {DEFAULT_CHUNK_SIZE // (1024 * 1024)}M)', + type=parse_size, default=DEFAULT_CHUNK_SIZE) + parser.add_argument('-k', '--sign-key', + help='sign the manifest with this ed25519 private key (PEM) - may be repeated', + action='append', default=[]) + parser.add_argument('-s', '--signature', + help='add the signature line(s) in this file to the manifest signature - may be repeated', + action='append', default=[]) + parser.add_argument('-f', '--force', + help='allow an unsupported manifest format, or payload types which need a manifest ' + 'in a legacy or transitional artefact, e.g. to test applicators', action='store_true') + parser.add_argument('--signed-data', + help='write the data to be signed to this file, and stop', + metavar='FILE') + parser.add_argument('--print-pubkey', + help='print the trusted keys line (and key id) for this key, and stop', + metavar='KEY') args = parser.parse_args() + if args.print_pubkey: + pubkey = public_key(args.print_pubkey) + print(f'ed25519 {base64.b64encode(pubkey).decode()} {os.path.basename(args.print_pubkey)}') + print(f'Key id: {key_id(pubkey)}') + return + if not args.contents: + parser.error('the contents file is required') + signing = args.sign_key or args.signature or args.signed_data + if signing and args.legacy: + parser.error('only a manifest can be signed') + # Transitional artefacts include a manifest + if not (args.manifest or args.legacy): + args.transitional = True + args.manifest = not args.legacy try: with open(args.contents, 'r', encoding='utf-8', errors='replace') as fh: @@ -91,8 +322,12 @@ def main(): state = BLOCK_NONE contents = [] payloads = [CONTENTS_FILE] + payloads_idx = None outfile = args.outfile errors = 0 + manifest_format = FIRST_MANIFEST_FORMAT + requested_format = None + format_payload = None # The payload which needs manifest_format, if any groups = [] # One per payload: name, type, source, comment item_prefix = None # The '- ' of a list item whose first line was dropped @@ -133,10 +368,23 @@ def main(): state = BLOCK_HEADER elif line.strip() == 'payloads:': state = BLOCK_PAYLOADS + payloads_idx = len(contents) - 1 elif state == BLOCK_HEADER: name_m = re.match(r'^\s+name:\s*(.+)$', line) if name_m and args.outfile is None: outfile = name_m.group(1).strip() + secver_m = re.match(r'^\s+security_version:\s*(.*)$', line) + if secver_m and not re.match(r'^\d+$', secver_m.group(1).strip()): + print(f"* error: security_version must be a non-negative integer") + errors += 1 + format_m = re.match(r'^\s+format:\s*(.*)$', line) + if format_m: + fmt = format_m.group(1).strip() + if not re.match(r'^\d+$', fmt): + print(f"* error: format must be a non-negative integer") + errors += 1 + else: + requested_format = int(fmt) elif state == BLOCK_PAYLOADS: # Normalise list item: '- name: x' -> ' name: x' payload_line = re.sub(r'^\s*-\s*', ' ', line) @@ -174,10 +422,17 @@ def main(): print(f"* error: payload '{name}' has no type") errors += 1 continue - if ptype not in KNOWN_PAYLOAD_TYPES: - print(f"* error: payload '{name}' has unknown type '{ptype}' " - f"(expected one of: {', '.join(KNOWN_PAYLOAD_TYPES)})") - errors += 1 + if PAYLOAD_TYPES.get(ptype, 1) > manifest_format: + manifest_format = PAYLOAD_TYPES[ptype] + format_payload = name + if ptype not in PAYLOAD_TYPES: + # Applicators verify and then ignore manifest payloads of unknown + # type, but would flash a legacy one to the slot of that name + print(f"* {'error' if args.legacy else 'warning'}: payload '{name}' " + f"has unknown type '{ptype}' " + f"(expected one of: {', '.join(PAYLOAD_TYPES)})") + if args.legacy: + errors += 1 if ptype != "tmpfile" and target.endswith(COMPRESSED_SUFFIXES): print(f"* warning: payload '{name}' file '{target}' looks pre-compressed; " f"the artefact is already compressed as a whole, " @@ -200,17 +455,100 @@ def main(): print(f"* error: payload {name}: cannot symlink '{target}'") errors += 1 + for g in groups: + # Names starting with '_' are reserved for members such as _manifest_.yaml + if g['name'] and g['name'].startswith('_'): + print(f"* error: payload {g['name']}: names starting with '_' are reserved") + errors += 1 + if g['type'] == 'hashlist': + print(f"* error: payload {g['name']}: hashlists are generated by otamaker") + errors += 1 + # Older applicators can only install format 1 types, and would skip or + # reject the others. -f allows them, e.g. to test applicators. + if PAYLOAD_TYPES.get(g['type'], 1) > 1 and (args.legacy or args.transitional): + print(f"* {'warning' if args.force else 'error'}: payload {g['name']}: " + f"{g['type']} payloads need a manifest-only artefact (-m)") + if not args.force: + errors += 1 + + # A legacy artefact has no manifest, so no format + if args.manifest: + if requested_format is not None: + # -f uses the requested format as given, e.g. a format too low for + # the contents, to test applicators + reason = f" for payload '{format_payload}'" if format_payload else "" + if requested_format >= manifest_format: + manifest_format = requested_format + elif args.force: + if format_payload: + print(f"* warning: format {requested_format} is too low{reason}") + manifest_format = requested_format + else: + print(f"* warning: format {requested_format} raised to {manifest_format}{reason}") + if not FIRST_MANIFEST_FORMAT <= manifest_format <= MANIFEST_FORMAT: + print(f"* {'warning' if args.force else 'error'}: unsupported format {manifest_format}") + if not args.force: + errors += 1 + if errors: sys.exit(1) - contents_path = os.path.join(tmpdir, CONTENTS_FILE) + contents_file = CONTENTS_FILE + legacy_contents = None + if args.manifest: + if payloads_idx is None or any(not g['name'] for g in groups): + print('* error: malformed payloads list') + sys.exit(1) + manifest, payloads = build_manifest(contents, payloads_idx, groups, + tmpdir, args.chunk_size, manifest_format) + if args.sign_key or args.signature: + # The signature immediately precedes the manifest + payloads = [SIGNATURE_FILE] + payloads + if args.transitional: + # The legacy contents come first, for older applicators + legacy_contents = build_legacy_contents(contents, payloads_idx, groups, payloads) + payloads = [CONTENTS_FILE] + payloads + contents = manifest + contents_file = MANIFEST_FILE + + contents_path = os.path.join(tmpdir, contents_file) try: with open(contents_path, 'w', encoding='utf-8') as ofh: ofh.write('\n'.join(contents) + '\n') + if legacy_contents: + with open(os.path.join(tmpdir, CONTENTS_FILE), 'w', encoding='utf-8') as ofh: + ofh.write('\n'.join(legacy_contents) + '\n') except OSError: print('* failed to create new contents file') sys.exit(1) + if args.signed_data: + with open(contents_path, 'rb') as fh, open(args.signed_data, 'wb') as ofh: + ofh.write(SIGNATURE_DOMAIN + fh.read()) + print(f'Signed data: {args.signed_data}') + return + + if args.sign_key or args.signature: + sig_lines = [sign_manifest(key, contents_path) for key in args.sign_key] + for sigfile in args.signature: + try: + with open(sigfile, 'r', encoding='utf-8') as fh: + sig_lines += [line.strip() for line in fh + if line.strip() and not line.startswith('#')] + except OSError: + print(f"* failed to open '{sigfile}'") + sys.exit(1) + for line in sig_lines: + if not re.match(r'^ed25519 [0-9a-f]{16} [A-Za-z0-9+/]{86}==$', line): + print(f"* error: malformed signature line '{line}'") + sys.exit(1) + sig_text = '\n'.join(sig_lines) + '\n' + if len(sig_text) > MAX_SIGNATURE_FILE_SIZE: + print('* error: too many signatures') + sys.exit(1) + with open(os.path.join(tmpdir, SIGNATURE_FILE), 'w', encoding='utf-8') as ofh: + ofh.write(sig_text) + if not outfile: print("* error: no output name (set 'name' in artefact header or pass as argument)") sys.exit(1) @@ -225,20 +563,29 @@ def main(): if r.returncode != 0: sys.exit(r.returncode) - proc = subprocess.Popen(['sha256sum', outfile], stdout=subprocess.PIPE) - line = proc.stdout.readline().decode().strip() - sum = line[:64] + # A manifest artefact is published by the hash of its manifest. A + # transitional artefact is published by both: the hash of the whole + # artefact for older applicators, and the manifest hash for newer ones. + sum = file_sha256(contents_path if args.manifest else outfile) + legacy_sum = file_sha256(outfile) if args.transitional else None if args.writesum: - sum_path = f'{outfile}.sha256sum' - with open(sum_path, 'w') as sf: - print(sum, file=sf) + if legacy_sum: + with open(f'{outfile}.sha256sum', 'w') as sf: + print(legacy_sum, file=sf) + with open(f'{outfile}.manifest.sha256sum', 'w') as sf: + print(sum, file=sf) + else: + with open(f'{outfile}.sha256sum', 'w') as sf: + print(sum, file=sf) print('Contents:') for payload in payloads: print(f' {payload}') print() print(f'Artefact: {outfile}') - print(f'SHA256: {sum}') + if legacy_sum: + print(f'SHA256: {legacy_sum} (whole artefact)') + print(f"SHA256: {sum}{' (manifest)' if args.manifest else ''}") if __name__ == '__main__': main()