Doughmination Manager, verified release manifest, per-app versions

.github/apps.json

@@ -0,0 +1,31 @@
+{
+  "_comment": "What the Manager shows for each app, and the certificate each published APK must be signed with (docs/releasing.md). CI fails rather than publish an APK whose certificate isn't the one pinned here. Keyed by the APK's file name in the release.",
+  "system.apk": {
+    "id": "system",
+    "name": "System",
+    "summary": "The plural system: members, fronting and relationships.",
+    "dir": "plural",
+    "signingCertSha256": "F1:0D:90:FE:C9:0C:97:18:C0:06:8B:70:5A:55:92:62:05:DF:11:74:95:62:A2:7D:30:B5:16:70:40:3E:1D:C1"
+  },
+  "inbox.apk": {
+    "id": "inbox",
+    "name": "Inbox",
+    "summary": "Mail for every Doughmination domain.",
+    "dir": "email",
+    "signingCertSha256": "79:7D:66:1F:F5:EE:00:FD:DB:E3:E0:7F:FF:07:C5:EF:AC:C3:50:8F:18:65:0F:60:F5:A6:A6:4B:95:30:A8:DD"
+  },
+  "device-reporter.apk": {
+    "id": "device-reporter",
+    "name": "Device Reporter",
+    "summary": "Reports this phone's battery and state to the Doughmination API.",
+    "dir": "api-helper",
+    "signingCertSha256": "D1:D0:FB:74:BA:38:5E:69:55:BF:58:97:FD:E7:18:4F:4F:2F:1B:91:FD:0E:FA:2D:9E:A8:FF:C0:59:B8:CF:D5"
+  },
+  "doughmination-manager.apk": {
+    "id": "manager",
+    "name": "Doughmination Manager",
+    "summary": "Installs and updates the Doughmination apps.",
+    "dir": "doughmination-manager",
+    "signingCertSha256": "22:8D:A3:BC:CF:8B:ED:60:21:B8:E6:E0:17:C2:FE:62:F1:EC:52:9C:8C:D4:2C:8D:41:A1:35:FA:62:7B:E8:AA"
+  }
+}

.github/scripts/build-apps.sh

@@ -6,16 +6,29 @@
 #
 #   build-apps.sh <out>
 #
+# Everything in versions.json is read back from the built APK itself (aapt2
+# for the package and versions, apksigner for the certificate), and the build
+# fails if the APK doesn't carry the version it was asked to build. So the
+# version an APK reports, the one Gradle was given and the one published are
+# always the same.
+#
 # Keys are read from $KEYS/<app>.p12 with their passwords in the environment
 # (see apps.yml). Without a key an app builds with the debug key (Inbox:
-# unsigned), which is fine for pull requests and never published.
+# unsigned), which is fine for pull requests and never published. With
+# REQUIRE_PINNED_CERT=true, an APK whose certificate isn't the one pinned in
+# .github/apps.json fails the build.
 set -euo pipefail
 
 out=$1
 keys=${KEYS:-${RUNNER_TEMP:-/tmp}/keys}
 mkdir -p "$out"
 manifest="[]"
 
+tools=$(ls -d "$ANDROID_HOME"/build-tools/*/ | sort -V | tail -1)
+aapt2="${tools}aapt2"
+apksigner="${tools}apksigner"
+pins=.github/apps.json
+
 # build <file name> <gradle module> <module dir> <package> <key name> <password> <alias>
 build() {
   local file=$1 module=$2 dir=$3 package=$4 key=$5 password=$6 alias=$7
@@ -32,14 +45,44 @@ build() {
   local apk
   apk=$(ls "$dir"/build/outputs/apk/release/*.apk | head -1)
   cp "$apk" "$out/$file"
+
+  # What the APK really is.
+  local badging pkg code name minsdk cert
+  badging=$("$aapt2" dump badging "$out/$file")
+  pkg=$(sed -n "s/^package: name='\([^']*\)'.*/\1/p" <<< "$badging")
+  code=$(sed -n "s/^package: .* versionCode='\([^']*\)'.*/\1/p" <<< "$badging")
+  name=$(sed -n "s/^package: .* versionName='\([^']*\)'.*/\1/p" <<< "$badging")
+  minsdk=$(sed -n "s/^\(minSdkVersion\|sdkVersion\):'\([0-9]*\)'.*/\2/p" <<< "$badging" | head -1)
+  # "Signer #1 certificate SHA-256 digest: …" or, newer, "V2 Signer: certificate SHA-256 digest: …"
+  cert=$("$apksigner" verify --print-certs "$out/$file" 2>/dev/null \
+    | sed -n 's/^.*[Ss]igner.*certificate SHA-256 digest: *//p' | head -1 || true)
+
+  local expected_code=${VERSION_CODE:-1} expected_name=${VERSION_NAME:-}
+  if [ "$pkg" != "$package" ]; then
+    echo "::error::$file is package '$pkg', expected '$package'"; exit 1
+  fi
+  if [ "$code" != "$expected_code" ] || { [ -n "$expected_name" ] && [ "$name" != "$expected_name" ]; }; then
+    echo "::error::$file reports $name ($code), but this build is ${expected_name:-?} ($expected_code)"; exit 1
+  fi
+  if [ "${REQUIRE_PINNED_CERT:-false}" = true ]; then
+    local pinned
+    pinned=$(jq -r --arg f "$file" '.[$f].signingCertSha256 // ""' "$pins" | tr -d ':' | tr 'A-F' 'a-f')
+    if [ -z "$cert" ] || [ "$cert" != "$pinned" ]; then
+      echo "::error::$file is signed by ${cert:-nothing}, not the key pinned in $pins ($pinned)"; exit 1
+    fi
+  fi
+  echo "$file: $pkg $name ($code), minSdk $minsdk, certificate ${cert:-none}"
+
   manifest=$(jq \
     --arg file "$file" \
-    --arg packageName "$package" \
-    --argjson versionCode "${VERSION_CODE:-1}" \
-    --arg versionName "${VERSION_NAME:-dev}" \
+    --arg packageName "$pkg" \
+    --argjson versionCode "$code" \
+    --arg versionName "$name" \
+    --argjson minSdk "${minsdk:-1}" \
+    --arg signingCertSha256 "$cert" \
     --arg sha256 "$(sha256sum "$out/$file" | cut -d' ' -f1)" \
     --argjson sizeBytes "$(stat -c %s "$out/$file")" \
-    '. + [{$file, $packageName, $versionCode, $versionName, $sha256, $sizeBytes}]' <<< "$manifest")
+    '. + [{$file, $packageName, $versionCode, $versionName, $minSdk, $signingCertSha256, $sha256, $sizeBytes}]' <<< "$manifest")
   echo "::endgroup::"
 }
 

.github/scripts/make-manifest.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Writes <dist>/manifest.json, the document Doughmination Manager reads
+# (docs/manifest.md), and <dist>/publish.txt, the APKs that need uploading.
+#
+#   make-manifest.sh <dist> [previous manifest.json]
+#
+# Inputs: <dist>/versions.json (what build-apps.sh read back from each APK),
+# .github/apps.json (names, summaries, pinned certificates), git history, and
+# the manifest that's published now.
+#
+# Each app gets a new version only when its sources changed. An app's
+# fingerprint covers its own directory and the shared build files; when it
+# matches the published one, the app keeps its published entry and APK, so the
+# Manager offers no update for it. Otherwise the freshly built APK is
+# published, and:
+#   - its release notes are the commits that touched its directory since its
+#     own last release;
+#   - its versionCode must be higher than the published one, or the script
+#     fails and nothing is published.
+set -euo pipefail
+
+dist=$1
+previous=${2:-}
+[ -n "$previous" ] && [ -s "$previous" ] || previous=""
+repo=${GITHUB_REPOSITORY:-doughmination/apps}
+download="https://github.com/$repo/releases/latest/download"
+release_url="https://github.com/$repo/releases/tag/latest"
+now=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+head_commit=$(git rev-parse HEAD)
+
+# Files every app's build depends on. Changing any of them rebuilds every app.
+shared="design gradle build.gradle.kts settings.gradle.kts gradle.properties gradlew"
+
+fingerprint() {
+  local paths="$1 $shared" p
+  for p in $paths; do echo "$p $(git rev-parse "HEAD:$p")"; done | sha256sum | cut -d' ' -f1
+}
+
+apps="[]"
+: > "$dist/publish.txt"
+for file in $(jq -r '.apps[].file' "$dist/versions.json"); do
+  built=$(jq --arg f "$file" '.apps[] | select(.file == $f)' "$dist/versions.json")
+  meta=$(jq --arg f "$file" '.[$f]' .github/apps.json)
+  if [ "$meta" = null ]; then
+    echo "::error::$file has no entry in .github/apps.json"; exit 1
+  fi
+  dir=$(jq -r '.dir' <<< "$meta")
+  package=$(jq -r '.packageName' <<< "$built")
+  code=$(jq -r '.versionCode' <<< "$built")
+  source_hash=$(fingerprint "$dir")
+
+  published="null"
+  if [ -n "$previous" ]; then
+    published=$(jq --arg p "$package" '[.apps[] | select(.packageName == $p)][0] // null' "$previous")
+  fi
+
+  if [ "$published" != null ] && [ "$(jq -r '.sourceHash // ""' <<< "$published")" = "$source_hash" ]; then
+    echo "$file: unchanged, keeping $(jq -r '.latest.versionName' <<< "$published")"
+    apps=$(jq --argjson app "$published" --argjson meta "$meta" \
+      '. + [$app + {id: $meta.id, name: $meta.name, summary: $meta.summary}]' <<< "$apps")
+    continue
+  fi
+
+  if [ "$published" != null ]; then
+    old=$(jq -r '.latest.versionCode' <<< "$published")
+    if [ "$code" -le "$old" ]; then
+      echo "::error::$file would be version code $code, but $old is already published. Version codes must only go up (docs/releasing.md)."
+      exit 1
+    fi
+  fi
+
+  # What changed in this app since its own last release.
+  since=$(jq -r '.sourceCommit // ""' <<< "$published")
+  if [ -n "$since" ] && git cat-file -e "$since^{commit}" 2>/dev/null; then
+    notes=$(git log --no-merges --format='- %s' "$since..$head_commit" -- "$dir" | head -40)
+    [ -n "$notes" ] || notes="- Rebuilt for a change to the shared build setup."
+  else
+    notes=$(git log --no-merges --format='- %s' -10 -- "$dir")
+  fi
+
+  echo "$file: changed, publishing $(jq -r '.versionName' <<< "$built")"
+  echo "$file" >> "$dist/publish.txt"
+  apps=$(jq \
+    --argjson built "$built" \
+    --argjson meta "$meta" \
+    --arg url "$download/$file" \
+    --arg notes "$notes" \
+    --arg now "$now" \
+    --arg releaseUrl "$release_url" \
+    --arg sourceHash "$source_hash" \
+    --arg sourceCommit "$head_commit" \
+    '. + [{
+      id: $meta.id,
+      packageName: $built.packageName,
+      name: $meta.name,
+      summary: $meta.summary,
+      signingCertSha256: [$meta.signingCertSha256],
+      sourceHash: $sourceHash,
+      sourceCommit: $sourceCommit,
+      latest: {
+        versionCode: $built.versionCode,
+        versionName: $built.versionName,
+        apkUrl: $url,
+        sha256: $built.sha256,
+        sizeBytes: $built.sizeBytes,
+        minSdk: $built.minSdk,
+        releasedAt: $now,
+        releaseNotes: $notes,
+        releaseUrl: $releaseUrl
+      }
+    }]' <<< "$apps")
+done
+
+jq -n --arg now "$now" --arg commit "$head_commit" --argjson apps "$apps" \
+  '{schemaVersion: 1, generatedAt: $now, commit: $commit, apps: $apps}' > "$dist/manifest.json"
+cat "$dist/manifest.json"
+echo "To publish: $(tr '\n' ' ' < "$dist/publish.txt")"

.github/scripts/manager-e2e.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# The whole update path, on the running emulator: install Doughmination
+# Manager, let it read the published manifest, then have it download, verify
+# and install Inbox from the rolling release, confirming Android's own install
+# prompt the way a person would. Fails unless Inbox ends up installed at
+# exactly the version the manifest lists.
+#
+#   manager-e2e.sh <manager apk> <output dir>
+set -euo pipefail
+
+apk=$1
+out=$2
+pkg=gay.doughmination.manager
+target=gay.doughmination.mail
+manifest_url="https://github.com/${GITHUB_REPOSITORY:-doughmination/apps}/releases/latest/download/manifest.json"
+mkdir -p "$out"
+
+if ! curl -fsSL "$manifest_url" -o "$out/manifest.json"; then
+  echo "::warning::No published manifest yet; skipping the Manager test."
+  exit 0
+fi
+want_code=$(jq -r --arg p "$target" '.apps[] | select(.packageName == $p) | .latest.versionCode' "$out/manifest.json")
+echo "The manifest lists $target at version code $want_code"
+
+ui() { adb exec-out uiautomator dump /dev/tty 2>/dev/null | sed 's/UI hierch*ary dumped to.*//' || true; }
+
+shot() {
+  sleep "${2:-2}"
+  adb exec-out screencap -p > "$out/$1.png"
+  ui > "$out/$1.xml"
+  echo "saved $1"
+}
+
+# Taps the first element whose text or description matches the regex $1 exactly.
+tap() {
+  local bounds
+  bounds=$(ui | grep -oE "(text|content-desc)=\"$1\"[^>]*bounds=\"\[[0-9]+,[0-9]+\]\[[0-9]+,[0-9]+\]\"" | head -1 \
+    | grep -oE 'bounds="[^"]+"' | grep -oE '[0-9]+' | tr '\n' ' ' || true)
+  if [ -z "$bounds" ]; then
+    echo "::error::Couldn't find \"$1\" on screen"
+    return 1
+  fi
+  set -- $bounds
+  adb shell input tap $(( ($1 + $3) / 2 )) $(( ($2 + $4) / 2 ))
+}
+
+expect() {
+  if ! ui | grep -qE "text=\"$1"; then
+    echo "::error::Expected \"$1\" on screen at step $2"
+    exit 1
+  fi
+  echo "found \"$1\""
+}
+
+adb uninstall "$target" > /dev/null 2>&1 || true
+adb install -r "$apk"
+# What a person does once in Settings: allow the Manager to install apps.
+adb shell appops set "$pkg" REQUEST_INSTALL_PACKAGES allow
+
+adb shell am start -W -n "$pkg/.MainActivity"
+shot 01-list 10
+expect "Doughmination Manager" "launch"
+expect "Inbox" "the app list"
+expect "Not installed" "the app list"
+
+tap "Inbox"
+shot 02-detail 2
+expect "What's new" "Inbox's page"
+tap "Install [^\"]*"
+echo "Downloading and verifying..."
+
+# Android's confirmation prompt appears once the APK is downloaded and checked.
+for i in $(seq 1 60); do
+  if ui | grep -qiE 'text="(Install|INSTALL|Update|UPDATE)"[^>]*package="com\.(google\.)?android\.packageinstaller"'; then
+    break
+  fi
+  if ui | grep -qE "doesn't match its SHA-256|isn't signed with|was interrupted|download failed|Couldn't install"; then
+    shot 03-failed 0
+    echo "::error::The Manager reported a failure"
+    exit 1
+  fi
+  sleep 2
+done
+shot 03-confirm 0
+tap "(Install|INSTALL|Update|UPDATE)"
+
+for i in $(seq 1 30); do
+  if adb shell pm list packages "$target" | grep -q "$target"; then break; fi
+  sleep 2
+done
+shot 04-installed 3
+
+got_code=$(adb shell dumpsys package "$target" | grep -oE 'versionCode=[0-9]+' | head -1 | cut -d= -f2)
+if [ "$got_code" != "$want_code" ]; then
+  echo "::error::Inbox installed at version code ${got_code:-none}, but the manifest says $want_code"
+  exit 1
+fi
+expect "Up to date" "after installing"
+echo "Manager installed Inbox $got_code, matching the manifest"
+
+adb logcat -d > "$out/logcat.txt"
+if grep -q "FATAL EXCEPTION" "$out/logcat.txt"; then
+  echo "::error::An app crashed"
+  grep -A 30 "FATAL EXCEPTION" "$out/logcat.txt"
+  exit 1
+fi