Skip to content

Commit 030bef8

Browse files
committed
build/bake: support configurable image layer compression
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
1 parent 1037c7b commit 030bef8

7 files changed

Lines changed: 267 additions & 51 deletions

File tree

‎.github/workflows/.test-bake.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -750,3 +750,86 @@ jobs:
750750
username: ${{ vars.DOCKERHUB_OIDC_USERNAME }}
751751
connection_id: ${{ vars.DOCKERHUB_OIDC_CONNECTIONID }}
752752
scope: 'dhi.io@pull'
753+
754+
bake-compression:
755+
strategy:
756+
matrix:
757+
include:
758+
- variant: default
759+
compression: ''
760+
level: ''
761+
force: false
762+
- variant: zstd
763+
compression: zstd
764+
level: '0'
765+
force: false
766+
- variant: zstd-force
767+
compression: zstd
768+
level: '9'
769+
force: true
770+
uses: ./.github/workflows/bake.yml
771+
permissions:
772+
contents: read
773+
id-token: write
774+
packages: write
775+
with:
776+
compression: ${{ matrix.compression }}
777+
compression-level: ${{ matrix.level }}
778+
force-compression: ${{ matrix.force }}
779+
output: image
780+
push: ${{ github.event_name != 'pull_request' }}
781+
context: test
782+
target: compression
783+
meta-images: ghcr.io/docker/github-builder-test
784+
meta-tags: |
785+
type=raw,value=bake-compression-${{ matrix.variant }}-${{ github.run_id }}-${{ github.run_attempt }}
786+
secrets:
787+
registry-auths: |
788+
- registry: ghcr.io
789+
username: ${{ github.actor }}
790+
password: ${{ secrets.GITHUB_TOKEN }}
791+
792+
bake-compression-check:
793+
runs-on: ubuntu-26.04
794+
if: ${{ github.event_name != 'pull_request' }}
795+
needs:
796+
- bake-compression
797+
strategy:
798+
matrix:
799+
include:
800+
- variant: default
801+
base: gzip
802+
added: gzip
803+
- variant: zstd
804+
base: gzip
805+
added: zstd
806+
- variant: zstd-force
807+
base: zstd
808+
added: zstd
809+
steps:
810+
-
811+
name: Check image layer compression
812+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
813+
env:
814+
INPUT_IMAGE: ghcr.io/docker/github-builder-test
815+
INPUT_TAG: bake-compression-${{ matrix.variant }}-${{ github.run_id }}-${{ github.run_attempt }}
816+
INPUT_BASE-COMPRESSION: ${{ matrix.base }}
817+
INPUT_ADDED-COMPRESSION: ${{ matrix.added }}
818+
with:
819+
script: |
820+
const assert = require('node:assert/strict');
821+
const image = core.getInput('image');
822+
const inspect = async ref => {
823+
const {stdout} = await exec.getExecOutput('docker', ['buildx', 'imagetools', 'inspect', '--raw', ref], {silent: true});
824+
return JSON.parse(stdout);
825+
};
826+
const index = await inspect(`${image}:${core.getInput('tag')}`);
827+
const manifests = index.manifests.filter(manifest => manifest.platform?.os === 'linux');
828+
assert.deepEqual(manifests.map(manifest => manifest.platform.architecture).sort(), ['amd64', 'arm64']);
829+
for (const {digest} of manifests) {
830+
const {layers} = await inspect(`${image}@${digest}`);
831+
assert.ok(layers.length >= 2, 'Expected base image layers and the fixture layer');
832+
const expected = layers.map((_, index) => core.getInput(index === layers.length - 1 ? 'added-compression' : 'base-compression'));
833+
const actual = layers.map(layer => layer.mediaType.split('+').pop());
834+
assert.deepEqual(actual, expected, `Unexpected layer compression for ${image}@${digest}`);
835+
}

‎.github/workflows/.test-build.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -759,3 +759,86 @@ jobs:
759759
username: ${{ vars.DOCKERHUB_OIDC_USERNAME }}
760760
connection_id: ${{ vars.DOCKERHUB_OIDC_CONNECTIONID }}
761761
scope: 'dhi.io@pull'
762+
763+
build-compression:
764+
strategy:
765+
matrix:
766+
include:
767+
- variant: default
768+
compression: ''
769+
level: ''
770+
force: false
771+
- variant: zstd
772+
compression: zstd
773+
level: '0'
774+
force: false
775+
- variant: zstd-force
776+
compression: zstd
777+
level: '9'
778+
force: true
779+
uses: ./.github/workflows/build.yml
780+
permissions:
781+
contents: read
782+
id-token: write
783+
packages: write
784+
with:
785+
compression: ${{ matrix.compression }}
786+
compression-level: ${{ matrix.level }}
787+
force-compression: ${{ matrix.force }}
788+
output: image
789+
push: ${{ github.event_name != 'pull_request' }}
790+
file: test/compression.Dockerfile
791+
platforms: linux/amd64,linux/arm64
792+
meta-images: ghcr.io/docker/github-builder-test
793+
meta-tags: |
794+
type=raw,value=build-compression-${{ matrix.variant }}-${{ github.run_id }}-${{ github.run_attempt }}
795+
secrets:
796+
registry-auths: |
797+
- registry: ghcr.io
798+
username: ${{ github.actor }}
799+
password: ${{ secrets.GITHUB_TOKEN }}
800+
801+
build-compression-check:
802+
runs-on: ubuntu-26.04
803+
if: ${{ github.event_name != 'pull_request' }}
804+
needs:
805+
- build-compression
806+
strategy:
807+
matrix:
808+
include:
809+
- variant: default
810+
base: gzip
811+
added: gzip
812+
- variant: zstd
813+
base: gzip
814+
added: zstd
815+
- variant: zstd-force
816+
base: zstd
817+
added: zstd
818+
steps:
819+
-
820+
name: Check image layer compression
821+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
822+
env:
823+
INPUT_IMAGE: ghcr.io/docker/github-builder-test
824+
INPUT_TAG: build-compression-${{ matrix.variant }}-${{ github.run_id }}-${{ github.run_attempt }}
825+
INPUT_BASE-COMPRESSION: ${{ matrix.base }}
826+
INPUT_ADDED-COMPRESSION: ${{ matrix.added }}
827+
with:
828+
script: |
829+
const assert = require('node:assert/strict');
830+
const image = core.getInput('image');
831+
const inspect = async ref => {
832+
const {stdout} = await exec.getExecOutput('docker', ['buildx', 'imagetools', 'inspect', '--raw', ref], {silent: true});
833+
return JSON.parse(stdout);
834+
};
835+
const index = await inspect(`${image}:${core.getInput('tag')}`);
836+
const manifests = index.manifests.filter(manifest => manifest.platform?.os === 'linux');
837+
assert.deepEqual(manifests.map(manifest => manifest.platform.architecture).sort(), ['amd64', 'arm64']);
838+
for (const {digest} of manifests) {
839+
const {layers} = await inspect(`${image}@${digest}`);
840+
assert.ok(layers.length >= 2, 'Expected base image layers and the fixture layer');
841+
const expected = layers.map((_, index) => core.getInput(index === layers.length - 1 ? 'added-compression' : 'base-compression'));
842+
const actual = layers.map(layer => layer.mediaType.split('+').pop());
843+
assert.deepEqual(actual, expected, `Unexpected layer compression for ${image}@${digest}`);
844+
}

‎.github/workflows/bake.yml‎

Lines changed: 44 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,19 @@ on:
5959
description: "Cache layers to export if cache enabled (min or max)"
6060
required: false
6161
default: 'min'
62+
compression:
63+
type: string
64+
description: "Image layer compression (uncompressed, gzip, estargz or zstd)"
65+
required: false
66+
compression-level:
67+
type: string
68+
description: "Image layer compression level (0-9 for gzip and estargz, 0-22 for zstd)"
69+
required: false
70+
force-compression:
71+
type: boolean
72+
description: "Recompress existing image layers using the selected compression type"
73+
required: false
74+
default: false
6275
context:
6376
type: string
6477
description: "Context to build from in the Git working tree"
@@ -219,6 +232,7 @@ jobs:
219232
include:
220233
- name: ${{ inputs.job-name-prefix }}
221234
outputs:
235+
buildOutput: ${{ steps.set.outputs.buildOutput }}
222236
includes: ${{ steps.set.outputs.includes }}
223237
metaImages: ${{ steps.set.outputs.metaImages }}
224238
sign: ${{ steps.set.outputs.sign }}
@@ -315,6 +329,10 @@ jobs:
315329
id: set
316330
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
317331
env:
332+
INPUT_LOCAL-EXPORT-DIR: ${{ env.LOCAL_EXPORT_DIR }}
333+
INPUT_COMPRESSION: ${{ inputs.compression }}
334+
INPUT_COMPRESSION-LEVEL: ${{ inputs.compression-level }}
335+
INPUT_FORCE-COMPRESSION: ${{ inputs.force-compression }}
318336
INPUT_SBOM-IMAGE: ${{ env.SBOM_IMAGE }}
319337
INPUT_MATRIX-SIZE-LIMIT: ${{ env.MATRIX_SIZE_LIMIT }}
320338
INPUT_META-IMAGES: ${{ inputs.meta-images }}
@@ -341,6 +359,7 @@ jobs:
341359
const { Build } = require('@docker/github-builder-runtime/lib/buildx/build');
342360
const { GitHub } = require('@docker/github-builder-runtime/lib/github/github');
343361
const { MetaImages } = require('@docker/github-builder-runtime/lib/github-builder/meta-images');
362+
const { BuildOutput } = require('@docker/github-builder-runtime/lib/github-builder/build-output');
344363
const { RunnerMapping } = require('@docker/github-builder-runtime/lib/github-builder/runner-mapping');
345364
const { BakeTargets } = require('@docker/github-builder-runtime/lib/github-builder/bake-targets');
346365
const { BakeVars } = require('@docker/github-builder-runtime/lib/github-builder/bake-vars');
@@ -364,6 +383,10 @@ jobs:
364383
const inpSbom = core.getBooleanInput('sbom');
365384
const inpSet = Util.getInputList('set', {ignoreComma: true, quote: false});
366385
const inpSign = core.getInput('sign');
386+
const inpCompression = core.getInput('compression');
387+
const inpCompressionLevel = core.getInput('compression-level');
388+
const inpForceCompression = core.getBooleanInput('force-compression');
389+
const inpLocalExportDir = core.getInput('local-export-dir');
367390
const inpTarget = core.getInput('target');
368391
const inpGitHubToken = core.getInput('github-token');
369392
const inpProxyNetwork = core.getBooleanInput('buildkit-proxy-network');
@@ -414,7 +437,26 @@ jobs:
414437
}).sort(), null, 2));
415438
});
416439
440+
if (inpCompressionLevel !== '' && (!/^\d+$/.test(inpCompressionLevel) || !Number.isSafeInteger(Number(inpCompressionLevel)))) {
441+
core.setFailed(`Invalid compression-level: ${inpCompressionLevel}`);
442+
return;
443+
}
444+
417445
const metaImages = MetaImages.resolve(inpMetaImages);
446+
const buildOutput = BuildOutput.resolve({
447+
output: inpOutput,
448+
images: metaImages,
449+
push: inpPush,
450+
localExportDir: inpLocalExportDir,
451+
compression: inpCompression || undefined,
452+
compressionLevel: inpCompressionLevel !== '' ? Number(inpCompressionLevel) : undefined,
453+
forceCompression: inpForceCompression || undefined
454+
});
455+
await core.group(`Set buildOutput output`, async () => {
456+
core.info(buildOutput);
457+
core.setOutput('buildOutput', buildOutput);
458+
});
459+
418460
await core.group(`Set metaImages output`, async () => {
419461
core.info(JSON.stringify(metaImages, null, 2));
420462
core.setOutput('metaImages', metaImages.join('\n'));
@@ -822,23 +864,20 @@ jobs:
822864
id: prepare
823865
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
824866
env:
867+
INPUT_BUILD-OUTPUT: ${{ needs.prepare.outputs.buildOutput }}
825868
INPUT_PLATFORM: ${{ matrix.platform }}
826869
INPUT_SBOM-IMAGE: ${{ env.SBOM_IMAGE }}
827-
INPUT_LOCAL-EXPORT-DIR: ${{ env.LOCAL_EXPORT_DIR }}
828870
INPUT_CACHE: ${{ inputs.cache }}
829871
INPUT_CACHE-SCOPE: ${{ inputs.cache-scope }}
830872
INPUT_CACHE-MODE: ${{ inputs.cache-mode }}
831873
INPUT_BUILD-SECRETS: ${{ secrets.build-secrets }}
832874
INPUT_TARGETS: ${{ needs.prepare.outputs.targets }}
833875
INPUT_CONTEXT: ${{ inputs.context }}
834876
INPUT_FILES: ${{ inputs.files }}
835-
INPUT_OUTPUT: ${{ inputs.output }}
836-
INPUT_PUSH: ${{ inputs.push }}
837877
INPUT_SBOM: ${{ inputs.sbom }}
838878
INPUT_SET: ${{ inputs.set }}
839879
INPUT_TARGET: ${{ inputs.target }}
840880
INPUT_VARS: ${{ inputs.vars }}
841-
INPUT_META-IMAGES: ${{ needs.prepare.outputs.metaImages }}
842881
INPUT_META-VERSION: ${{ steps.meta.outputs.version }}
843882
INPUT_META-TAGS: ${{ steps.meta.outputs.tags }}
844883
INPUT_SET-META-ANNOTATIONS: ${{ inputs.set-meta-annotations }}
@@ -861,7 +900,6 @@ jobs:
861900
core.setOutput('platform-pair-suffix', platformPairSuffix);
862901
863902
const inpSbomImage = core.getInput('sbom-image');
864-
const inpLocalExportDir = core.getInput('local-export-dir');
865903
866904
const inpCache = core.getBooleanInput('cache');
867905
const inpCacheScope = core.getInput('cache-scope');
@@ -870,15 +908,12 @@ jobs:
870908
const inpTargets = core.getInput('targets');
871909
const inpContext = core.getInput('context');
872910
const inpFiles = Util.getInputList('files');
873-
const inpOutput = core.getInput('output');
874-
const inpPush = core.getBooleanInput('push');
875911
const inpSbom = core.getBooleanInput('sbom');
876912
const inpSet = Util.getInputList('set', {ignoreComma: true, quote: false});
877913
const inpTarget = core.getInput('target');
878914
const inpVars = Util.getInputList('vars', {ignoreComma: true, quote: false});
879915
const inpBuildkitProxyNetwork = core.getBooleanInput('buildkit-proxy-network');
880916
881-
const inpMetaImages = core.getMultilineInput('meta-images');
882917
const inpMetaVersion = core.getInput('meta-version');
883918
const inpMetaTags = core.getMultilineInput('meta-tags');
884919
const inpSetMetaAnnotations = core.getBooleanInput('set-meta-annotations');
@@ -947,23 +982,6 @@ jobs:
947982
core.setOutput('files', bakeFiles.join(os.EOL));
948983
});
949984
950-
let outputOverride = '';
951-
switch (inpOutput) {
952-
case 'image':
953-
if (inpMetaImages.length == 0) {
954-
core.setFailed('meta-images is required when output is image');
955-
return;
956-
}
957-
outputOverride = `*.output=type=image,"name=${inpMetaImages.join(',')}",oci-artifact=true,push-by-digest=true,name-canonical=true,push=${inpPush}`;
958-
break;
959-
case 'local':
960-
outputOverride = `*.output=type=local,platform-split=true,dest=${inpLocalExportDir}`;
961-
break;
962-
default:
963-
core.setFailed(`Invalid output: ${inpOutput}`);
964-
return;
965-
}
966-
967985
let bakeOverrides = [];
968986
try {
969987
bakeOverrides = inpSet.map(override => renderTemplate(override));
@@ -972,7 +990,7 @@ jobs:
972990
return;
973991
}
974992
await core.group(`Set bake overrides`, async () => {
975-
bakeOverrides.push(outputOverride);
993+
bakeOverrides.push(`*.output=${core.getInput('build-output')}`);
976994
bakeOverrides.push('*.tags=');
977995
if (GitHub.context.payload.repository?.private ?? false) {
978996
// if this is a private repository, we set min provenance mode

0 commit comments

Comments
 (0)