diff --git a/internal/cli/build-minirootfs.go b/internal/cli/build-minirootfs.go index 0429b7b96..b3ca2f173 100644 --- a/internal/cli/build-minirootfs.go +++ b/internal/cli/build-minirootfs.go @@ -40,6 +40,7 @@ func buildMinirootFS() *cobra.Command { var extraRepos []string var extraPackages []string var sizeLimits options.SizeLimits + var compression string cmd := &cobra.Command{ Use: "build-minirootfs", @@ -60,6 +61,7 @@ func buildMinirootFS() *cobra.Command { build.WithArch(types.ParseArchitecture(buildArch)), build.WithIgnoreSignatures(ignoreSignatures), build.WithSizeLimits(sizeLimits), + build.WithCompression(compression), ) }, } @@ -68,6 +70,7 @@ func buildMinirootFS() *cobra.Command { cmd.Flags().StringVar(&buildArch, "build-arch", runtime.GOARCH, "architecture to build for -- default is Go runtime architecture") cmd.Flags().StringVar(&sbomPath, "sbom-path", "", "generate an SBOM") cmd.Flags().BoolVar(&ignoreSignatures, "ignore-signatures", false, "ignore repository signature verification") + cmd.Flags().StringVar(&compression, "compression", "gzip", "compression algorithm to use for layers (gzip or zstd)") cmd.Flags().StringSliceVarP(&extraKeys, "keyring-append", "k", []string{}, "path to extra keys to include in the keyring") cmd.Flags().StringSliceVarP(&extraBuildRepos, "build-repository-append", "b", []string{}, "path to extra repositories to include") cmd.Flags().StringSliceVarP(&extraRepos, "repository-append", "r", []string{}, "path to extra repositories to include") diff --git a/internal/cli/build.go b/internal/cli/build.go index 946623846..70cbad1fb 100644 --- a/internal/cli/build.go +++ b/internal/cli/build.go @@ -60,6 +60,7 @@ func buildCmd() *cobra.Command { var includePaths []string var ignoreSignatures bool var sizeLimits options.SizeLimits + var compression string cmd := &cobra.Command{ Use: "build", @@ -119,6 +120,7 @@ Along the image, apko will generate SBOMs (software bill of materials) describin build.WithIncludePaths(includePaths), build.WithIgnoreSignatures(ignoreSignatures), build.WithSizeLimits(sizeLimits), + build.WithCompression(compression), ) }, } @@ -139,6 +141,7 @@ Along the image, apko will generate SBOMs (software bill of materials) describin cmd.Flags().StringVar(&lockfile, "lockfile", "", "a path to .lock.json file (e.g. produced by apko lock) that constraints versions of packages to the listed ones (default '' means no additional constraints)") cmd.Flags().StringSliceVar(&includePaths, "include-paths", []string{}, "Additional include paths where to look for input files (config, base image, etc.). By default apko will search for paths only in workdir. Include paths may be absolute, or relative. Relative paths are interpreted relative to workdir. For adding extra paths for packages, use --repository-append.") cmd.Flags().BoolVar(&ignoreSignatures, "ignore-signatures", false, "ignore repository signature verification") + cmd.Flags().StringVar(&compression, "compression", "gzip", "compression algorithm to use for layers (gzip or zstd)") addClientLimitFlags(cmd, &sizeLimits) return cmd } diff --git a/internal/cli/publish.go b/internal/cli/publish.go index 71cbd0ee2..36a6518b1 100644 --- a/internal/cli/publish.go +++ b/internal/cli/publish.go @@ -56,6 +56,7 @@ func publish() *cobra.Command { var offline bool var lockfile string var ignoreSignatures bool + var compression string cmd := &cobra.Command{ Use: "publish ", @@ -122,6 +123,7 @@ in a keychain.`, build.WithLockFile(lockfile), build.WithTempDir(tmp), build.WithIgnoreSignatures(ignoreSignatures), + build.WithCompression(compression), }, []PublishOption{ // these are extra here just for publish; everything before is the same for BuildCmd as PublishCmd @@ -150,6 +152,7 @@ in a keychain.`, cmd.Flags().BoolVar(&offline, "offline", false, "do not use network to fetch packages (cache must be pre-populated)") cmd.Flags().StringVar(&lockfile, "lockfile", "", "a path to .lock.json file (e.g. produced by apko lock) that constraints versions of packages to the listed ones (default '' means no additional constraints)") cmd.Flags().BoolVar(&ignoreSignatures, "ignore-signatures", false, "ignore repository signature verification") + cmd.Flags().StringVar(&compression, "compression", "gzip", "compression algorithm to use for layers (gzip or zstd)") // these are extra here just for publish; everything before is the same for BuildCmd as PublishCmd cmd.Flags().BoolVar(&local, "local", false, "publish image just to local Docker daemon") diff --git a/pkg/build/build.go b/pkg/build/build.go index aa6c41afa..8b5f071b0 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -32,6 +32,7 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" v1types "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/klauspost/compress/zstd" "go.opentelemetry.io/otel" "gopkg.in/yaml.v3" @@ -187,7 +188,7 @@ func (bc *Context) ImageLayoutToLayer(ctx context.Context) (string, v1.Layer, er bc.o.TarballPath = outfile.Name() defer outfile.Close() - lw := newLayerWriter(outfile) + lw := newLayerWriter(outfile, bc.o.Compression) if err := writeTar(ctx, lw.w, bc.fs); err != nil { return "", nil, fmt.Errorf("generating tarball: %w", err) @@ -361,6 +362,7 @@ type layer struct { mu sync.Mutex uncompressed string compressed string + compression options.Compression diffid *v1.Hash desc *v1.Descriptor } @@ -379,7 +381,9 @@ func (l *layer) compress() error { } defer in.Close() - out, err := os.Create(l.uncompressed + ".gz") + ext := l.compression.Extension() + + out, err := os.Create(l.uncompressed + ext) if err != nil { return err } @@ -388,23 +392,39 @@ func (l *layer) compress() error { defer bufioPool.Put(buf) digest := sha256.New() - gzw := pooledGzipWriter(io.MultiWriter(digest, buf)) - defer pgzipPool.Put(gzw) + var cw io.WriteCloser + if l.compression == options.Zstd { + zw, err := zstd.NewWriter(io.MultiWriter(digest, buf), zstd.WithEncoderConcurrency(pgzipThreads)) + if err != nil { + out.Close() + return err + } + cw = zw + } else { + gzw := pooledGzipWriter(io.MultiWriter(digest, buf)) + defer pgzipPool.Put(gzw) + cw = gzw + } - if _, err := io.Copy(gzw, in); err != nil { + if _, err := io.Copy(cw, in); err != nil { + cw.Close() + out.Close() return err } - if err := gzw.Close(); err != nil { - return fmt.Errorf("closing gzip writer: %w", err) + if err := cw.Close(); err != nil { + out.Close() + return fmt.Errorf("closing compressor: %w", err) } if err := buf.Flush(); err != nil { + out.Close() return fmt.Errorf("flushing %s: %w", out.Name(), err) } stat, err := out.Stat() if err != nil { + out.Close() return fmt.Errorf("statting %s: %w", out.Name(), err) } @@ -420,7 +440,7 @@ func (l *layer) compress() error { descCopy := *l.desc compressionCache.Store(l.diffid.String(), &descCopy) - l.compressed = l.uncompressed + ".gz" + l.compressed = l.uncompressed + ext return out.Close() } diff --git a/pkg/build/build_implementation.go b/pkg/build/build_implementation.go index f301e654b..5e6c3b058 100644 --- a/pkg/build/build_implementation.go +++ b/pkg/build/build_implementation.go @@ -88,10 +88,10 @@ type layerWriter struct { finalize func() (*layer, error) } -// newLayerWriter wraps a file with a gzipping tar writer that computes +// newLayerWriter wraps a file with a tar writer that computes // everything we need to know to implement a v1.Layer, which it will // produce when finalize() is called. -func newLayerWriter(out *os.File) *layerWriter { +func newLayerWriter(out *os.File, compression options.Compression) *layerWriter { diffid := sha256.New() buf := pooledBufioWriter(out) @@ -114,10 +114,18 @@ func newLayerWriter(out *os.File) *layerWriter { return nil, fmt.Errorf("flushing %s: %w", out.Name(), err) } + var mediaType v1types.MediaType + if compression == options.Zstd { + mediaType = v1types.OCILayerZStd + } else { + mediaType = v1types.OCILayer + } + l := &layer{ uncompressed: out.Name(), + compression: compression, desc: &v1.Descriptor{ - MediaType: v1types.OCILayer, + MediaType: mediaType, }, diffid: &v1.Hash{ Algorithm: "sha256", diff --git a/pkg/build/build_test.go b/pkg/build/build_test.go index 79ccfa5f5..ff1eac0f5 100644 --- a/pkg/build/build_test.go +++ b/pkg/build/build_test.go @@ -19,6 +19,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -27,6 +28,7 @@ import ( "sync/atomic" "testing" + "github.com/klauspost/compress/zstd" "github.com/stretchr/testify/require" "chainguard.dev/apko/pkg/apk/apk" @@ -437,3 +439,36 @@ func TestAuth_bad(t *testing.T) { require.Error(t, err, "build should have failed to init keyring") require.True(t, called) } + +func TestBuildImageWithZstd(t *testing.T) { + ctx := context.Background() + + opts := []build.Option{ + build.WithConfig("apko.yaml", []string{"testdata"}), + build.WithCompression("zstd"), + } + + bc, err := build.New(ctx, fs.NewMemFS(), opts...) + require.NoError(t, err) + + err = bc.BuildImage(ctx) + require.NoError(t, err) + + _, layer, err := bc.ImageLayoutToLayer(ctx) + require.NoError(t, err) + + mediaType, err := layer.MediaType() + require.NoError(t, err) + require.Equal(t, "application/vnd.oci.image.layer.v1.tar+zstd", string(mediaType)) + + rc, err := layer.Compressed() + require.NoError(t, err) + defer rc.Close() + + decoder, err := zstd.NewReader(rc) + require.NoError(t, err) + defer decoder.Close() + + _, err = io.Copy(io.Discard, decoder) + require.NoError(t, err, "failed to decode zstd compressed layer") +} diff --git a/pkg/build/layers.go b/pkg/build/layers.go index 91c2692f9..353c3650e 100644 --- a/pkg/build/layers.go +++ b/pkg/build/layers.go @@ -85,7 +85,7 @@ func (bc *Context) buildLayers(ctx context.Context) ([]v1.Layer, error) { } // Then partition that single fs.FS into multiple layers based on our layering strategy. - return splitLayers(ctx, bc.fs, groups, pkgToDiff, bc.o.TempDir()) + return bc.splitLayers(ctx, bc.fs, groups, pkgToDiff, bc.o.TempDir()) } func replacesGroup(rep string, g *group) (bool, error) { @@ -253,21 +253,26 @@ func merge(groups ...*group) *group { return merged } -func splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToDiff map[*apk.Package][]byte, tmpdir string) ([]v1.Layer, error) { +func (bc *Context) splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToDiff map[*apk.Package][]byte, tmpdir string) ([]v1.Layer, error) { buf := make([]byte, 1<<20) + ext := "*.tar.gz" + if bc.o.Compression == "zstd" { + ext = "*.tar.zst" + } + // We'll create a writer for each layer and a map to quickly access the writer given a package or group. packageToWriter := map[string]*layerWriter{} groupToWriter := map[*group]*layerWriter{} for _, g := range groups { - f, err := os.CreateTemp(tmpdir, "layer-*.tar.gz") + f, err := os.CreateTemp(tmpdir, "layer-"+ext) if err != nil { return nil, err } defer f.Close() - w := newLayerWriter(f) + w := newLayerWriter(f, bc.o.Compression) groupToWriter[g] = w for _, pkg := range g.pkgs { @@ -276,13 +281,13 @@ func splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToD } // The top layer holds anything that doesn't belong to a package. - f, err := os.CreateTemp(tmpdir, "layer-*.tar.gz") + f, err := os.CreateTemp(tmpdir, "layer-"+ext) if err != nil { return nil, err } defer f.Close() - top := newLayerWriter(f) + top := newLayerWriter(f, bc.o.Compression) // In a tar file, it is customary to include directories before files in those directories. // In order to know which directories we need to include, we maintain a directory stack for each layer. diff --git a/pkg/build/layers_test.go b/pkg/build/layers_test.go index bd5d00243..bcf78d2b3 100644 --- a/pkg/build/layers_test.go +++ b/pkg/build/layers_test.go @@ -280,7 +280,8 @@ func TestSplitLayersDirectoryCreation(t *testing.T) { // Call splitLayers to create the layers ctx := context.Background() - layers, err := splitLayers(ctx, fsys, groups, pkgToDiff, tmpDir) + bc := &Context{} + layers, err := bc.splitLayers(ctx, fsys, groups, pkgToDiff, tmpDir) if err != nil { t.Fatalf("splitLayers failed: %v", err) } diff --git a/pkg/build/options.go b/pkg/build/options.go index 4b1f1406c..6d7b8c1f2 100644 --- a/pkg/build/options.go +++ b/pkg/build/options.go @@ -286,3 +286,15 @@ func WithSizeLimits(limits options.SizeLimits) Option { return nil } } + +// WithCompression sets the compression algorithm for building layers. +func WithCompression(compression string) Option { + return func(bc *Context) error { + comp := options.Compression(compression) + if !comp.IsValid() { + return fmt.Errorf("invalid compression algorithm %q (supported: gzip, zstd)", compression) + } + bc.o.Compression = comp + return nil + } +} diff --git a/pkg/options/compression.go b/pkg/options/compression.go new file mode 100644 index 000000000..b8a10b0e6 --- /dev/null +++ b/pkg/options/compression.go @@ -0,0 +1,47 @@ +// Copyright 2026 Chainguard, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +// Compression represents the compression algorithm used for image layers. +type Compression string + +const ( + // Gzip is the gzip compression algorithm. + Gzip Compression = "gzip" + // Zstd is the zstd compression algorithm. + Zstd Compression = "zstd" +) + +// IsValid returns true if the compression algorithm is supported. +func (c Compression) IsValid() bool { + switch c { + case Gzip, Zstd: + return true + default: + return false + } +} + +// Extension returns the file extension associated with the compression algorithm. +func (c Compression) Extension() string { + switch c { + case Zstd: + return ".zst" + case Gzip: + return ".gz" + default: + return ".gz" + } +} diff --git a/pkg/options/options.go b/pkg/options/options.go index 5866f0ddf..55ba1f210 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -86,6 +86,7 @@ type Options struct { Transport http.RoundTripper `json:"-"` PackageGetter apk.PackageGetter `json:"-"` SizeLimits SizeLimits `json:"sizeLimits,omitempty"` + Compression Compression `json:"compression,omitempty"` } type Auth struct{ User, Pass string } @@ -97,6 +98,7 @@ var Default = Options{ DiskCacheEnabled: true, SharedCache: apk.NewCache(false), SizeLimits: DefaultSizeLimits(), + Compression: Gzip, } // Tempdir returns the temporary directory where apko will create @@ -116,9 +118,9 @@ func (o *Options) TempDir() string { // TarballFileName returns a deterministic filename for the layer taball func (o Options) TarballFileName() string { - tarName := "apko.tar.gz" + ext := "tar" + o.Compression.Extension() if o.Arch.String() != "" { - tarName = fmt.Sprintf("apko-%s.tar.gz", o.Arch.ToAPK()) + return fmt.Sprintf("apko-%s.%s", o.Arch.ToAPK(), ext) } - return tarName + return fmt.Sprintf("apko.%s", ext) }