Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions internal/cli/build-minirootfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ func buildMinirootFS() *cobra.Command {
var extraRepos []string
var extraPackages []string
var sizeLimits options.SizeLimits
var compression string

cmd := &cobra.Command{
Use: "build-minirootfs",
Expand All @@ -60,6 +61,7 @@ func buildMinirootFS() *cobra.Command {
build.WithArch(types.ParseArchitecture(buildArch)),
build.WithIgnoreSignatures(ignoreSignatures),
build.WithSizeLimits(sizeLimits),
build.WithCompression(compression),
)
},
}
Expand All @@ -68,6 +70,7 @@ func buildMinirootFS() *cobra.Command {
cmd.Flags().StringVar(&buildArch, "build-arch", runtime.GOARCH, "architecture to build for -- default is Go runtime architecture")
cmd.Flags().StringVar(&sbomPath, "sbom-path", "", "generate an SBOM")
cmd.Flags().BoolVar(&ignoreSignatures, "ignore-signatures", false, "ignore repository signature verification")
cmd.Flags().StringVar(&compression, "compression", "gzip", "compression algorithm to use for layers (gzip or zstd)")
cmd.Flags().StringSliceVarP(&extraKeys, "keyring-append", "k", []string{}, "path to extra keys to include in the keyring")
cmd.Flags().StringSliceVarP(&extraBuildRepos, "build-repository-append", "b", []string{}, "path to extra repositories to include")
cmd.Flags().StringSliceVarP(&extraRepos, "repository-append", "r", []string{}, "path to extra repositories to include")
Expand Down
3 changes: 3 additions & 0 deletions internal/cli/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ func buildCmd() *cobra.Command {
var includePaths []string
var ignoreSignatures bool
var sizeLimits options.SizeLimits
var compression string

cmd := &cobra.Command{
Use: "build",
Expand Down Expand Up @@ -119,6 +120,7 @@ Along the image, apko will generate SBOMs (software bill of materials) describin
build.WithIncludePaths(includePaths),
build.WithIgnoreSignatures(ignoreSignatures),
build.WithSizeLimits(sizeLimits),
build.WithCompression(compression),
)
},
}
Expand All @@ -139,6 +141,7 @@ Along the image, apko will generate SBOMs (software bill of materials) describin
cmd.Flags().StringVar(&lockfile, "lockfile", "", "a path to .lock.json file (e.g. produced by apko lock) that constraints versions of packages to the listed ones (default '' means no additional constraints)")
cmd.Flags().StringSliceVar(&includePaths, "include-paths", []string{}, "Additional include paths where to look for input files (config, base image, etc.). By default apko will search for paths only in workdir. Include paths may be absolute, or relative. Relative paths are interpreted relative to workdir. For adding extra paths for packages, use --repository-append.")
cmd.Flags().BoolVar(&ignoreSignatures, "ignore-signatures", false, "ignore repository signature verification")
cmd.Flags().StringVar(&compression, "compression", "gzip", "compression algorithm to use for layers (gzip or zstd)")
addClientLimitFlags(cmd, &sizeLimits)
return cmd
}
Expand Down
3 changes: 3 additions & 0 deletions internal/cli/publish.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ func publish() *cobra.Command {
var offline bool
var lockfile string
var ignoreSignatures bool
var compression string

cmd := &cobra.Command{
Use: "publish <config.yaml> <tag...>",
Expand Down Expand Up @@ -122,6 +123,7 @@ in a keychain.`,
build.WithLockFile(lockfile),
build.WithTempDir(tmp),
build.WithIgnoreSignatures(ignoreSignatures),
build.WithCompression(compression),
},
[]PublishOption{
// these are extra here just for publish; everything before is the same for BuildCmd as PublishCmd
Expand Down Expand Up @@ -150,6 +152,7 @@ in a keychain.`,
cmd.Flags().BoolVar(&offline, "offline", false, "do not use network to fetch packages (cache must be pre-populated)")
cmd.Flags().StringVar(&lockfile, "lockfile", "", "a path to .lock.json file (e.g. produced by apko lock) that constraints versions of packages to the listed ones (default '' means no additional constraints)")
cmd.Flags().BoolVar(&ignoreSignatures, "ignore-signatures", false, "ignore repository signature verification")
cmd.Flags().StringVar(&compression, "compression", "gzip", "compression algorithm to use for layers (gzip or zstd)")

// these are extra here just for publish; everything before is the same for BuildCmd as PublishCmd
cmd.Flags().BoolVar(&local, "local", false, "publish image just to local Docker daemon")
Expand Down
36 changes: 28 additions & 8 deletions pkg/build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
v1types "github.com/google/go-containerregistry/pkg/v1/types"
"github.com/klauspost/compress/zstd"
"go.opentelemetry.io/otel"
"gopkg.in/yaml.v3"

Expand Down Expand Up @@ -187,7 +188,7 @@ func (bc *Context) ImageLayoutToLayer(ctx context.Context) (string, v1.Layer, er
bc.o.TarballPath = outfile.Name()
defer outfile.Close()

lw := newLayerWriter(outfile)
lw := newLayerWriter(outfile, bc.o.Compression)

if err := writeTar(ctx, lw.w, bc.fs); err != nil {
return "", nil, fmt.Errorf("generating tarball: %w", err)
Expand Down Expand Up @@ -361,6 +362,7 @@ type layer struct {
mu sync.Mutex
uncompressed string
compressed string
compression options.Compression
diffid *v1.Hash
desc *v1.Descriptor
}
Expand All @@ -379,7 +381,9 @@ func (l *layer) compress() error {
}
defer in.Close()

out, err := os.Create(l.uncompressed + ".gz")
ext := l.compression.Extension()

out, err := os.Create(l.uncompressed + ext)
if err != nil {
return err
}
Expand All @@ -388,23 +392,39 @@ func (l *layer) compress() error {
defer bufioPool.Put(buf)

digest := sha256.New()
gzw := pooledGzipWriter(io.MultiWriter(digest, buf))
defer pgzipPool.Put(gzw)
var cw io.WriteCloser
if l.compression == options.Zstd {
zw, err := zstd.NewWriter(io.MultiWriter(digest, buf), zstd.WithEncoderConcurrency(pgzipThreads))
if err != nil {
out.Close()
return err
}
cw = zw
} else {
gzw := pooledGzipWriter(io.MultiWriter(digest, buf))
defer pgzipPool.Put(gzw)
cw = gzw
}

if _, err := io.Copy(gzw, in); err != nil {
if _, err := io.Copy(cw, in); err != nil {
cw.Close()
out.Close()
return err
}

if err := gzw.Close(); err != nil {
return fmt.Errorf("closing gzip writer: %w", err)
if err := cw.Close(); err != nil {
out.Close()
return fmt.Errorf("closing compressor: %w", err)
}

if err := buf.Flush(); err != nil {
out.Close()
return fmt.Errorf("flushing %s: %w", out.Name(), err)
}

stat, err := out.Stat()
if err != nil {
out.Close()
return fmt.Errorf("statting %s: %w", out.Name(), err)
}

Expand All @@ -420,7 +440,7 @@ func (l *layer) compress() error {
descCopy := *l.desc
compressionCache.Store(l.diffid.String(), &descCopy)

l.compressed = l.uncompressed + ".gz"
l.compressed = l.uncompressed + ext

return out.Close()
}
Expand Down
14 changes: 11 additions & 3 deletions pkg/build/build_implementation.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,10 @@ type layerWriter struct {
finalize func() (*layer, error)
}

// newLayerWriter wraps a file with a gzipping tar writer that computes
// newLayerWriter wraps a file with a tar writer that computes
// everything we need to know to implement a v1.Layer, which it will
// produce when finalize() is called.
func newLayerWriter(out *os.File) *layerWriter {
func newLayerWriter(out *os.File, compression options.Compression) *layerWriter {
diffid := sha256.New()

buf := pooledBufioWriter(out)
Expand All @@ -114,10 +114,18 @@ func newLayerWriter(out *os.File) *layerWriter {
return nil, fmt.Errorf("flushing %s: %w", out.Name(), err)
}

var mediaType v1types.MediaType
if compression == options.Zstd {
mediaType = v1types.OCILayerZStd
} else {
mediaType = v1types.OCILayer
}

l := &layer{
uncompressed: out.Name(),
compression: compression,
desc: &v1.Descriptor{
MediaType: v1types.OCILayer,
MediaType: mediaType,
},
diffid: &v1.Hash{
Algorithm: "sha256",
Expand Down
35 changes: 35 additions & 0 deletions pkg/build/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
Expand All @@ -27,6 +28,7 @@ import (
"sync/atomic"
"testing"

"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/require"

"chainguard.dev/apko/pkg/apk/apk"
Expand Down Expand Up @@ -437,3 +439,36 @@ func TestAuth_bad(t *testing.T) {
require.Error(t, err, "build should have failed to init keyring")
require.True(t, called)
}

func TestBuildImageWithZstd(t *testing.T) {
ctx := context.Background()

opts := []build.Option{
build.WithConfig("apko.yaml", []string{"testdata"}),
build.WithCompression("zstd"),
}

bc, err := build.New(ctx, fs.NewMemFS(), opts...)
require.NoError(t, err)

err = bc.BuildImage(ctx)
require.NoError(t, err)

_, layer, err := bc.ImageLayoutToLayer(ctx)
require.NoError(t, err)

mediaType, err := layer.MediaType()
require.NoError(t, err)
require.Equal(t, "application/vnd.oci.image.layer.v1.tar+zstd", string(mediaType))

rc, err := layer.Compressed()
require.NoError(t, err)
defer rc.Close()

decoder, err := zstd.NewReader(rc)
require.NoError(t, err)
defer decoder.Close()

_, err = io.Copy(io.Discard, decoder)
require.NoError(t, err, "failed to decode zstd compressed layer")
}
17 changes: 11 additions & 6 deletions pkg/build/layers.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func (bc *Context) buildLayers(ctx context.Context) ([]v1.Layer, error) {
}

// Then partition that single fs.FS into multiple layers based on our layering strategy.
return splitLayers(ctx, bc.fs, groups, pkgToDiff, bc.o.TempDir())
return bc.splitLayers(ctx, bc.fs, groups, pkgToDiff, bc.o.TempDir())
}

func replacesGroup(rep string, g *group) (bool, error) {
Expand Down Expand Up @@ -253,21 +253,26 @@ func merge(groups ...*group) *group {
return merged
}

func splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToDiff map[*apk.Package][]byte, tmpdir string) ([]v1.Layer, error) {
func (bc *Context) splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToDiff map[*apk.Package][]byte, tmpdir string) ([]v1.Layer, error) {
buf := make([]byte, 1<<20)

ext := "*.tar.gz"
if bc.o.Compression == "zstd" {
ext = "*.tar.zst"
}

// We'll create a writer for each layer and a map to quickly access the writer given a package or group.
packageToWriter := map[string]*layerWriter{}
groupToWriter := map[*group]*layerWriter{}

for _, g := range groups {
f, err := os.CreateTemp(tmpdir, "layer-*.tar.gz")
f, err := os.CreateTemp(tmpdir, "layer-"+ext)
if err != nil {
return nil, err
}
defer f.Close()

w := newLayerWriter(f)
w := newLayerWriter(f, bc.o.Compression)
groupToWriter[g] = w

for _, pkg := range g.pkgs {
Expand All @@ -276,13 +281,13 @@ func splitLayers(ctx context.Context, fsys apkfs.FullFS, groups []*group, pkgToD
}

// The top layer holds anything that doesn't belong to a package.
f, err := os.CreateTemp(tmpdir, "layer-*.tar.gz")
f, err := os.CreateTemp(tmpdir, "layer-"+ext)
if err != nil {
return nil, err
}
defer f.Close()

top := newLayerWriter(f)
top := newLayerWriter(f, bc.o.Compression)

// In a tar file, it is customary to include directories before files in those directories.
// In order to know which directories we need to include, we maintain a directory stack for each layer.
Expand Down
3 changes: 2 additions & 1 deletion pkg/build/layers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,8 @@ func TestSplitLayersDirectoryCreation(t *testing.T) {

// Call splitLayers to create the layers
ctx := context.Background()
layers, err := splitLayers(ctx, fsys, groups, pkgToDiff, tmpDir)
bc := &Context{}
layers, err := bc.splitLayers(ctx, fsys, groups, pkgToDiff, tmpDir)
if err != nil {
t.Fatalf("splitLayers failed: %v", err)
}
Expand Down
12 changes: 12 additions & 0 deletions pkg/build/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,3 +286,15 @@ func WithSizeLimits(limits options.SizeLimits) Option {
return nil
}
}

// WithCompression sets the compression algorithm for building layers.
func WithCompression(compression string) Option {
return func(bc *Context) error {
comp := options.Compression(compression)
if !comp.IsValid() {
return fmt.Errorf("invalid compression algorithm %q (supported: gzip, zstd)", compression)
}
bc.o.Compression = comp
return nil
}
}
47 changes: 47 additions & 0 deletions pkg/options/compression.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2026 Chainguard, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package options

// Compression represents the compression algorithm used for image layers.
type Compression string

const (
// Gzip is the gzip compression algorithm.
Gzip Compression = "gzip"
// Zstd is the zstd compression algorithm.
Zstd Compression = "zstd"
)

// IsValid returns true if the compression algorithm is supported.
func (c Compression) IsValid() bool {
switch c {
case Gzip, Zstd:
return true
default:
return false
}
}

// Extension returns the file extension associated with the compression algorithm.
func (c Compression) Extension() string {
switch c {
case Zstd:
return ".zst"
case Gzip:
return ".gz"
default:
return ".gz"
}
}
Loading