diff --git a/build/build.go b/build/build.go index 27714f5848d6..31d2010017fa 100644 --- a/build/build.go +++ b/build/build.go @@ -1345,7 +1345,7 @@ func resultKey(node *noderesolver.ResolvedNode, name string) string { // and creates a separate session that will be used by all detected requests. func detectSharedMounts(ctx context.Context, reqs map[string][]*reqForNode) (_ map[string][]*session.Session, err error) { type fsTracker struct { - fs fsutil.FS + fs *fsMount so []*client.SolveOpt } type fsKey struct { @@ -1353,7 +1353,7 @@ func detectSharedMounts(ctx context.Context, reqs map[string][]*reqForNode) (_ m dir string } - m := map[string]map[fsKey]*fsTracker{} + m := map[string]map[fsKey][]*fsTracker{} for _, reqs := range reqs { for _, req := range reqs { nodeName := req.Node().Name @@ -1362,26 +1362,30 @@ func detectSharedMounts(ctx context.Context, reqs map[string][]*reqForNode) (_ m continue } if _, ok := m[nodeName]; !ok { - m[nodeName] = map[fsKey]*fsTracker{} + m[nodeName] = map[fsKey][]*fsTracker{} } fsMap := m[nodeName] - for name, m := range req.so.LocalMounts { - fs, ok := m.(*fsMount) + for name, mount := range req.so.LocalMounts { + fs, ok := mount.(*fsMount) if !ok { continue } key := fsKey{name: name, dir: fs.dir} - if _, ok := fsMap[key]; !ok { - fsMap[key] = &fsTracker{fs: fs.FS} + idx := slices.IndexFunc(fsMap[key], func(t *fsTracker) bool { + return slices.Equal(t.fs.patterns, fs.patterns) + }) + if idx == -1 { + fsMap[key] = append(fsMap[key], &fsTracker{fs: fs}) + idx = len(fsMap[key]) - 1 } - fsMap[key].so = append(fsMap[key].so, req.so) + fsMap[key][idx].so = append(fsMap[key][idx].so, req.so) } } } type sharedSession struct { *session.Session - fsMap map[string]fsutil.FS + fsMap map[string]*fsMount } sessionMap := map[string][]*sharedSession{} @@ -1397,38 +1401,40 @@ func detectSharedMounts(ctx context.Context, reqs map[string][]*reqForNode) (_ m }() for node, fsMap := range m { - for key, fs := range fsMap { - if len(fs.so) <= 1 { - continue - } + for key, trackers := range fsMap { + for _, fs := range trackers { + if len(fs.so) <= 1 { + continue + } - sessions := sessionMap[node] + sessions := sessionMap[node] - // find session that doesn't have the fs name reserved - idx := slices.IndexFunc(sessions, func(s *sharedSession) bool { - _, ok := s.fsMap[key.name] - return !ok - }) + // find session that doesn't have the fs name reserved + idx := slices.IndexFunc(sessions, func(s *sharedSession) bool { + _, ok := s.fsMap[key.name] + return !ok + }) - var ss *sharedSession - if idx == -1 { - s, err := session.NewSession(ctx, fs.so[0].SharedKey) - if err != nil { - return nil, err + var ss *sharedSession + if idx == -1 { + s, err := session.NewSession(ctx, fs.so[0].SharedKey) + if err != nil { + return nil, err + } + ss = &sharedSession{Session: s, fsMap: map[string]*fsMount{}} + sessions = append(sessions, ss) + sessionMap[node] = sessions + } else { + ss = sessions[idx] } - ss = &sharedSession{Session: s, fsMap: map[string]fsutil.FS{}} - sessions = append(sessions, ss) - sessionMap[node] = sessions - } else { - ss = sessions[idx] - } - ss.fsMap[key.name] = fs.fs - for _, so := range fs.so { - if so.FrontendAttrs == nil { - so.FrontendAttrs = map[string]string{} + ss.fsMap[key.name] = fs.fs + for _, so := range fs.so { + if so.FrontendAttrs == nil { + so.FrontendAttrs = map[string]string{} + } + so.FrontendAttrs["local-sessionid:"+key.name] = ss.ID() } - so.FrontendAttrs["local-sessionid:"+key.name] = ss.ID() } } } @@ -1447,8 +1453,8 @@ func detectSharedMounts(ctx context.Context, reqs map[string][]*reqForNode) (_ m arr = append(arr, s.Session) src := make(filesync.StaticDirSource, len(s.fsMap)) - for name, fs := range s.fsMap { - fs, err := fsutil.NewFilterFS(fs, &fsutil.FilterOpt{ + for name, mount := range s.fsMap { + fs, err := fsutil.NewFilterFS(mount.FS, &fsutil.FilterOpt{ Map: resetUIDAndGID, }) if err != nil { diff --git a/build/opt.go b/build/opt.go index 421ef585570c..d22352eb7851 100644 --- a/build/opt.go +++ b/build/opt.go @@ -50,6 +50,7 @@ import ( "github.com/moby/buildkit/util/archiveutil" "github.com/moby/buildkit/util/entitlements" "github.com/moby/buildkit/util/gitutil" + "github.com/moby/patternmatcher/ignorefile" "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -577,7 +578,7 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver so.FrontendAttrs["label:"+k] = v } - releaseLoad, err := loadInputs(ctx, nodeDriver, &opt.Inputs, pw, &so) + releaseLoad, err := loadInputs(ctx, nodeDriver, &opt.Inputs, cfg, pw, &so) if err != nil { return nil, nil, err } @@ -897,7 +898,7 @@ func policyEnvFilename(inp Inputs) string { return "Dockerfile" } -func loadInputs(ctx context.Context, d *driver.DriverHandle, inp *Inputs, pw progress.Writer, target *client.SolveOpt) (func(), error) { +func loadInputs(ctx context.Context, d *driver.DriverHandle, inp *Inputs, cfg *confutil.Config, pw progress.Writer, target *client.SolveOpt) (func(), error) { if inp.ContextPath == "" { return nil, errors.New("please specify build context (e.g. \".\" for the current directory)") } @@ -959,7 +960,7 @@ func loadInputs(ctx context.Context, d *driver.DriverHandle, inp *Inputs, pw pro } } case osutil.IsLocalDir(inp.ContextPath): - if err := setLocalMount("context", inp.ContextPath, target); err != nil { + if err := setLocalContextMount("context", inp.ContextPath, cfg, target); err != nil { return nil, err } contextDir = inp.ContextPath @@ -1139,7 +1140,7 @@ func loadInputs(ctx context.Context, d *driver.DriverHandle, inp *Inputs, pw pro if k == "context" || k == "dockerfile" { localName = "_" + k // underscore to avoid collisions } - if err := setLocalMount(localName, v.Path, target); err != nil { + if err := setLocalContextMount(localName, v.Path, cfg, target); err != nil { return nil, err } target.FrontendAttrs["context:"+k] = "local:" + localName @@ -1231,6 +1232,42 @@ func setLocalMount(name, dir string, so *client.SolveOpt) error { return nil } +func setLocalContextMount(name, dir string, cfg *confutil.Config, so *client.SolveOpt) error { + if err := setLocalMount(name, dir, so); err != nil { + return err + } + + filename := filepath.Join(cfg.Dir(), ".dockerignore") + f, err := os.Open(filename) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return errors.Wrapf(err, "failed to open global ignore file %s", filename) + } + defer f.Close() + + patterns, err := ignorefile.ReadAll(f) + if err != nil { + return errors.Wrapf(err, "failed to read global ignore file %s", filename) + } + if len(patterns) == 0 { + return nil + } + // BuildKit reads the context's ignore file through this mount before + // applying its repository-specific rules. + mount := so.LocalMounts[name].(*fsMount) + filtered, err := fsutil.NewFilterFS(mount.FS, &fsutil.FilterOpt{ + ExcludePatterns: append(slices.Clone(patterns), "!.dockerignore"), + }) + if err != nil { + return errors.Wrapf(err, "failed to parse global ignore file %s", filename) + } + mount.FS = filtered + mount.patterns = patterns + return nil +} + func createTempDockerfile(r io.Reader, multiReader *SyncMultiReader) (string, error) { dir, err := os.MkdirTemp("", "dockerfile") if err != nil { @@ -1341,7 +1378,8 @@ func handleLowercaseDockerfile(dir, p string) string { type fsMount struct { fsutil.FS - dir string + dir string + patterns []string } var _ fsutil.FS = &fsMount{} diff --git a/build/opt_test.go b/build/opt_test.go index b7442c2dec4f..3b15ccc6221e 100644 --- a/build/opt_test.go +++ b/build/opt_test.go @@ -5,6 +5,8 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "sync" "sync/atomic" @@ -32,6 +34,7 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tonistiigi/fsutil" ) type exporterTestDriver struct { @@ -204,6 +207,92 @@ func TestDriverFeatureFailurePreservesProvenance(t *testing.T) { require.Equal(t, 2, calls) } +func TestGlobalDockerignore(t *testing.T) { + contextDir := t.TempDir() + configDir := t.TempDir() + for name, content := range map[string]string{ + "keep.txt": "keep", + "keep.bin": "keep", + "drop.txt": "drop", + "secret.pem": "secret", + ".dockerignore": "*.log\n", + "debug.log": "log", + ".idea/state": "editor", + ".idea/keep": "editor", + } { + path := filepath.Join(contextDir, name) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + } + require.NoError(t, os.WriteFile(filepath.Join(configDir, ".dockerignore"), []byte(".idea/*\n!.idea/keep\n.dockerignore\n*.txt\n*.pem\n!secret.pem\n"), 0o644)) + + so := &client.SolveOpt{} + require.NoError(t, setLocalContextMount("context", contextDir, confutil.NewConfig(nil, confutil.WithDir(configDir)), so)) + // The global filter keeps the ignore file available for the frontend. + f, err := so.LocalMounts["context"].Open(".dockerignore") + require.NoError(t, err) + require.NoError(t, f.Close()) + opt := &fsutil.FilterOpt{ExcludePatterns: []string{"!keep.txt", "secret.pem", ".dockerignore"}} + filtered, err := fsutil.NewFilterFS(so.LocalMounts["context"], opt) + require.NoError(t, err) + for _, name := range []string{"keep.bin", "debug.log", ".idea/keep"} { + f, err := filtered.Open(name) + require.NoError(t, err, name) + _, err = io.ReadAll(f) + require.NoError(t, err) + require.NoError(t, f.Close()) + } + _, err = filtered.Open(".idea/state") + require.ErrorIs(t, err, os.ErrNotExist) + _, err = filtered.Open("drop.txt") + require.ErrorIs(t, err, os.ErrNotExist) + _, err = filtered.Open("keep.txt") + require.ErrorIs(t, err, os.ErrNotExist) + _, err = filtered.Open("secret.pem") + require.ErrorIs(t, err, os.ErrNotExist) + _, err = filtered.Open(".dockerignore") + require.ErrorIs(t, err, os.ErrNotExist) + + // A missing global file leaves local mounts unchanged. + require.NoError(t, setLocalContextMount("other", contextDir, confutil.NewConfig(nil, confutil.WithDir(t.TempDir())), so)) + f, err = so.LocalMounts["other"].Open(".idea/state") + require.NoError(t, err) + require.NoError(t, f.Close()) +} + +func TestGlobalDockerignoreInvalidPattern(t *testing.T) { + configDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(configDir, ".dockerignore"), []byte("[\n"), 0o644)) + err := setLocalContextMount("context", t.TempDir(), confutil.NewConfig(nil, confutil.WithDir(configDir)), &client.SolveOpt{}) + require.ErrorContains(t, err, "failed to parse global ignore file") +} + +func TestLoadInputsGlobalDockerignore(t *testing.T) { + contextDir := t.TempDir() + namedDir := t.TempDir() + configDir := t.TempDir() + for _, dir := range []string{contextDir, namedDir} { + require.NoError(t, os.WriteFile(filepath.Join(dir, "local.txt"), []byte("local"), 0o644)) + } + require.NoError(t, os.WriteFile(filepath.Join(contextDir, "Containerfile"), []byte("FROM scratch\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(configDir, ".dockerignore"), []byte("local.txt\n"), 0o644)) + + so := &client.SolveOpt{FrontendAttrs: map[string]string{}} + inp := &Inputs{ContextPath: contextDir, DockerfilePath: filepath.Join(contextDir, "Containerfile"), NamedContexts: map[string]NamedContext{ + "source": {Path: namedDir}, + }} + release, err := loadInputs(context.Background(), nil, inp, confutil.NewConfig(nil, confutil.WithDir(configDir)), testProgressWriter{}, so) + require.NoError(t, err) + t.Cleanup(release) + for _, name := range []string{"context", "source"} { + _, err = so.LocalMounts[name].Open("local.txt") + require.ErrorIs(t, err, os.ErrNotExist) + } + f, err := so.LocalMounts["dockerfile"].Open("local.txt") + require.NoError(t, err) + require.NoError(t, f.Close()) +} + func TestCacheOptions_DerivedVars(t *testing.T) { t.Setenv("ACTIONS_RUNTIME_TOKEN", "sensitive_token") t.Setenv("ACTIONS_CACHE_URL", "https://cache.github.com") @@ -467,7 +556,7 @@ func TestLoadInputsOCILayoutNamedContext(t *testing.T) { }, } - release, err := loadInputs(context.Background(), nil, inp, testProgressWriter{}, target) + release, err := loadInputs(context.Background(), nil, inp, nil, testProgressWriter{}, target) require.NoError(t, err) require.NotNil(t, release) t.Cleanup(release) diff --git a/go.mod b/go.mod index aedf0984e762..6e534d06a90d 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/moby/buildkit v0.34.0-rc1 github.com/moby/moby/api v1.56.0 github.com/moby/moby/client v0.6.0 + github.com/moby/patternmatcher v0.6.1 github.com/moby/policy-helpers v0.0.0-20260901142052-72f704e6cdb6 github.com/moby/sys/atomicwriter v0.1.0 github.com/moby/sys/mountinfo v0.7.2 @@ -170,7 +171,6 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/locker v1.0.1 // indirect - github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/signal v0.7.1 // indirect diff --git a/tests/bake.go b/tests/bake.go index 2e664eab89e6..822f83049d9f 100644 --- a/tests/bake.go +++ b/tests/bake.go @@ -49,6 +49,7 @@ var bakeTests = []func(t *testing.T, sb integration.Sandbox){ testBakePrintRemoteContextSubdir, testBakeLocal, testBakeLocalMulti, + testBakeGlobalDockerignoreSharedContext, testBakeDeferOutput, testBakeFailFast, testBakeDeferError, @@ -734,6 +735,52 @@ services: require.FileExists(t, filepath.Join(dirDest2, "foo")) } +func testBakeGlobalDockerignoreSharedContext(t *testing.T, sb integration.Sandbox) { + dir := tmpdir(t, + fstest.CreateFile("docker-bake.hcl", []byte(` +group "default" { + targets = ["a", "b"] +} +target "a" { + dockerfile = "Dockerfile.a" +} +target "b" { + dockerfile = "Dockerfile.b" +} +`), 0o600), + fstest.CreateFile("Dockerfile.a", []byte("FROM scratch\nCOPY . /a/\n"), 0o600), + fstest.CreateFile("Dockerfile.b", []byte("FROM scratch\nCOPY . /b/\n"), 0o600), + fstest.CreateFile(".dockerignore", []byte("*\n!src\n"), 0o600), + fstest.CreateDir("src", 0o700), + fstest.CreateFile("src/keep.txt", []byte(identity.NewID()), 0o600), + fstest.CreateFile("src/.DS_Store", []byte("ignored"), 0o600), + fstest.CreateDir("src/.idea", 0o700), + fstest.CreateFile("src/.idea/ws.xml", []byte("ignored"), 0o600), + ) + configDir := buildxConfig(sb) + require.NotEmpty(t, configDir) + require.NoError(t, os.MkdirAll(configDir, 0o700)) + globalIgnore := filepath.Join(configDir, ".dockerignore") + require.NoError(t, os.WriteFile(globalIgnore, []byte("**/.idea\n**/.DS_Store\n"), 0o600)) + t.Cleanup(func() { _ = os.Remove(globalIgnore) }) + + destA, destB := t.TempDir(), t.TempDir() + out, err := bakeCmd(sb, withDir(dir), withArgs( + "--progress=plain", + "--set", "a.output=type=local,dest="+destA, + "--set", "b.output=type=local,dest="+destB, + )) + require.NoError(t, err, out) + require.Equal(t, 1, strings.Count(out, "internal] load build context"), out) + for _, output := range []struct{ dir, prefix string }{{destA, "a"}, {destB, "b"}} { + require.FileExists(t, filepath.Join(output.dir, output.prefix, "src", "keep.txt")) + for _, path := range []string{".dockerignore", "src/.DS_Store", "src/.idea/ws.xml"} { + _, err := os.Stat(filepath.Join(output.dir, output.prefix, path)) + require.ErrorIs(t, err, os.ErrNotExist) + } + } +} + func testBakeDeferOutput(t *testing.T, sb integration.Sandbox) { dir := bakeExecutionFailureDir(t, []byte(` FROM scratch diff --git a/tests/build.go b/tests/build.go index e7d6b8a9b523..5357d69023fd 100644 --- a/tests/build.go +++ b/tests/build.go @@ -53,6 +53,7 @@ func buildCmd(sb integration.Sandbox, opts ...cmdOpt) (string, error) { var buildTests = []func(t *testing.T, sb integration.Sandbox){ testBuild, + testBuildGlobalDockerignore, testBuildAlias, testBuildStdin, testBuildRemote, @@ -103,6 +104,51 @@ func testBuild(t *testing.T, sb integration.Sandbox) { require.NoError(t, err, string(out)) } +func testBuildGlobalDockerignore(t *testing.T, sb integration.Sandbox) { + contextDir := tmpdir(t, + fstest.CreateFile("Dockerfile", []byte("FROM scratch\nCOPY . /main/\nCOPY --from=extra . /named/\n"), 0o600), + fstest.CreateFile(".dockerignore", []byte(".dockerignore\nrepo-only.bin\n!keep.txt\n"), 0o600), + fstest.CreateFile("keep.txt", []byte("main"), 0o600), + fstest.CreateFile("keep.bin", []byte("main"), 0o600), + fstest.CreateFile("drop.txt", []byte("global"), 0o600), + fstest.CreateFile("repo-only.bin", []byte("repo"), 0o600), + fstest.CreateDir(".idea", 0o700), + fstest.CreateFile(".idea/state", []byte("editor"), 0o600), + ) + namedDir := tmpdir(t, + fstest.CreateFile("keep.bin", []byte("named"), 0o600), + fstest.CreateFile("drop.txt", []byte("global"), 0o600), + fstest.CreateDir(".idea", 0o700), + fstest.CreateFile(".idea/state", []byte("editor"), 0o600), + ) + configDir := buildxConfig(sb) + require.NotEmpty(t, configDir) + require.NoError(t, os.MkdirAll(configDir, 0o700)) + globalIgnore := filepath.Join(configDir, ".dockerignore") + require.NoError(t, os.WriteFile(globalIgnore, []byte(".idea\n*.txt\n.dockerignore\n"), 0o600)) + t.Cleanup(func() { _ = os.Remove(globalIgnore) }) + + dest := t.TempDir() + out, err := buildCmd(sb, withArgs( + "--build-context", "extra="+namedDir, + "--output=type=local,dest="+dest, + contextDir, + )) + require.NoError(t, err, out) + for path, want := range map[string]string{ + "main/keep.bin": "main", + "named/keep.bin": "named", + } { + data, err := os.ReadFile(filepath.Join(dest, path)) + require.NoError(t, err) + require.Equal(t, want, string(data)) + } + for _, path := range []string{"main/.dockerignore", "main/.idea/state", "named/.idea/state", "main/repo-only.bin", "main/keep.txt", "main/drop.txt", "named/drop.txt"} { + _, err := os.Stat(filepath.Join(dest, path)) + require.ErrorIs(t, err, os.ErrNotExist) + } +} + func testBuildAlias(t *testing.T, sb integration.Sandbox) { dir := createTestProject(t) cmd := buildxCmd(sb, withDir(dir), withArgs("b", dir))