From a8715c16e9c069d8cb5b9c7fa3c87d5edb6c8568 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Mon, 12 Oct 2026 10:54:33 -0700 Subject: [PATCH 1/2] fix: create a missing local database when init runs inside a Git worktree runInit skipped store.Open whenever the database path was inside a Git worktree. Init exited 0 without creating the file, and doctor then refused repair for an unmarked checkout. Create the file when it is missing. Leave an existing worktree database unread so an unmarked portable source is not migrated in place. Signed-off-by: Sebastien Tardif --- CHANGELOG.md | 2 + internal/cli/init.go | 40 +++++++++--- internal/cli/init_worktree_test.go | 99 ++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 9 deletions(-) create mode 100644 internal/cli/init_worktree_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 035181e..007eba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Create a missing local database during `init` when the path is inside a Git worktree, and leave an existing database in that worktree unchanged. Thanks @SebTardif. + ## 0.15.1 - 2026-10-11 **Highlights:** Faster embedding on large archives, reliable local database setup, and Go security fixes. diff --git a/internal/cli/init.go b/internal/cli/init.go index 8c37142..603fc91 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -2,9 +2,11 @@ package cli import ( "context" + "errors" "flag" "fmt" "io" + "os" "path/filepath" "strings" @@ -124,16 +126,8 @@ func (a *App) runInit(ctx context.Context, args []string) error { } } if remoteEndpointValue == "" && portableStoreURL == "" { - if _, portable, err := portableStoreRoot(ctx, cfg.DBPath); err != nil { + if err := initializeMissingLocalDatabase(ctx, cfg.DBPath); err != nil { return err - } else if !portable { - st, err := store.Open(ctx, cfg.DBPath) - if err != nil { - return fmt.Errorf("initialize local database: %w", err) - } - if err := st.Close(); err != nil { - return fmt.Errorf("close initialized local database: %w", err) - } } } result := initResult{ @@ -155,6 +149,34 @@ func (a *App) runInit(ctx context.Context, args []string) error { return a.writeInitOutput(result) } +// initializeMissingLocalDatabase creates the local SQLite file when it is +// absent. An existing file inside a Git worktree is left untouched: that +// checkout can be an unmarked portable source, and opening it would migrate +// the source in place. +func initializeMissingLocalDatabase(ctx context.Context, dbPath string) error { + _, portable, err := portableStoreRoot(ctx, dbPath) + if err != nil { + return err + } + if portable { + _, err = os.Stat(dbPath) + if err == nil { + return nil + } + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect local database: %w", err) + } + } + st, err := store.Open(ctx, dbPath) + if err != nil { + return fmt.Errorf("initialize local database: %w", err) + } + if err := st.Close(); err != nil { + return fmt.Errorf("close initialized local database: %w", err) + } + return nil +} + func (a *App) absoluteInitPath(path string) (string, error) { originalPath := strings.TrimSpace(path) path = crawlconfig.ExpandHome(path) diff --git a/internal/cli/init_worktree_test.go b/internal/cli/init_worktree_test.go new file mode 100644 index 0000000..1e30a97 --- /dev/null +++ b/internal/cli/init_worktree_test.go @@ -0,0 +1,99 @@ +package cli + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/openclaw/gitcrawl/internal/store" +) + +func TestInitCreatesMissingDatabaseInsideGitWorktree(t *testing.T) { + ctx := context.Background() + + t.Run("db flag", func(t *testing.T) { + repo := initOrdinaryGitRepo(t, ctx) + dbPath := filepath.Join(repo, "gitcrawl.db") + assertOrdinaryGitWorktree(t, ctx, repo, dbPath) + configPath := filepath.Join(repo, "config.toml") + if err := New().Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatalf("init: %v", err) + } + assertInitializedLocalDatabase(t, ctx, dbPath) + if err := New().Run(ctx, []string{"--config", configPath, "doctor"}); err != nil { + t.Fatalf("doctor: %v", err) + } + }) + + t.Run("runtime dir", func(t *testing.T) { + repo := initOrdinaryGitRepo(t, ctx) + runtimeDir := filepath.Join(repo, "runtime") + dbPath := filepath.Join(runtimeDir, "gitcrawl.db") + assertOrdinaryGitWorktree(t, ctx, repo, dbPath) + configPath := filepath.Join(repo, "config.toml") + if err := New().Run(ctx, []string{"--config", configPath, "init", "--runtime-dir", runtimeDir}); err != nil { + t.Fatalf("init: %v", err) + } + assertInitializedLocalDatabase(t, ctx, dbPath) + if err := New().Run(ctx, []string{"--config", configPath, "doctor"}); err != nil { + t.Fatalf("doctor: %v", err) + } + }) +} + +func TestInitLeavesExistingWorktreeDatabaseUntouched(t *testing.T) { + ctx := context.Background() + repo := initOrdinaryGitRepo(t, ctx) + dbPath := filepath.Join(repo, "gitcrawl.db") + assertOrdinaryGitWorktree(t, ctx, repo, dbPath) + const sentinel = "not a gitcrawl database\n" + if err := os.WriteFile(dbPath, []byte(sentinel), 0o644); err != nil { + t.Fatalf("write sentinel: %v", err) + } + configPath := filepath.Join(repo, "config.toml") + if err := New().Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { + t.Fatalf("init: %v", err) + } + got, err := os.ReadFile(dbPath) + if err != nil { + t.Fatalf("read database: %v", err) + } + if string(got) != sentinel { + t.Fatalf("existing worktree database changed to %q", got) + } +} + +func initOrdinaryGitRepo(t *testing.T, ctx context.Context) string { + t.Helper() + repo := t.TempDir() + if err := runGit(ctx, repo, "init", "-b", "main"); err != nil { + t.Fatalf("git init: %v", err) + } + return repo +} + +func assertOrdinaryGitWorktree(t *testing.T, ctx context.Context, repo, dbPath string) { + t.Helper() + root, portable, err := portableStoreRoot(ctx, dbPath) + if err != nil { + t.Fatalf("portable store root: %v", err) + } + if !portable || root != repo { + t.Fatalf("portable store root = %q, portable=%v, want ordinary worktree %q", root, portable, repo) + } +} + +func assertInitializedLocalDatabase(t *testing.T, ctx context.Context, dbPath string) { + t.Helper() + st, err := store.OpenReadOnly(ctx, dbPath) + if err != nil { + t.Fatalf("open initialized database: %v", err) + } + if _, err := st.Status(ctx); err != nil { + t.Fatalf("read initialized database: %v", err) + } + if err := st.Close(); err != nil { + t.Fatalf("close initialized database: %v", err) + } +} From 44cc95e3b6c4fb3860a87b5a2c1cf239a640b6b9 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Mon, 12 Oct 2026 11:14:54 -0700 Subject: [PATCH 2/2] fix: keep gzip portable sources intact during init A missing raw database is not a missing source when its manifest selects a gzip archive. init --db created that raw file and could add sibling runtime directories, so strict refresh rejected the checkout. Skip creation when the file already exists or the manifest selects gzip. Regression: TestInitDBPreservesCompressedPortableCheckout. Signed-off-by: Sebastien Tardif --- CHANGELOG.md | 2 +- internal/cli/init.go | 56 ++++++++++++++++++++---------- internal/cli/init_worktree_test.go | 32 +++++++++++++++++ 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 007eba3..5e129ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Create a missing local database during `init` when the path is inside a Git worktree, and leave an existing database in that worktree unchanged. Thanks @SebTardif. +- Create a missing local database during `init` when the path is inside a Git worktree. Leave an existing database unchanged, and leave a gzip portable source without a raw database file unchanged. Thanks @SebTardif. ## 0.15.1 - 2026-10-11 diff --git a/internal/cli/init.go b/internal/cli/init.go index 603fc91..ea079e4 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -120,13 +120,12 @@ func (a *App) runInit(ctx context.Context, args []string) error { if err := config.Save(a.configPath, cfg); err != nil { return err } - if !(cfg.Remote.Enabled() && cfg.Remote.Mode == crawlremote.ModeCloud) { - if err := config.EnsureRuntimeDirs(cfg); err != nil { + if remoteEndpointValue == "" && portableStoreURL == "" { + if err := initializeMissingLocalDatabase(ctx, cfg); err != nil { return err } - } - if remoteEndpointValue == "" && portableStoreURL == "" { - if err := initializeMissingLocalDatabase(ctx, cfg.DBPath); err != nil { + } else if !(cfg.Remote.Enabled() && cfg.Remote.Mode == crawlremote.ModeCloud) { + if err := config.EnsureRuntimeDirs(cfg); err != nil { return err } } @@ -150,24 +149,21 @@ func (a *App) runInit(ctx context.Context, args []string) error { } // initializeMissingLocalDatabase creates the local SQLite file when it is -// absent. An existing file inside a Git worktree is left untouched: that -// checkout can be an unmarked portable source, and opening it would migrate -// the source in place. -func initializeMissingLocalDatabase(ctx context.Context, dbPath string) error { - _, portable, err := portableStoreRoot(ctx, dbPath) +// absent. An existing file inside a Git worktree is left untouched, as is a +// missing raw file whose manifest selects a gzip archive. Opening either +// would migrate or dirty an unmarked portable source. +func initializeMissingLocalDatabase(ctx context.Context, cfg config.Config) error { + preserve, err := preservePortableDatabase(ctx, cfg.DBPath) if err != nil { return err } - if portable { - _, err = os.Stat(dbPath) - if err == nil { - return nil - } - if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("inspect local database: %w", err) - } + if preserve { + return nil + } + if err := config.EnsureRuntimeDirs(cfg); err != nil { + return err } - st, err := store.Open(ctx, dbPath) + st, err := store.Open(ctx, cfg.DBPath) if err != nil { return fmt.Errorf("initialize local database: %w", err) } @@ -177,6 +173,28 @@ func initializeMissingLocalDatabase(ctx context.Context, dbPath string) error { return nil } +func preservePortableDatabase(ctx context.Context, dbPath string) (bool, error) { + _, portable, err := portableStoreRoot(ctx, dbPath) + if err != nil { + return false, err + } + if !portable { + return false, nil + } + _, err = os.Stat(dbPath) + if err == nil { + return true, nil + } + if !errors.Is(err, os.ErrNotExist) { + return false, fmt.Errorf("inspect local database: %w", err) + } + _, _, compressed, err := portableSourceArtifact(dbPath) + if err != nil { + return false, err + } + return compressed, nil +} + func (a *App) absoluteInitPath(path string) (string, error) { originalPath := strings.TrimSpace(path) path = crawlconfig.ExpandHome(path) diff --git a/internal/cli/init_worktree_test.go b/internal/cli/init_worktree_test.go index 1e30a97..71b01f2 100644 --- a/internal/cli/init_worktree_test.go +++ b/internal/cli/init_worktree_test.go @@ -1,7 +1,9 @@ package cli import ( + "bytes" "context" + "errors" "os" "path/filepath" "testing" @@ -64,6 +66,36 @@ func TestInitLeavesExistingWorktreeDatabaseUntouched(t *testing.T) { } } +func TestInitDBPreservesCompressedPortableCheckout(t *testing.T) { + fixture := newPortableRefreshFixture(t, true) + logical := filepath.Join(fixture.checkout, fixture.relative) + if _, err := os.Stat(logical); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("fixture raw database: %v", err) + } + before := portableTestGit(t, fixture.checkout, "status", "--porcelain=v1", "-z", "--untracked-files=all", "--ignored=matching") + if before != "" { + t.Fatalf("fixture not clean: %q", before) + } + configPath := filepath.Join(filepath.Dir(fixture.checkout), "local.toml") + app := New() + var stdout, stderr bytes.Buffer + app.Stdout, app.Stderr = &stdout, &stderr + if err := app.Run(context.Background(), []string{"--config", configPath, "init", "--db", logical}); err != nil { + t.Fatalf("init: %v\n%s", err, stderr.String()) + } + if _, err := os.Stat(logical); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("init created raw database: %v", err) + } + after := portableTestGit(t, fixture.checkout, "status", "--porcelain=v1", "-z", "--untracked-files=all", "--ignored=matching") + if after != "" { + t.Fatalf("init dirtied compressed checkout: %q", after) + } + result, err := fixture.refresh(t) + if err != nil { + t.Fatalf("refresh: %+v %v", result, err) + } +} + func initOrdinaryGitRepo(t *testing.T, ctx context.Context) string { t.Helper() repo := t.TempDir()