From 04541ecd67a044c922a8d034fcd8cb8f5c3f8200 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 16 Oct 2026 14:13:19 +0200 Subject: [PATCH 01/29] fixup! odb: scan all sources' packfiles before loose objects This reverts commit 78c3276c2605e7b4237bda34b0f41d23a1daacfd. Make way for Patrick Steinhardt's replacement from upstream/seen rather than carry duplicate logic into future rebases. Assisted-by: GPT-6.1 Sol Signed-off-by: Johannes Schindelin --- odb.c | 53 ++++++++++------------------------------ odb.h | 12 --------- odb/source-files.c | 26 ++++++++------------ odb/source.h | 4 --- t/t5615-alternate-env.sh | 43 -------------------------------- 5 files changed, 23 insertions(+), 115 deletions(-) diff --git a/odb.c b/odb.c index 95adfdb9f07db8..7fe6c996aa2141 100644 --- a/odb.c +++ b/odb.c @@ -752,27 +752,6 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } -static enum odb_read_status read_object_info_from_sources( - struct object_database *odb, const struct object_id *oid, - struct object_info *oi, enum object_info_flags flags, - struct strbuf *errmsg) -{ - struct odb_source *source; - enum odb_read_status ret = ODB_READ_NOT_FOUND; - - for (source = odb->sources; source; source = source->next) { - enum odb_read_status source_ret = odb_source_read_object_info( - source, oid, oi, flags, errmsg->len ? NULL : errmsg); - - if (!source_ret) - return ODB_READ_OK; - if (source_ret != ODB_READ_NOT_FOUND) - ret = source_ret; - } - - return ret; -} - static enum odb_read_status do_oid_object_info_extended(struct object_database *odb, const struct object_id *oid, struct object_info *oi, unsigned flags) @@ -797,25 +776,17 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * while (1) { extern int core_use_gvfs_helper; - enum object_info_flags source_flags = flags; + struct odb_source *source; - if (odb_has_alternates(odb)) { - ret = read_object_info_from_sources(odb, real, oi, - flags | OBJECT_INFO_SKIP_LOOSE, &corrupt_err); + for (source = odb->sources; source; source = source->next) { + ret = odb_source_read_object_info(source, real, oi, flags, + corrupt_err.len ? NULL : &corrupt_err); if (!ret) goto out; if (ret != ODB_READ_NOT_FOUND) corrupt = true; - source_flags |= OBJECT_INFO_SKIP_PACKED; } - ret = read_object_info_from_sources(odb, real, oi, source_flags, - &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; - if (core_use_gvfs_helper && !tried_gvfs_helper && !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { enum gh_client__created ghc; @@ -844,13 +815,15 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * * caches or reload on-disk state. */ if (!(flags & OBJECT_INFO_QUICK)) { - ret = read_object_info_from_sources(odb, real, oi, - flags | OBJECT_INFO_SECOND_READ, - &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; + for (source = odb->sources; source; source = source->next) { + ret = odb_source_read_object_info(source, real, oi, + flags | OBJECT_INFO_SECOND_READ, + corrupt_err.len ? NULL : &corrupt_err); + if (!ret) + goto out; + if (ret != ODB_READ_NOT_FOUND) + corrupt = true; + } if (gvfs_virtualize_objects(odb->repo) && !tried_hook && !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { // TODO Assert or at least trace2 if gvfs-helper diff --git a/odb.h b/odb.h index f6deef02f6b2cf..2f0867c75a9d0c 100644 --- a/odb.h +++ b/odb.h @@ -445,18 +445,6 @@ enum object_info_flags { */ OBJECT_INFO_SECOND_READ = (1 << 4), - /* - * Only consult the packed object store of a source, skipping its loose - * object store (OBJECT_INFO_SKIP_LOOSE), or vice versa - * (OBJECT_INFO_SKIP_PACKED). These are used by - * odb_read_object_info_extended() to scan the packfiles of all sources - * before consulting any source's loose object store, so that an object - * that resides in an alternate's packfile is not preceded by a spurious - * loose-object lookup on an earlier source. - */ - OBJECT_INFO_SKIP_LOOSE = (1 << 5), - OBJECT_INFO_SKIP_PACKED = (1 << 6), - /* * This is meant for bulk prefetching of missing blobs in a partial * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK. diff --git a/odb/source-files.c b/odb/source-files.c index dc212ccb1e0d76..ed1f8e7ecf7dbc 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -83,23 +83,17 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source struct strbuf *errmsg) { struct odb_source_files *files = odb_source_files_downcast(source); - enum odb_read_status ret_packed = ODB_READ_NOT_FOUND; - enum odb_read_status ret_loose = ODB_READ_NOT_FOUND; - - if (!(flags & OBJECT_INFO_SKIP_PACKED)) { - ret_packed = odb_source_read_object_info(&files->packed->base, - oid, oi, flags, errmsg); - if (!ret_packed) - return 0; - } + enum odb_read_status ret_packed, ret_loose; - if (!(flags & OBJECT_INFO_SKIP_LOOSE)) { - ret_loose = odb_source_read_object_info(&files->loose->base, - oid, oi, flags, - ret_packed == ODB_READ_NOT_FOUND ? errmsg : NULL); - if (!ret_loose) - return 0; - } + ret_packed = odb_source_read_object_info(&files->packed->base, oid, oi, + flags, errmsg); + if (!ret_packed) + return 0; + + ret_loose = odb_source_read_object_info(&files->loose->base, oid, oi, flags, + ret_packed == ODB_READ_NOT_FOUND ? errmsg : NULL); + if (!ret_loose) + return 0; /* * Reading the packed object may have failed even though the object diff --git a/odb/source.h b/odb/source.h index cafeef1f8f8367..8e776f1df25da7 100644 --- a/odb/source.h +++ b/odb/source.h @@ -136,10 +136,6 @@ struct odb_source { * second read in case they know that the first read would have * already surfaced the object without reloading any on-disk state. * - * - `OBJECT_INFO_SKIP_LOOSE` and `OBJECT_INFO_SKIP_PACKED` tell the - * files backend not to consult its loose or packed source, - * respectively. - * * The callback is expected to return an `enum odb_read_status`. Please * refer to the individual values that can be returned. In case reading * the object has failed with a generic error and `errmsg` is non-NULL, diff --git a/t/t5615-alternate-env.sh b/t/t5615-alternate-env.sh index d82dba65d94b14..1bfeccdeb49958 100755 --- a/t/t5615-alternate-env.sh +++ b/t/t5615-alternate-env.sh @@ -88,47 +88,4 @@ test_expect_success !MINGW 'broken quoting falls back to interpreting raw' ' EOF ' -test_expect_success 'packs across sources are checked before loose objects' ' - # Regression test for a performance issue in which an object that - # resides in an alternate as a packed object caused a spurious loose - # object lookup (a filesystem stat) on the main object store before the - # alternate packfile was consulted. Reading such an object must resolve - # to the alternate packfile, never to a loose copy in the main store. - # - # Build an alternate whose object "B" is stored as a delta in a - # packfile. git deltifies successive versions of a tracked file, so the - # older, shorter blob "B" becomes a delta against the newer, longer - # blob "O". A loose object has no delta base, so %(deltabase) tells us - # which store answered the read: the alternate pack (O) or a loose copy - # (the zero oid). - git init alt-src && - test_seq 1 200 >alt-src/file && - git -C alt-src add file && - git -C alt-src commit -q -m base && - B=$(git -C alt-src rev-parse HEAD:file) && - git -C alt-src cat-file blob "$B" >b-content && - test_seq 1 210 >alt-src/file && - git -C alt-src add file && - git -C alt-src commit -q -m more && - O=$(git -C alt-src rev-parse HEAD:file) && - git -C alt-src repack -adf --window=10 --depth=50 && - - # Precondition: in the alternate, B is a delta based on O. - echo "$B" >in && - echo "$O" >expect && - git -C alt-src cat-file --batch-check="%(deltabase)" actual && - test_cmp expect actual && - - # Main repo: write B as a loose object, before any alternate is active. - git init main && - git -C main hash-object -w --stdin /dev/null && - test -e "main/.git/objects/$(test_oid_to_path "$B")" && - - # With the alternate active, B must resolve to the alternate packfile - # (deltabase O), not to the main store loose copy (deltabase zero oid). - GIT_ALTERNATE_OBJECT_DIRECTORIES="$PWD/alt-src/.git/objects" \ - git -C main cat-file --batch-check="%(deltabase)" actual && - test_cmp expect actual -' - test_done From b9f2eeec1a4e0d1833f7c07ba7ae64aee80e8512 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:11 +0200 Subject: [PATCH 02/29] setup: split up concerns of `init_db()` The function `init_db()` is responsible for creating the on-disk directory structure required for a Git repository. It is used by both git-init(1) and git-clone(1), and because their expected behaviour is different we support a couple of flags: - The `QUIET` flag controls whether the command is quiet or not. For git-init(1) this is user-controllable, whereas for git-clone(1) we're always quiet. - The `EXIST_OK` flag controls whether a preexisting repository is okay or not. For git-init(1) it is, for git-clone(1) it's not. - The `SKIP_REFDB` flag controls whether the reference database should already be created or not. For git-init(1) we do, but for git-clone(1) we don't because it does not yet know about the default branch and about the remote object hash. Furthermore, we're about to add another divergence in behaviour, where we have to also skip creation of the object database in git-clone(1). This is becoming quite cumbersome though. Instead of introducing another flag, start to split up concerns of the function so that we never create the reference or object database. This becomes the responsibility of the caller, which is thus free to defer their creation to a later point in time. This lets us get rid of most of the divergent behaviour: - We don't need the `SKIP_REFDB` and a potential `SKIP_ODB` flags anymore. - We don't need the `QUIET` flag anymore, as nothing prints output except for the final status message that tells the user that the repository has been (re)initialized. But as this message is specific to git-init(1), we can easily move it there. The only piece of information we still have to convey is whether or not reinitialization of a preexisting repository is okay. This is handled by a new `reinit_ok` pointer that, if non-`NULL`, indicates that it is okay to reinitialize the repository. Furthermore, the pointer will be written to to indicate whether the repository was reinitialized or not, which we need in git-init(1) to print the correct initialization message. With these refactorings, `init_db()` is named quite misleadingly though, as we don't create any of the reference or object databases anymore. Rename it to `create_repository()`. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/clone.c | 9 ++++---- builtin/init-db.c | 32 +++++++++++++++++++++------- setup.c | 54 +++++++++++++++-------------------------------- setup.h | 45 ++++++++++++++++++++++++++++----------- 4 files changed, 79 insertions(+), 61 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 690da1a09bdb66..76c15e4b814b4e 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1185,10 +1185,10 @@ int cmd_clone(int argc, * repository, and reference backends may persist that information into * their on-disk data structures. */ - init_db(the_repository, git_dir, real_git_dir, work_tree, option_template, - GIT_HASH_UNKNOWN, ref_storage_format, NULL, - do_not_override_repo_unix_permissions, - INIT_DB_QUIET | INIT_DB_SKIP_REFDB); + create_repository(the_repository, git_dir, real_git_dir, work_tree, + option_template, GIT_HASH_UNKNOWN, ref_storage_format, + do_not_override_repo_unix_permissions, NULL); + create_object_database(the_repository); if (real_git_dir) { free((char *)git_dir); @@ -1445,6 +1445,7 @@ int cmd_clone(int argc, initialize_repository_version(the_repository, hash_algo, the_repository->ref_storage_format, 1); repo_set_hash_algo(the_repository, hash_algo); create_reference_database(the_repository, NULL, 1); + startup_info->have_repository = 1; /* * Before fetching from the remote, download and install bundle diff --git a/builtin/init-db.c b/builtin/init-db.c index e96b1283b7010d..f2c7e3be6d2458 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -80,7 +80,7 @@ int cmd_init_db(int argc, char *work_tree = NULL; const char *template_dir = NULL; char *template_dir_to_free = NULL; - unsigned int flags = 0; + int quiet = 0; int bare = startup_info->force_bare_repository ? 1 : -1; const char *object_format = NULL; const char *ref_format = NULL; @@ -102,7 +102,7 @@ int cmd_init_db(int argc, .flags = PARSE_OPT_OPTARG | PARSE_OPT_NONEG, .callback = shared_callback }, - OPT_BIT('q', "quiet", &flags, N_("be quiet"), INIT_DB_QUIET), + OPT_BOOL('q', "quiet", &quiet, N_("be quiet")), OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"), N_("separate git dir from working tree")), OPT_STRING('b', "initial-branch", &initial_branch, N_("name"), @@ -113,7 +113,7 @@ int cmd_init_db(int argc, N_("specify the reference format to use")), OPT_END() }; - int ret; + int reinit; argc = parse_options(argc, argv, prefix, init_db_options, init_db_usage, 0); @@ -247,14 +247,30 @@ int cmd_init_db(int argc, die(_("--separate-git-dir incompatible with bare repository")); } - flags |= INIT_DB_EXIST_OK; - ret = init_db(the_repository, git_dir, real_git_dir, work_tree, - template_dir, hash_algo, ref_storage_format, initial_branch, - init_shared_repository, flags); + create_repository(the_repository, git_dir, real_git_dir, work_tree, + template_dir, hash_algo, ref_storage_format, + init_shared_repository, &reinit); + create_reference_database(the_repository, initial_branch, quiet); + create_object_database(the_repository); + + if (!quiet) { + int len = strlen(git_dir); + + if (reinit) + printf(repo_settings_get_shared_repository(the_repository) + ? _("Reinitialized existing shared Git repository in %s%s\n") + : _("Reinitialized existing Git repository in %s%s\n"), + git_dir, len && git_dir[len-1] != '/' ? "/" : ""); + else + printf(repo_settings_get_shared_repository(the_repository) + ? _("Initialized empty shared Git repository in %s%s\n") + : _("Initialized empty Git repository in %s%s\n"), + git_dir, len && git_dir[len-1] != '/' ? "/" : ""); + } free(template_dir_to_free); free(real_git_dir_to_free); free(work_tree); free(git_dir); - return ret; + return 0; } diff --git a/setup.c b/setup.c index a8ac9495b32cff..926e38eabb3d7e 100644 --- a/setup.c +++ b/setup.c @@ -2711,7 +2711,7 @@ static int create_default_files(struct repository *repo, return reinit; } -static void create_object_database(struct repository *repo) +void create_object_database(struct repository *repo) { /* * Create the "objects" directory in the common directory. This is done @@ -2886,17 +2886,17 @@ static void repository_format_configure(struct repository_format *repo_fmt, } } -int init_db(struct repository *repo, - const char *git_dir, - const char *real_git_dir, - const char *worktree, - const char *template_dir, int hash, - enum ref_storage_format ref_storage_format, - const char *initial_branch, - int init_shared_repository, unsigned int flags) +void create_repository(struct repository *repo, + const char *git_dir, + const char *real_git_dir, + const char *worktree, + const char *template_dir, + int hash, + enum ref_storage_format ref_storage_format, + int init_shared_repository, + int *reinit_ok) { - int reinit; - int exist_ok = flags & INIT_DB_EXIST_OK; + int reinit_ignored; char *original_git_dir = real_pathdup(git_dir, 1); struct repository_format repo_fmt = REPOSITORY_FORMAT_INIT; struct strbuf err = STRBUF_INIT; @@ -2904,10 +2904,10 @@ int init_db(struct repository *repo, if (real_git_dir) { struct stat st; - if (!exist_ok && !stat(git_dir, &st)) + if (!reinit_ok && !stat(git_dir, &st)) die(_("%s already exists"), git_dir); - if (!exist_ok && !stat(real_git_dir, &st)) + if (!reinit_ok && !stat(real_git_dir, &st)) die(_("%s already exists"), real_git_dir); apply_and_export_relative_gitdir(repo, real_git_dir, 1); @@ -2941,8 +2941,10 @@ int init_db(struct repository *repo, safe_create_dir(repo, git_dir, 0); - reinit = create_default_files(repo, template_dir, original_git_dir, - &repo_fmt, init_shared_repository); + if (!reinit_ok) + reinit_ok = &reinit_ignored; + *reinit_ok = create_default_files(repo, template_dir, original_git_dir, + &repo_fmt, init_shared_repository); if (repo_settings_get_shared_repository(repo)) { char buf[10]; @@ -2965,29 +2967,7 @@ int init_db(struct repository *repo, repo_config_set(repo, "receive.denyNonFastforwards", "true"); } - if (!(flags & INIT_DB_SKIP_REFDB)) - create_reference_database(repo, initial_branch, flags & INIT_DB_QUIET); - create_object_database(repo); - - startup_info->have_repository = 1; - - if (!(flags & INIT_DB_QUIET)) { - int len = strlen(git_dir); - - if (reinit) - printf(repo_settings_get_shared_repository(repo) - ? _("Reinitialized existing shared Git repository in %s%s\n") - : _("Reinitialized existing Git repository in %s%s\n"), - git_dir, len && git_dir[len-1] != '/' ? "/" : ""); - else - printf(repo_settings_get_shared_repository(repo) - ? _("Initialized empty shared Git repository in %s%s\n") - : _("Initialized empty Git repository in %s%s\n"), - git_dir, len && git_dir[len-1] != '/' ? "/" : ""); - } - clear_repository_format(&repo_fmt); strbuf_release(&err); free(original_git_dir); - return 0; } diff --git a/setup.h b/setup.h index 7394473e954df1..7198e1893035c0 100644 --- a/setup.h +++ b/setup.h @@ -257,24 +257,45 @@ int apply_repository_format(struct repository *repo, const char *get_template_dir(const char *option_template); -#define INIT_DB_QUIET (1 << 0) -#define INIT_DB_EXIST_OK (1 << 1) -#define INIT_DB_SKIP_REFDB (1 << 2) - -int init_db(struct repository *repo, - const char *git_dir, - const char *real_git_dir, - const char *worktree, - const char *template_dir, int hash_algo, - enum ref_storage_format ref_storage_format, - const char *initial_branch, int init_shared_repository, - unsigned int flags); +/* + * Create the repository by creating the necessary directory structures, + * setting up the configuration and configuring the repository's format. If + * `template_dir` is set, copy over templates from that directory. Furthermore, + * if and only if `reinit_ok` is a non-NULL pointer, then the function may + * reinitialize a preexisting repository. In that case, the pointer will be set + * to `1` in case the repo was reinitialized and `0` if it didn't exist yet. + * + * Note that this function does not create the reference and object databases. + */ +void create_repository(struct repository *repo, + const char *git_dir, + const char *real_git_dir, + const char *worktree, + const char *template_dir, + int hash_algo, + enum ref_storage_format ref_storage_format, + int init_shared_repository, + int *reinit_ok); + void initialize_repository_version(struct repository *repo, int hash_algo, enum ref_storage_format ref_storage_format, int reinit); + +/* + * Create the reference database for the repository. The repository and its ref + * storage format must have already been configured properly before calling + * this function. When set, `initial_branch` overrides the default branch that + * HEAD will point to. + */ void create_reference_database(struct repository *repo, const char *initial_branch, int quiet); +/* + * Create the object database for the repository. The repository must have + * already been configured properly before calling this function. + */ +void create_object_database(struct repository *repo); + /* * NOTE NOTE NOTE!! * From d73f69cbe827759e7bd20361a56a57ed62025a35 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:12 +0200 Subject: [PATCH 03/29] builtin/clone: defer setup of the object database When cloning a repository we defer initialization of the reference database. This is because we don't yet know all details required for us to initialize the refdb in the first place. Most importantly, what we are missing is information about the object hash. We don't do the same thing for the object database yet, but here we essentially have the same problem. While the "files" database does not need any information about the object format at creation time, alternate backends are likely to require that information so that they can properly set up their data structures. Besides this forward-looking future proofing though, we also have a second use case for deferring initialization of the object database, namely alternates. When initializing the object database we do not yet know whether we'll need alternates or not because this depends on the repository we're about to clone from. If it is a local repository and the user has passed "--reference{,-if-able}", then we will end up writing alternates into the object database. The ugly part though is that we cannot determine where the repository is getting cloned from before it has been initialized. While we of course already have access to the user-provided URI, that URI can be very well rewritten via "url..insteadOf". We can of course read the global- and system-level configuration to resolve it. But we explicitly resolve the URI a second time after we have initialized the repository because it can happen that we copy a ".git/config" over from our templates, and that file may cause us to rewrite the path. In a subsequent commit though we'll start to write alternates as part of the repository initialization, so we'll need to have the URI properly resolved before we can initialize the object database. This is ugly, but as mentioned above it makes sense for us to defer its initialization anyway so that we also know about the object hash already. Defer creation of the object database until after we have resolved the URI. Note that this also requires us to defer the call to `setup_reference()` until after we have created the object database. While you might think that this function has something to do with references ("refs/*"), it is in fact responsible for setting up alternates. Consequently, we can only call it after we have created the object database already. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/clone.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 76c15e4b814b4e..92c60a89251849 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1188,7 +1188,6 @@ int cmd_clone(int argc, create_repository(the_repository, git_dir, real_git_dir, work_tree, option_template, GIT_HASH_UNKNOWN, ref_storage_format, do_not_override_repo_unix_permissions, NULL); - create_object_database(the_repository); if (real_git_dir) { free((char *)git_dir); @@ -1311,9 +1310,6 @@ int cmd_clone(int argc, strbuf_reset(&key); } - if (option_required_reference.nr || option_optional_reference.nr) - setup_reference(); - remote = remote_get_early(remote_name); if (!option_rev) @@ -1342,6 +1338,10 @@ int cmd_clone(int argc, if (option_local > 0 && !is_local) warning(_("--local is ignored")); + create_object_database(the_repository); + if (option_required_reference.nr || option_optional_reference.nr) + setup_reference(); + transport = transport_get(remote, path ? path : remote->url.v[0]); transport_set_verbosity(transport, option_verbosity, option_progress); transport->family = family; From b781ec0eebd24edf8700dc4edd47551a81e3380a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:13 +0200 Subject: [PATCH 04/29] builtin/clone: move around `setup_reference()` In a subsequent commit, `setup_reference()` will start to call `copy_alternates()`. Prepare for this by moving the function further down so that we can avoid adding a declaration. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/clone.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 92c60a89251849..3565b4b6d6efc2 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -179,16 +179,6 @@ static int add_one_reference(struct string_list_item *item, void *cb_data) return 0; } -static void setup_reference(void) -{ - int required = 1; - for_each_string_list(&option_required_reference, - add_one_reference, &required); - required = 0; - for_each_string_list(&option_optional_reference, - add_one_reference, &required); -} - static void copy_alternates(struct strbuf *src, const char *src_repo) { /* @@ -228,6 +218,16 @@ static void copy_alternates(struct strbuf *src, const char *src_repo) fclose(in); } +static void setup_reference(void) +{ + int required = 1; + for_each_string_list(&option_required_reference, + add_one_reference, &required); + required = 0; + for_each_string_list(&option_optional_reference, + add_one_reference, &required); +} + static void mkdir_if_missing(const char *pathname, mode_t mode) { struct stat st; From e6ea2e28a734ba02847d687b750877d3b6837293 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:14 +0200 Subject: [PATCH 05/29] builtin/clone: refactor handling of "--reference{,-if-able}" Users can pass "--reference{,-if-able}" to git-clone(1) to instruct it to set up alternates for the newly created repository. This allows it to reuse objects from the source repository so that in the best case we don't have to clone all objects over. Those options are handled by the confusingly named `setup_reference()` function -- without the above context, one might rightfully believe that it was about refs, not about alternates. The function itself is rather simple: we loop through all provided alternate paths and then, if such an alternate is valid, we write it to the object database. In subsequent commits we're about to consolidate the complete setup of alternates into this function, and furthermore we'll refactor the setup of the object database to handle doing this for us instead of writing the alternates into it one by one. Prepare for this refactoring by collecting the alternates into a strvec. Rename the function to `collect_alternates()` to clarify its scope, as it does not set up the references itself anymore. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/clone.c | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 3565b4b6d6efc2..67a2d21689f3ad 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -154,24 +154,26 @@ static char *get_repo_path(const char *repo, int *is_bundle) return canon; } -static int add_one_reference(struct string_list_item *item, void *cb_data) +struct collect_alternates_data { + struct strvec *alternates; + bool required; +}; + +static int collect_one_alternate(struct string_list_item *item, void *cb_data) { + struct collect_alternates_data *data = cb_data; struct strbuf err = STRBUF_INIT; - int *required = cb_data; char *ref_git = compute_alternate_path(item->string, &err); if (!ref_git) { - if (*required) + if (data->required) die("%s", err.buf); else fprintf(stderr, _("info: Could not add alternate for '%s': %s\n"), item->string, err.buf); } else { - struct strbuf sb = STRBUF_INIT; - strbuf_addf(&sb, "%s/objects", ref_git); - odb_add_to_alternates_file(the_repository->objects, sb.buf); - strbuf_release(&sb); + strvec_pushf(data->alternates, "%s/objects", ref_git); } strbuf_release(&err); @@ -218,14 +220,20 @@ static void copy_alternates(struct strbuf *src, const char *src_repo) fclose(in); } -static void setup_reference(void) +static void collect_alternates(struct strvec *alternates) { - int required = 1; - for_each_string_list(&option_required_reference, - add_one_reference, &required); - required = 0; - for_each_string_list(&option_optional_reference, - add_one_reference, &required); + if (option_required_reference.nr || option_optional_reference.nr) { + struct collect_alternates_data data = { + .alternates = alternates, + .required = true, + }; + + for_each_string_list(&option_required_reference, + collect_one_alternate, &data); + data.required = false; + for_each_string_list(&option_optional_reference, + collect_one_alternate, &data); + } } static void mkdir_if_missing(const char *pathname, mode_t mode) @@ -999,6 +1007,7 @@ int cmd_clone(int argc, N_("uri"), N_("a URI for downloading bundles before fetching from origin remote")), OPT_END() }; + struct strvec alternates = STRVEC_INIT; const char * const builtin_clone_usage[] = { N_("git clone [] [--] []"), @@ -1339,8 +1348,10 @@ int cmd_clone(int argc, warning(_("--local is ignored")); create_object_database(the_repository); - if (option_required_reference.nr || option_optional_reference.nr) - setup_reference(); + collect_alternates(&alternates); + + for (size_t i = 0; i < alternates.nr; i++) + odb_add_to_alternates_file(the_repository->objects, alternates.v[i]); transport = transport_get(remote, path ? path : remote->url.v[0]); transport_set_verbosity(transport, option_verbosity, option_progress); @@ -1638,6 +1649,7 @@ int cmd_clone(int argc, string_list_clear(&option_not, 0); string_list_clear(&option_config, 0); string_list_clear(&server_options, 0); + strvec_clear(&alternates); free(remote_name); strbuf_release(&reflog_msg); From bbf30b622205604dd3e8a128ae2b2e6c36a2b1f5 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:15 +0200 Subject: [PATCH 06/29] builtin/clone: move setup of alternates for shared local clones When cloning a local repository with "--shared" we add that repository to the new repository's alternates. This is done in `clone_local()`, which is responsible for performing local clones. Move the logic into `collect_alternates()` to unify our setup of alternates. Furthermore, this will allow us to set up alternates right at creation time of the object database. Note that the logic for cloning a local repository with "--no-shared" is not yet part of `collect_alternates()`. This will be handled in the next commit, but means that at this step, we may compute `commondir` without it being used. It will become used in the next step though. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/clone.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 67a2d21689f3ad..85ff69d2261344 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -220,7 +220,8 @@ static void copy_alternates(struct strbuf *src, const char *src_repo) fclose(in); } -static void collect_alternates(struct strvec *alternates) +static void collect_alternates(struct strvec *alternates, + const char *local_source_repo) { if (option_required_reference.nr || option_optional_reference.nr) { struct collect_alternates_data data = { @@ -234,6 +235,16 @@ static void collect_alternates(struct strvec *alternates) for_each_string_list(&option_optional_reference, collect_one_alternate, &data); } + + if (local_source_repo) { + struct strbuf commondir = STRBUF_INIT; + + get_common_dir(&commondir, local_source_repo); + if (option_shared) + strvec_pushf(alternates, "%s/objects", commondir.buf); + + strbuf_release(&commondir); + } } static void mkdir_if_missing(const char *pathname, mode_t mode) @@ -357,13 +368,7 @@ static void copy_or_link_directory(struct strbuf *src, struct strbuf *dest, static void clone_local(const char *src_repo, const char *dest_repo) { - if (option_shared) { - struct strbuf alt = STRBUF_INIT; - get_common_dir(&alt, src_repo); - strbuf_addstr(&alt, "/objects"); - odb_add_to_alternates_file(the_repository->objects, alt.buf); - strbuf_release(&alt); - } else { + if (!option_shared) { struct strbuf src = STRBUF_INIT; struct strbuf dest = STRBUF_INIT; get_common_dir(&src, src_repo); @@ -1348,7 +1353,7 @@ int cmd_clone(int argc, warning(_("--local is ignored")); create_object_database(the_repository); - collect_alternates(&alternates); + collect_alternates(&alternates, is_local ? path : NULL); for (size_t i = 0; i < alternates.nr; i++) odb_add_to_alternates_file(the_repository->objects, alternates.v[i]); From 4f21bd09a2b325882a91a54c46a5d221be6e4d5d Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:16 +0200 Subject: [PATCH 07/29] builtin/clone: move setup of alternates for non-shared local clones Similar as in the preceding commit, move the setup of alternates for local clones with "--no-shared" into `collect_alternates()`. With this step, the complete setup of alternates is now handled by that function. Note that besides moving stuff around, it also fixes a bug: previously, we did not know to resolve the referenced repository's common directory. Consequently, when referencing a worktree we failed to resolve alternates. But as `collect_alternates()` already knows to resolve the commondir for "--local" we can simply reuse this resolved path for our purpose. Add two tests, the first one of which exercises this bug to avoid future regressions. The second test ensures that we properly handle relative alternates for a referenced worktree. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/clone.c | 34 +++++++++++++++++++++++----------- t/t5604-clone-reference.sh | 25 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 85ff69d2261344..8e801224ab944c 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -181,7 +181,7 @@ static int collect_one_alternate(struct string_list_item *item, void *cb_data) return 0; } -static void copy_alternates(struct strbuf *src, const char *src_repo) +static void read_alternates(struct strvec *alternates, const char *src_repo) { /* * Read from the source objects/info/alternates file @@ -195,29 +195,41 @@ static void copy_alternates(struct strbuf *src, const char *src_repo) * to turn entries with paths relative to the original * absolute, so that they can be used in the new repository. */ - FILE *in = xfopen(src->buf, "r"); + FILE *in; + struct strbuf path = STRBUF_INIT; struct strbuf line = STRBUF_INIT; + strbuf_addf(&path, "%s/objects/info/alternates", src_repo); + + in = fopen(path.buf, "r"); + if (!in) { + if (errno == ENOENT) + goto out; + die_errno("could not read alternates file '%s'", path.buf); + } + while (strbuf_getline(&line, in) != EOF) { char *abs_path; if (!line.len || line.buf[0] == '#') continue; if (is_absolute_path(line.buf)) { - odb_add_to_alternates_file(the_repository->objects, - line.buf); + strvec_push(alternates, line.buf); continue; } abs_path = mkpathdup("%s/objects/%s", src_repo, line.buf); if (!normalize_path_copy(abs_path, abs_path)) - odb_add_to_alternates_file(the_repository->objects, - abs_path); + strvec_push(alternates, abs_path); else warning("skipping invalid relative alternate: %s/%s", src_repo, line.buf); free(abs_path); } + +out: + strbuf_release(&path); strbuf_release(&line); - fclose(in); + if (in) + fclose(in); } static void collect_alternates(struct strvec *alternates, @@ -242,6 +254,8 @@ static void collect_alternates(struct strvec *alternates, get_common_dir(&commondir, local_source_repo); if (option_shared) strvec_pushf(alternates, "%s/objects", commondir.buf); + else + read_alternates(alternates, commondir.buf); strbuf_release(&commondir); } @@ -320,11 +334,9 @@ static void copy_or_link_directory(struct strbuf *src, struct strbuf *dest, continue; } - /* Files that cannot be copied bit-for-bit... */ - if (!fspathcmp(iter->relative_path, "info/alternates")) { - copy_alternates(src, src_repo); + /* Alternates were already handled earlier. */ + if (!fspathcmp(iter->relative_path, "info/alternates")) continue; - } if (unlink(dest->buf) && errno != ENOENT) die_errno(_("failed to unlink '%s'"), dest->buf); diff --git a/t/t5604-clone-reference.sh b/t/t5604-clone-reference.sh index 39a0c318dfb2d4..9e4b98fdb82597 100755 --- a/t/t5604-clone-reference.sh +++ b/t/t5604-clone-reference.sh @@ -383,4 +383,29 @@ test_expect_success 'dissociate from repo with commit graph' ' git clone --no-local --reference graph.git --dissociate orig clone ' +test_expect_success 'local clone from linked worktree carries over alternates' ' + rm -fr base derived derived-wt dst expect && + git init base && + test_commit -C base one && + git clone --shared base derived && + git -C derived worktree add ../derived-wt && + git clone derived-wt dst && + echo "$(pwd)/base/.git/objects" >expect && + test_cmp expect dst/.git/objects/info/alternates && + git -C dst fsck +' + +test_expect_success 'local clone from linked worktree resolves relative alternates' ' + rm -fr base derived derived-wt dst expect && + git init base && + test_commit -C base one && + git clone --shared base derived && + echo "../../../base/.git/objects" >derived/.git/objects/info/alternates && + git -C derived worktree add ../derived-wt && + git clone derived-wt dst && + echo "$(pwd)/base/.git/objects" >expect && + test_cmp expect dst/.git/objects/info/alternates && + git -C dst fsck +' + test_done From fdfff05082be9e8b038c1511a7be47db542dc158 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:17 +0200 Subject: [PATCH 08/29] odb/source: support writing alternates when creating the database Add the ability to write alternates when creating the object database. This change allows us to remove the `write_alternates()` callback in a subsequent patch. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-files.c | 76 ++++++++++++++++++++++++++++++++++++++++++++-- odb/source.h | 17 +++++++++-- setup.c | 4 ++- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/odb/source-files.c b/odb/source-files.c index ed1f8e7ecf7dbc..d628aa54ef738b 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -19,6 +19,7 @@ #include "run-command.h" #include "strbuf.h" #include "string-list.h" +#include "strmap.h" #include "strvec.h" #include "tree.h" #include "write-or-die.h" @@ -51,9 +52,14 @@ static void odb_source_files_close(struct odb_source *source) odb_source_close(&files->packed->base); } -static int odb_source_files_create_on_disk(struct odb_source *source) +static int odb_source_files_create_on_disk(struct odb_source *source, + const struct odb_create_on_disk_options *opts) { + struct lock_file alternates_lock = LOCK_INIT; struct strbuf path = STRBUF_INIT; + struct strset seen = STRSET_INIT; + struct strbuf line = STRBUF_INIT; + int ret; safe_create_dir(source->odb->repo, source->path, 1); @@ -64,8 +70,74 @@ static int odb_source_files_create_on_disk(struct odb_source *source) strbuf_addf(&path, "%s/info", source->path); safe_create_dir(source->odb->repo, path.buf, 1); + if (opts->alternates && opts->alternates->nr) { + FILE *alternates, *orig; + + strbuf_reset(&path); + strbuf_addf(&path, "%s/info/alternates", source->path); + + repo_hold_lock_file_for_update(source->odb->repo, &alternates_lock, + path.buf, LOCK_DIE_ON_ERROR); + + alternates = fdopen_lock_file(&alternates_lock, "w"); + if (!alternates) { + ret = error_errno(_("unable to fdopen alternates lockfile")); + goto out; + } + + /* + * The alternates file may already exist, e.g. when it has been + * seeded from a template directory. Read any preexisting + * entries so that we don't end up writing duplicates. + */ + orig = fopen(path.buf, "r"); + if (orig) { + while (strbuf_getline(&line, orig) != EOF) { + strset_add(&seen, line.buf); + fprintf(alternates, "%s\n", line.buf); + } + + if (ferror(orig)) { + ret = error_errno(_("unable to read alternates file")); + fclose(orig); + goto out; + } + + fclose(orig); + } else if (errno != ENOENT) { + ret = error_errno(_("unable to read alternates file")); + goto out; + } + + for (size_t i = 0; i < opts->alternates->nr; i++) { + const char *alternate = opts->alternates->v[i]; + if (!strset_add(&seen, alternate)) + continue; + fprintf(alternates, "%s\n", alternate); + } + + if (ferror(alternates)) { + ret = error_errno(_("unable to write alternates file")); + goto out; + } + + if (commit_lock_file(&alternates_lock)) { + ret = error_errno(_("unable to commit alternates file")); + goto out; + } + } + + /* Reprepare the object database to activate alternates. */ + odb_reprepare(source->odb); + + ret = 0; + +out: + rollback_lock_file(&alternates_lock); + strbuf_release(&line); strbuf_release(&path); - return 0; + strset_clear(&seen); + return ret; } static void odb_source_files_prepare(struct odb_source *source, diff --git a/odb/source.h b/odb/source.h index 8e776f1df25da7..ae6ecef829985e 100644 --- a/odb/source.h +++ b/odb/source.h @@ -37,6 +37,15 @@ struct odb_stream; struct strbuf; struct strvec; +struct odb_create_on_disk_options { + /* + * Alternates that shall be written into the newly created object + * database. Whether or not this option can be handled is specific to + * the backend. + */ + const struct strvec *alternates; +}; + /* * The source is the part of the object database that stores the actual * objects. It thus encapsulates the logic to read and write the specific @@ -107,7 +116,8 @@ struct odb_source { * This callback may be NULL in case the source does not need any * on-disk setup. */ - int (*create_on_disk)(struct odb_source *source); + int (*create_on_disk)(struct odb_source *source, + const struct odb_create_on_disk_options *opts); /* * This callback is expected to prepare the source so that it becomes @@ -390,11 +400,12 @@ static inline void odb_source_close(struct odb_source *source) * Create on-disk data structures that are required for this source to operate * correctly. Returns 0 on success, a negative error code otherwise. */ -static inline int odb_source_create_on_disk(struct odb_source *source) +static inline int odb_source_create_on_disk(struct odb_source *source, + const struct odb_create_on_disk_options *opts) { if (!source->create_on_disk) return 0; - return source->create_on_disk(source); + return source->create_on_disk(source, opts); } /* diff --git a/setup.c b/setup.c index 926e38eabb3d7e..c11a49da343324 100644 --- a/setup.c +++ b/setup.c @@ -2713,6 +2713,8 @@ static int create_default_files(struct repository *repo, void create_object_database(struct repository *repo) { + struct odb_create_on_disk_options opts = { 0 }; + /* * Create the "objects" directory in the common directory. This is done * so that the repository can be discovered regardless of the backend @@ -2732,7 +2734,7 @@ void create_object_database(struct repository *repo) repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); - if (odb_source_create_on_disk(repo->objects->sources) < 0) + if (odb_source_create_on_disk(repo->objects->sources, &opts) < 0) die(_("failed creating object database")); } From 7f2a3605105b93ee04e44a087dd5920772007516 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:18 +0200 Subject: [PATCH 09/29] builtin/clone: write alternates via `odb_create_on_disk()` When creating a repository with alternates we first initialize the object database and then write alternates to it in a separate step. This is unfortunate due to a couple of reasons: - It requires us to have a `write_alternates()` callback, which is unfortunate as we never even write alternates to an object database after it has been created. - We're about to make alternates an implementation detail of the object database's backend in a future patch series, so having this callback is suboptimal there. - The backend has more flexibility with how exactly alternates are configured when it itself is in full control over their setup at the time where it creates the object database itself. We have thus introduced the ability to write alternates right at creation time in the preceding commits, and we have unified setup of alternates into a single location. All that's left to do for us now is to wire up alternates as an option for the database creation. Do so. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/clone.c | 5 +---- builtin/init-db.c | 2 +- setup.c | 7 +++++-- setup.h | 6 ++++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 8e801224ab944c..938d7cd51d017a 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1364,11 +1364,8 @@ int cmd_clone(int argc, if (option_local > 0 && !is_local) warning(_("--local is ignored")); - create_object_database(the_repository); collect_alternates(&alternates, is_local ? path : NULL); - - for (size_t i = 0; i < alternates.nr; i++) - odb_add_to_alternates_file(the_repository->objects, alternates.v[i]); + create_object_database(the_repository, &alternates); transport = transport_get(remote, path ? path : remote->url.v[0]); transport_set_verbosity(transport, option_verbosity, option_progress); diff --git a/builtin/init-db.c b/builtin/init-db.c index f2c7e3be6d2458..5c22eae2f3f2d1 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -251,7 +251,7 @@ int cmd_init_db(int argc, template_dir, hash_algo, ref_storage_format, init_shared_repository, &reinit); create_reference_database(the_repository, initial_branch, quiet); - create_object_database(the_repository); + create_object_database(the_repository, NULL); if (!quiet) { int len = strlen(git_dir); diff --git a/setup.c b/setup.c index c11a49da343324..cb4cb9a5dbe188 100644 --- a/setup.c +++ b/setup.c @@ -2711,9 +2711,12 @@ static int create_default_files(struct repository *repo, return reinit; } -void create_object_database(struct repository *repo) +void create_object_database(struct repository *repo, + const struct strvec *alternates) { - struct odb_create_on_disk_options opts = { 0 }; + struct odb_create_on_disk_options opts = { + .alternates = alternates, + }; /* * Create the "objects" directory in the common directory. This is done diff --git a/setup.h b/setup.h index 7198e1893035c0..9d0c5182119248 100644 --- a/setup.h +++ b/setup.h @@ -292,9 +292,11 @@ void create_reference_database(struct repository *repo, const char *initial_bran /* * Create the object database for the repository. The repository must have - * already been configured properly before calling this function. + * already been configured properly before calling this function. When set, + * `alternates` is the list of alternates that should be written into the + * object database. */ -void create_object_database(struct repository *repo); +void create_object_database(struct repository *repo, const struct strvec *alternates); /* * NOTE NOTE NOTE!! From 9343eea9be7a4ff72480e7435f36c6182d64ec2a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 10 Sep 2026 17:09:19 +0200 Subject: [PATCH 10/29] odb/source: remove the ability to write alternates There are no users of `odb_source_write_alternates()` in our tree anymore. Remove that function and its supporting infrastructure. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- gvfs-helper.c | 6 +++-- gvfs.h | 4 ---- odb.c | 13 ++--------- odb.h | 9 ++++---- odb/source-files.c | 54 ------------------------------------------- odb/source-inmemory.c | 7 ------ odb/source-loose.c | 7 ------ odb/source-packed.c | 7 ------ odb/source.h | 26 --------------------- 9 files changed, 11 insertions(+), 122 deletions(-) diff --git a/gvfs-helper.c b/gvfs-helper.c index abb68914b7b370..de704826c11b55 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -1416,7 +1416,8 @@ static int option_parse_shared_cache_directory(const struct option *opt, gvfs_shared_cache_pathname.buf); mkdir(gvfs_shared_cache_pathname.buf, 0777); - add_gvfs_shared_cache_to_alternates(the_repository->objects, buf_arg.buf); + odb_add_to_alternates_memory(the_repository->objects, + buf_arg.buf); strbuf_release(&buf_arg); return 0; @@ -1438,7 +1439,8 @@ static int option_parse_shared_cache_directory(const struct option *opt, gvfs_shared_cache_pathname.buf); mkdir(gvfs_shared_cache_pathname.buf, 0777); - add_gvfs_shared_cache_to_alternates(the_repository->objects, buf_arg.buf); + odb_add_to_alternates_memory(the_repository->objects, + buf_arg.buf); /* * alt_odb_usable() releases gvfs_shared_cache_pathname diff --git a/gvfs.h b/gvfs.h index f305c5050b8ac0..02fa0040c948cf 100644 --- a/gvfs.h +++ b/gvfs.h @@ -43,8 +43,4 @@ struct repository; int gvfs_config_is_set(struct repository *r, int mask); int gvfs_virtualize_objects(struct repository *r); -struct object_database; -struct odb_source *add_gvfs_shared_cache_to_alternates(struct object_database *odb, - const char *dir); - #endif /* GVFS_H */ diff --git a/odb.c b/odb.c index 7fe6c996aa2141..689fe527a6cdf2 100644 --- a/odb.c +++ b/odb.c @@ -295,21 +295,12 @@ static struct odb_source *odb_add_alternate_recursively(struct object_database * return alternate; } -struct odb_source *add_gvfs_shared_cache_to_alternates(struct object_database *odb, - const char *dir) +struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, + const char *dir) { return odb_add_alternate_recursively(odb, dir, 0); } -void odb_add_to_alternates_file(struct object_database *odb, - const char *dir) -{ - int ret = odb_source_write_alternate(odb->sources, dir); - if (ret < 0) - die(NULL); - odb_add_alternate_recursively(odb, dir, 0); -} - struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, const char *dir, int will_destroy, struct odb_source **prev_source) diff --git a/odb.h b/odb.h index 2f0867c75a9d0c..cecdf38ecf6b7f 100644 --- a/odb.h +++ b/odb.h @@ -285,11 +285,12 @@ int odb_mkstemp(struct object_database *odb, int odb_has_alternates(struct object_database *odb); /* - * Add the directory to the on-disk alternates file; the new entry will also - * take effect in the current process. + * Add the directory to the in-memory list of alternate sources (along with any + * recursive alternates it points to), but do not modify the on-disk alternates + * file. */ -void odb_add_to_alternates_file(struct object_database *odb, - const char *dir); +struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, + const char *dir); /* * Read an object from the database. Returns the object data and assigns object diff --git a/odb/source-files.c b/odb/source-files.c index d628aa54ef738b..68575e9fdf2958 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -326,59 +326,6 @@ static int odb_source_files_read_alternates(struct odb_source *source, return 0; } -static int odb_source_files_write_alternate(struct odb_source *source, - const char *alternate) -{ - struct lock_file lock = LOCK_INIT; - char *path = xstrfmt("%s/%s", source->path, "info/alternates"); - FILE *in, *out; - int found = 0; - int ret; - - repo_hold_lock_file_for_update(source->odb->repo, &lock, path, - LOCK_DIE_ON_ERROR); - out = fdopen_lock_file(&lock, "w"); - if (!out) { - ret = error_errno(_("unable to fdopen alternates lockfile")); - goto out; - } - - in = fopen(path, "r"); - if (in) { - struct strbuf line = STRBUF_INIT; - - while (strbuf_getline(&line, in) != EOF) { - if (!strcmp(alternate, line.buf)) { - found = 1; - break; - } - fprintf_or_die(out, "%s\n", line.buf); - } - - strbuf_release(&line); - fclose(in); - } else if (errno != ENOENT) { - ret = error_errno(_("unable to read alternates file")); - goto out; - } - - if (found) { - rollback_lock_file(&lock); - } else { - fprintf_or_die(out, "%s\n", alternate); - if (commit_lock_file(&lock)) { - ret = error_errno(_("unable to move new alternates file into place")); - goto out; - } - } - - ret = 0; - -out: - free(path); - return ret; -} - static int too_many_loose_objects(struct odb_source_files *files, int limit) { unsigned long loose_count; @@ -1027,7 +974,6 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, files->base.write_object_stream = odb_source_files_write_object_stream; files->base.begin_transaction = odb_source_files_begin_transaction; files->base.read_alternates = odb_source_files_read_alternates; - files->base.write_alternate = odb_source_files_write_alternate; files->base.optimize = odb_source_files_optimize; files->base.optimize_required = odb_source_files_optimize_required; files->base.generate_pack = odb_source_files_generate_pack; diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index ed5db345c957ae..93345666505db3 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -329,12 +329,6 @@ static int odb_source_inmemory_read_alternates(struct odb_source *source UNUSED, return 0; } -static int odb_source_inmemory_write_alternate(struct odb_source *source UNUSED, - const char *alternate UNUSED) -{ - return error("in-memory source does not support alternates"); -} - static void odb_source_inmemory_close(struct odb_source *source UNUSED) { } @@ -398,7 +392,6 @@ struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb) source->base.freshen_object = odb_source_inmemory_freshen_object; source->base.begin_transaction = odb_source_inmemory_begin_transaction; source->base.read_alternates = odb_source_inmemory_read_alternates; - source->base.write_alternate = odb_source_inmemory_write_alternate; return source; } diff --git a/odb/source-loose.c b/odb/source-loose.c index 94d2372669aea0..6806840c237410 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -1002,12 +1002,6 @@ static int odb_source_loose_read_alternates(struct odb_source *source UNUSED, return 0; } -static int odb_source_loose_write_alternate(struct odb_source *source UNUSED, - const char *alternate UNUSED) -{ - return error("loose source does not support alternates"); -} - void odb_source_loose_cache_add_new_oid(struct odb_source *source, const struct object_id *oid) { @@ -1184,7 +1178,6 @@ struct odb_source_loose *odb_source_loose_new(struct object_database *odb, loose->base.write_object_stream = odb_source_loose_write_object_stream; loose->base.begin_transaction = odb_source_loose_begin_transaction; loose->base.read_alternates = odb_source_loose_read_alternates; - loose->base.write_alternate = odb_source_loose_write_alternate; if (!is_absolute_path(loose->base.path)) chdir_notify_register(odb_source_loose_reparent, loose); diff --git a/odb/source-packed.c b/odb/source-packed.c index ef09fde1b91c9b..df1418c9e39256 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -694,12 +694,6 @@ static int odb_source_packed_read_alternates(struct odb_source *source UNUSED, return 0; } -static int odb_source_packed_write_alternate(struct odb_source *source UNUSED, - const char *alternate UNUSED) -{ - return error("packed backend cannot write alternates"); -} - void (*report_garbage)(unsigned seen_bits, const char *path); static void report_helper(const struct string_list *list, @@ -1027,7 +1021,6 @@ struct odb_source_packed *odb_source_packed_new(struct object_database *odb, packed->base.write_object_stream = odb_source_packed_write_object_stream; packed->base.begin_transaction = odb_source_packed_begin_transaction; packed->base.read_alternates = odb_source_packed_read_alternates; - packed->base.write_alternate = odb_source_packed_write_alternate; if (!is_absolute_path(path)) chdir_notify_register(odb_source_packed_reparent, packed); diff --git a/odb/source.h b/odb/source.h index ae6ecef829985e..5a4e502a265462 100644 --- a/odb/source.h +++ b/odb/source.h @@ -292,19 +292,6 @@ struct odb_source { int (*read_alternates)(struct odb_source *source, struct strvec *out); - /* - * This callback is expected to persist the singular alternate passed - * to it into its list of alternates. Any pre-existing alternates are - * expected to remain active. Subsequent calls to `read_alternates` are - * thus expected to yield the pre-existing list of alternates plus the - * newly added alternate appended to its end. - * - * The callback is expected to return 0 on success, a negative error - * code otherwise. - */ - int (*write_alternate)(struct odb_source *source, - const char *alternate); - /* * This callback is expected to optimize the object database source. * Returns 0 on success, a negative error code otherwise. @@ -559,19 +546,6 @@ static inline int odb_source_read_alternates(struct odb_source *source, return source->read_alternates(source, out); } -/* - * Write and persist a new alternate object database source for the given - * source. Any preexisting alternates are expected to stay valid, and the new - * alternate shall be appended to the end of the list. - * - * Returns 0 on success, a negative error code otherwise. - */ -static inline int odb_source_write_alternate(struct odb_source *source, - const char *alternate) -{ - return source->write_alternate(source, alternate); -} - /* * Create a new transaction that can be used to write objects into a temporary * staging area. The objects will only be persisted when the transaction is From 3c4a88b5c4ff69b7c4146750b07f1de0f56fdfb9 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:51 +0200 Subject: [PATCH 11/29] commit-graph: require resolved packfile paths for `stdin_packs` Users can ask git-commit-graph(1) to write a commit graph specifically for a set of packfiles via the "--stdin-packs" option. Those users are expected to pass in relative paths, and those eventually get resolved in `fill_oids_from_packs()`. This ties the logic in "commit-graph.c" to the specific object database source, as the subsystem now needs to assume where a specific packfile is located relative to the source itself. Refactor the logic to instead require the caller to pass in resolved packfiles to untangle that dependency. This also makes the next change easier to implement, where we'll get rid of passing the source to the commit-graph subsystem. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/commit-graph.c | 17 ++++++++++++++--- commit-graph.c | 15 +++++---------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index d62005edc0469c..b5784ad3c7c7f0 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -8,6 +8,7 @@ #include "parse-options.h" #include "commit-graph.h" #include "odb.h" +#include "odb/source.h" #include "progress.h" #include "replace-object.h" #include "strbuf.h" @@ -302,9 +303,19 @@ static int graph_write(int argc, const char **argv, const char *prefix, } if (opts.stdin_packs) { - while (strbuf_getline(&buf, stdin) != EOF) - string_list_append_nodup(&pack_indexes, - strbuf_detach(&buf, NULL)); + struct strbuf packname = STRBUF_INIT; + size_t dirlen; + + strbuf_addf(&packname, "%s/pack/", source->path); + dirlen = packname.len; + + while (strbuf_getline(&buf, stdin) != EOF) { + strbuf_setlen(&packname, dirlen); + strbuf_addbuf(&packname, &buf); + string_list_append(&pack_indexes, packname.buf); + } + + strbuf_release(&packname); } else if (opts.stdin_commits) { oidset_init(&commits, 0); if (opts.progress) diff --git a/commit-graph.c b/commit-graph.c index 25b325a1ee4ac9..6f5922acc0c574 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -1936,12 +1936,8 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, { uint32_t i; struct strbuf progress_title = STRBUF_INIT; - struct strbuf packname = STRBUF_INIT; - int dirlen; int ret = 0; - strbuf_addf(&packname, "%s/pack/", ctx->odb_source->path); - dirlen = packname.len; if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Finding commits for commit graph in %"PRIuMAX" pack", @@ -1954,15 +1950,15 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, } for (i = 0; i < pack_indexes->nr; i++) { struct packed_git *p; - strbuf_setlen(&packname, dirlen); - strbuf_addstr(&packname, pack_indexes->items[i].string); - p = add_packed_git(ctx->r, packname.buf, packname.len, 1); + + p = add_packed_git(ctx->r, pack_indexes->items[i].string, + strlen(pack_indexes->items[i].string), 1); if (!p) { - ret = error(_("error adding pack %s"), packname.buf); + ret = error(_("error adding pack %s"), pack_indexes->items[i].string); goto cleanup; } if (open_pack_index(p)) { - ret = error(_("error opening index for %s"), packname.buf); + ret = error(_("error opening index for %s"), pack_indexes->items[i].string); close_pack(p); free(p); goto cleanup; @@ -1976,7 +1972,6 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, cleanup: stop_progress(&ctx->progress); strbuf_release(&progress_title); - strbuf_release(&packname); return ret; } From 61329f1759cb18df4180f98371ecb49f4e12a04a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:52 +0200 Subject: [PATCH 12/29] commit-graph: stop depending on `struct odb_source` To read or write a commit graph we require access to the repository that the graph should be read from or written for as well as the object directory to derive its location. Instead of passing in these two bits of information explicitly though, we pass in a `struct odb_source`, which carries with it both of these pieces of information. But ultimately, this is somewhat flawed as we now assume that the source even has an on-disk representation, and furthermore we assume that the commit graphs would live in ".git/objects/info". This is true for the "files" backend, but it's not necessarily true for any other backend that we may eventually want to introduce. So eventually, we'll want to evolve the commit-graph subsystem to become agnostic of the backend's layout and let the backend itself decide where to read a commit graph from or where to write it to. We're not there yet to do that switch, but the current design is already causing issues for the intent of this patch series where we want to move alternates into the "files" backend. Convert the subsystem to take a repository plus an object directory path instead. This unblocks moving around alternates, and it's also a step in the right direction for moving commit graphs into the source in a later series. Note that we previously compared the `struct odb_source` pointers of two commit graphs to figure out whether they were located in the same object directory, whereas we now have to compare their paths. Callers may pass in those paths in different forms though, for example relative to the current working directory or as absolute paths. Convert the object directory into an absolute path both when loading and when writing commit graphs so that the comparisons become more robust. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/commit-graph.c | 10 ++-- builtin/commit.c | 2 +- builtin/fetch.c | 4 +- builtin/gc.c | 3 +- builtin/merge.c | 2 +- commit-graph.c | 115 ++++++++++++++++++++----------------- commit-graph.h | 31 +++++++--- t/helper/test-read-graph.c | 3 +- t/t4216-log-bloom.sh | 4 +- 9 files changed, 102 insertions(+), 72 deletions(-) diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index b5784ad3c7c7f0..a986f08a947580 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -104,8 +104,8 @@ static int graph_verify(int argc, const char **argv, const char *prefix, flags |= COMMIT_GRAPH_WRITE_PROGRESS; source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); - graph_name = get_commit_graph_filename(source); - chain_name = get_commit_graph_chain_filename(source); + graph_name = get_commit_graph_filename(source->path); + chain_name = get_commit_graph_chain_filename(source->path); if (open_commit_graph(graph_name, &fd, &st)) opened = OPENED_GRAPH; else if (errno != ENOENT) @@ -123,7 +123,7 @@ static int graph_verify(int argc, const char **argv, const char *prefix, if (opened == OPENED_NONE) return 0; else if (opened == OPENED_GRAPH) - graph = load_commit_graph_one_fd_st(source, fd, &st); + graph = load_commit_graph_one_fd_st(the_repository, source->path, fd, &st); else graph = load_commit_graph_chain_fd_st(the_repository->objects, fd, &st, &incomplete_chain); @@ -297,7 +297,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); if (opts.reachable) { - if (write_commit_graph_reachable(source, flags, &write_opts)) + if (write_commit_graph_reachable(the_repository, source->path, flags, &write_opts)) result = 1; goto cleanup; } @@ -334,7 +334,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, stop_progress(&progress); } - if (write_commit_graph(source, + if (write_commit_graph(the_repository, source->path, opts.stdin_packs ? &pack_indexes : NULL, opts.stdin_commits ? &commits : NULL, flags, diff --git a/builtin/commit.c b/builtin/commit.c index 577755d54ed4fb..3c13d24cb1bff5 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -2218,7 +2218,7 @@ int cmd_commit(int argc, "new index file. Check that disk is not full and quota is\n" "not exceeded, and then \"git restore --staged :/\" to recover.")); - git_test_write_commit_graph_or_die(the_repository->objects->sources); + git_test_write_commit_graph_or_die(the_repository); repo_rerere(the_repository, 0); run_auto_maintenance(the_repository, quiet); diff --git a/builtin/fetch.c b/builtin/fetch.c index 46151a8edb1e25..c02370a9eb03af 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -15,6 +15,7 @@ #include "refspec.h" #include "object-name.h" #include "odb.h" +#include "odb/source.h" #include "oidset.h" #include "oid-array.h" #include "commit.h" @@ -2919,7 +2920,8 @@ int cmd_fetch(int argc, commit_graph_flags |= COMMIT_GRAPH_WRITE_PROGRESS; trace2_region_enter("fetch", "write-commit-graph", the_repository); - write_commit_graph_reachable(the_repository->objects->sources, + write_commit_graph_reachable(the_repository, + the_repository->objects->sources->path, commit_graph_flags, NULL); trace2_region_leave("fetch", "write-commit-graph", the_repository); diff --git a/builtin/gc.c b/builtin/gc.c index 99d6424eff54ef..bd122b1e878074 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -745,7 +745,8 @@ int cmd_gc(int argc, } if (the_repository->settings.gc_write_commit_graph == 1) - write_commit_graph_reachable(the_repository->objects->sources, + write_commit_graph_reachable(the_repository, + the_repository->objects->sources->path, !opts.quiet && !daemonized ? COMMIT_GRAPH_WRITE_PROGRESS : 0, NULL); diff --git a/builtin/merge.c b/builtin/merge.c index 02a87430f48f97..839f5ef31d223c 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -1864,7 +1864,7 @@ int cmd_merge(int argc, if (squash) { finish(head_commit, remoteheads, NULL, NULL); - git_test_write_commit_graph_or_die(the_repository->objects->sources); + git_test_write_commit_graph_or_die(the_repository); } else write_merge_state(remoteheads); diff --git a/commit-graph.c b/commit-graph.c index 6f5922acc0c574..2cd81419d575d4 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -1,6 +1,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "abspath.h" #include "config.h" #include "csum-file.h" #include "environment.h" @@ -28,7 +29,7 @@ #include "tree.h" #include "chunk-format.h" -void git_test_write_commit_graph_or_die(struct odb_source *source) +void git_test_write_commit_graph_or_die(struct repository *repo) { int flags = 0; if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0)) @@ -37,7 +38,7 @@ void git_test_write_commit_graph_or_die(struct odb_source *source) if (git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags = COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - if (write_commit_graph_reachable(source, flags, NULL)) + if (write_commit_graph_reachable(repo, repo->objects->sources->path, flags, NULL)) die("failed to write commit-graph under GIT_TEST_COMMIT_GRAPH"); } @@ -196,21 +197,21 @@ static int commit_gen_cmp(const void *va, const void *vb) return 0; } -char *get_commit_graph_filename(struct odb_source *source) +char *get_commit_graph_filename(const char *dir) { - return xstrfmt("%s/info/commit-graph", source->path); + return xstrfmt("%s/info/commit-graph", dir); } -static char *get_split_graph_filename(struct odb_source *source, +static char *get_split_graph_filename(const char *dir, const char *oid_hex) { - return xstrfmt("%s/info/commit-graphs/graph-%s.graph", source->path, + return xstrfmt("%s/info/commit-graphs/graph-%s.graph", dir, oid_hex); } -char *get_commit_graph_chain_filename(struct odb_source *source) +char *get_commit_graph_chain_filename(const char *dir) { - return xstrfmt("%s/info/commit-graphs/commit-graph-chain", source->path); + return xstrfmt("%s/info/commit-graphs/commit-graph-chain", dir); } static struct commit_graph *alloc_commit_graph(void) @@ -253,7 +254,8 @@ int open_commit_graph(const char *graph_file, int *fd, struct stat *st) return 1; } -struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, +struct commit_graph *load_commit_graph_one_fd_st(struct repository *repo, + const char *dir, int fd, struct stat *st) { void *graph_map; @@ -262,7 +264,7 @@ struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, graph_size = xsize_t(st->st_size); - if (graph_size < graph_min_size(source->odb->repo->hash_algo)) { + if (graph_size < graph_min_size(repo->hash_algo)) { close(fd); error(_("commit-graph file is too small")); return NULL; @@ -270,9 +272,9 @@ struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - ret = parse_commit_graph(source->odb->repo, graph_map, graph_size); + ret = parse_commit_graph(repo, graph_map, graph_size); if (ret) - ret->odb_source = source; + ret->dir = absolute_pathdup(dir); else munmap(graph_map, graph_size); @@ -410,6 +412,7 @@ struct commit_graph *parse_commit_graph(struct repository *r, graph = alloc_commit_graph(); + graph->repo = r; graph->hash_algo = r->hash_algo; graph->num_chunks = *(unsigned char*)(data + 6); graph->data = graph_map; @@ -490,7 +493,8 @@ struct commit_graph *parse_commit_graph(struct repository *r, return NULL; } -static struct commit_graph *load_commit_graph_one(struct odb_source *source, +static struct commit_graph *load_commit_graph_one(struct repository *repo, + const char *dir, const char *graph_file) { struct stat st; @@ -501,17 +505,18 @@ static struct commit_graph *load_commit_graph_one(struct odb_source *source, if (!open_ok) return NULL; - g = load_commit_graph_one_fd_st(source, fd, &st); + g = load_commit_graph_one_fd_st(repo, dir, fd, &st); if (g) - g->filename = xstrdup(graph_file); + g->filename = absolute_pathdup(graph_file); return g; } -static struct commit_graph *load_commit_graph_v1(struct odb_source *source) +static struct commit_graph *load_commit_graph_v1(struct repository *repo, + const char *dir) { - char *graph_name = get_commit_graph_filename(source); - struct commit_graph *g = load_commit_graph_one(source, graph_name); + char *graph_name = get_commit_graph_filename(dir); + struct commit_graph *g = load_commit_graph_one(repo, dir, graph_name); free(graph_name); return g; @@ -666,8 +671,8 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, valid = 0; for (source = odb->sources; source; source = source->next) { - char *graph_name = get_split_graph_filename(source, line.buf); - struct commit_graph *g = load_commit_graph_one(source, graph_name); + char *graph_name = get_split_graph_filename(source->path, line.buf); + struct commit_graph *g = load_commit_graph_one(odb->repo, source->path, graph_name); free(graph_name); @@ -700,29 +705,31 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, return graph_chain; } -static struct commit_graph *load_commit_graph_chain(struct odb_source *source) +static struct commit_graph *load_commit_graph_chain(struct repository *repo, + const char *dir) { - char *chain_file = get_commit_graph_chain_filename(source); + char *chain_file = get_commit_graph_chain_filename(dir); struct stat st; int fd; struct commit_graph *g = NULL; - if (open_commit_graph_chain(chain_file, &fd, &st, source->odb->repo->hash_algo)) { + if (open_commit_graph_chain(chain_file, &fd, &st, repo->hash_algo)) { int incomplete; /* ownership of fd is taken over by load function */ - g = load_commit_graph_chain_fd_st(source->odb, fd, &st, &incomplete); + g = load_commit_graph_chain_fd_st(repo->objects, fd, &st, &incomplete); } free(chain_file); return g; } -struct commit_graph *read_commit_graph_one(struct odb_source *source) +struct commit_graph *read_commit_graph_one(struct repository *repo, + const char *dir) { - struct commit_graph *g = load_commit_graph_v1(source); + struct commit_graph *g = load_commit_graph_v1(repo, dir); if (!g) - g = load_commit_graph_chain(source); + g = load_commit_graph_chain(repo, dir); return g; } @@ -767,7 +774,7 @@ static struct commit_graph *prepare_commit_graph(struct repository *r) return NULL; for (source = r->objects->sources; source; source = source->next) { - r->objects->commit_graph = read_commit_graph_one(source); + r->objects->commit_graph = read_commit_graph_one(r, source->path); if (r->objects->commit_graph) break; } @@ -866,7 +873,7 @@ static struct commit_list **insert_parent_or_die(struct commit_graph *g, die("invalid parent position %"PRIu32, pos); load_oid_from_graph(g, pos, &oid); - c = lookup_commit(g->odb_source->odb->repo, &oid); + c = lookup_commit(g->repo, &oid); if (!c) die(_("could not find commit %s"), oid_to_hex(&oid)); commit_graph_data_at(c)->graph_pos = pos; @@ -1103,7 +1110,7 @@ static struct tree *load_tree_for_commit(struct commit_graph *g, graph_pos - g->num_commits_in_base); oidread(&oid, commit_data, g->hash_algo); - set_commit_tree(c, lookup_tree(g->odb_source->odb->repo, &oid)); + set_commit_tree(c, lookup_tree(g->repo, &oid)); return c->maybe_tree; } @@ -1126,7 +1133,7 @@ struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit struct write_commit_graph_context { struct repository *r; - struct odb_source *odb_source; + char *dir; char *graph_name; struct oid_array oids; struct commit_stack commits; @@ -1902,7 +1909,8 @@ static int add_ref_to_set(const struct reference *ref, void *cb_data) return 0; } -int write_commit_graph_reachable(struct odb_source *source, +int write_commit_graph_reachable(struct repository *repo, + const char *dir, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts) { @@ -1911,20 +1919,20 @@ int write_commit_graph_reachable(struct odb_source *source, int result; memset(&data, 0, sizeof(data)); - data.repo = source->odb->repo; + data.repo = repo; data.commits = &commits; if (flags & COMMIT_GRAPH_WRITE_PROGRESS) data.progress = start_delayed_progress( - source->odb->repo, + repo, _("Collecting referenced commits"), 0); - refs_for_each_ref(get_main_ref_store(source->odb->repo), add_ref_to_set, + refs_for_each_ref(get_main_ref_store(repo), add_ref_to_set, &data); stop_progress(&data.progress); - result = write_commit_graph(source, NULL, &commits, + result = write_commit_graph(repo, dir, NULL, &commits, flags, opts); oidset_clear(&commits); @@ -2103,10 +2111,10 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) strbuf_addf(&tmp_file, "%s/info/commit-graphs/tmp_graph_XXXXXX", - ctx->odb_source->path); + ctx->dir); ctx->graph_name = strbuf_detach(&tmp_file, NULL); } else { - ctx->graph_name = get_commit_graph_filename(ctx->odb_source); + ctx->graph_name = get_commit_graph_filename(ctx->dir); } if (safe_create_leading_directories(ctx->r, ctx->graph_name)) { @@ -2116,7 +2124,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) } if (ctx->split) { - char *lock_name = get_commit_graph_chain_filename(ctx->odb_source); + char *lock_name = get_commit_graph_chain_filename(ctx->dir); repo_hold_lock_file_for_update_mode(ctx->r, &lk, lock_name, LOCK_DIE_ON_ERROR, 0444); @@ -2205,7 +2213,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) if (ctx->split && ctx->base_graph_name && ctx->num_commit_graphs_after > 1) { char *new_base_hash = xstrdup(oid_to_hex(&ctx->new_base_graph->oid)); - char *new_base_name = get_split_graph_filename(ctx->new_base_graph->odb_source, new_base_hash); + char *new_base_name = get_split_graph_filename(ctx->new_base_graph->dir, new_base_hash); free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2]); free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2]); @@ -2245,7 +2253,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) } } } else { - char *graph_name = get_commit_graph_filename(ctx->odb_source); + char *graph_name = get_commit_graph_filename(ctx->dir); unlink(graph_name); free(graph_name); } @@ -2253,7 +2261,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]); ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1] = xstrdup(hash_to_hex_algop(file_hash, ctx->r->hash_algo)); - final_graph_name = get_split_graph_filename(ctx->odb_source, + final_graph_name = get_split_graph_filename(ctx->dir, ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]); free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1]); ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1] = final_graph_name; @@ -2305,7 +2313,7 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx, flags != COMMIT_GRAPH_SPLIT_REPLACE) { while (g && (g->num_commits <= st_mult(size_mult, num_commits) || (max_commits && num_commits > max_commits))) { - if (g->odb_source != ctx->odb_source) + if (strcmp(g->dir, ctx->dir)) break; if (unsigned_add_overflows(num_commits, g->num_commits)) @@ -2327,10 +2335,10 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx, "should be 1 with --split=replace"); if (ctx->num_commit_graphs_after == 2) { - char *old_graph_name = get_commit_graph_filename(g->odb_source); + char *old_graph_name = get_commit_graph_filename(g->dir); if (!strcmp(g->filename, old_graph_name) && - g->odb_source != ctx->odb_source) { + strcmp(g->dir, ctx->dir)) { ctx->num_commit_graphs_after = 1; ctx->new_base_graph = NULL; } @@ -2500,13 +2508,13 @@ static void expire_commit_graphs(struct write_commit_graph_context *ctx) if (ctx->opts && ctx->opts->expire_time) expire_time = ctx->opts->expire_time; if (!ctx->split) { - char *chain_file_name = get_commit_graph_chain_filename(ctx->odb_source); + char *chain_file_name = get_commit_graph_chain_filename(ctx->dir); unlink(chain_file_name); free(chain_file_name); ctx->num_commit_graphs_after = 0; } - strbuf_addstr(&path, ctx->odb_source->path); + strbuf_addstr(&path, ctx->dir); strbuf_addstr(&path, "/info/commit-graphs"); dir = opendir(path.buf); @@ -2548,16 +2556,15 @@ static void expire_commit_graphs(struct write_commit_graph_context *ctx) strbuf_release(&path); } -int write_commit_graph(struct odb_source *source, +int write_commit_graph(struct repository *r, + const char *dir, const struct string_list *const pack_indexes, struct oidset *commits, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts) { - struct repository *r = source->odb->repo; struct write_commit_graph_context ctx = { .r = r, - .odb_source = source, .append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0, .report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0, .split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0, @@ -2588,6 +2595,8 @@ int write_commit_graph(struct odb_source *source, return 0; } + ctx.dir = absolute_pathdup(dir); + bloom_settings.hash_version = r->settings.commit_graph_changed_paths_version; bloom_settings.bits_per_entry = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY", bloom_settings.bits_per_entry); @@ -2711,6 +2720,7 @@ int write_commit_graph(struct odb_source *source, cleanup: free(ctx.graph_name); free(ctx.base_graph_name); + free(ctx.dir); commit_stack_clear(&ctx.commits); oid_array_clear(&ctx.oids); clear_topo_level_slab(&topo_levels); @@ -2763,7 +2773,7 @@ static int verify_one_commit_graph(struct commit_graph *g, struct progress *progress, uint64_t *seen) { - struct repository *r = g->odb_source->odb->repo; + struct repository *r = g->repo; uint32_t i, cur_fanout_pos = 0; struct object_id prev_oid, cur_oid; struct commit *seen_gen_zero = NULL; @@ -2932,7 +2942,7 @@ int verify_commit_graph(struct commit_graph *g, int flags) if (!(flags & COMMIT_GRAPH_VERIFY_SHALLOW)) total += g->num_commits_in_base; - progress = start_progress(g->odb_source->odb->repo, + progress = start_progress(g->repo, _("Verifying commits in commit graph"), total); } @@ -2955,6 +2965,7 @@ void free_commit_graph(struct commit_graph *g) if (g->data) munmap((void *)g->data, g->data_len); + free(g->dir); free(g->filename); free(g->bloom_filter_settings); free(g); diff --git a/commit-graph.h b/commit-graph.h index 13ca4ff010fa18..bccf9c5c844832 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -21,7 +21,7 @@ * call this method outside of a builtin, and only if you know what * you are doing! */ -void git_test_write_commit_graph_or_die(struct odb_source *source); +void git_test_write_commit_graph_or_die(struct repository *repo); struct commit; struct bloom_filter_settings; @@ -29,8 +29,8 @@ struct repository; struct object_database; struct string_list; -char *get_commit_graph_filename(struct odb_source *source); -char *get_commit_graph_chain_filename(struct odb_source *source); +char *get_commit_graph_filename(const char *dir); +char *get_commit_graph_chain_filename(const char *dir); int open_commit_graph(const char *graph_file, int *fd, struct stat *st); int open_commit_graph_chain(const char *chain_file, int *fd, struct stat *st, const struct git_hash_algo *hash_algo); @@ -85,12 +85,13 @@ struct commit_graph { const unsigned char *data; size_t data_len; + struct repository *repo; const struct git_hash_algo *hash_algo; unsigned char num_chunks; uint32_t num_commits; struct object_id oid; + char *dir; char *filename; - struct odb_source *odb_source; uint32_t num_commits_in_base; unsigned int read_generation_data; @@ -114,12 +115,20 @@ struct commit_graph { struct bloom_filter_settings *bloom_filter_settings; }; -struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, +/* + * Load commit graphs from the given object directory `dir`. The directory may + * be given as a relative path; it is canonicalized internally so that graphs + * loaded from the same directory compare equal regardless of how the caller + * spelled the path. + */ +struct commit_graph *load_commit_graph_one_fd_st(struct repository *repo, + const char *dir, int fd, struct stat *st); struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, int fd, struct stat *st, int *incomplete_chain); -struct commit_graph *read_commit_graph_one(struct odb_source *source); +struct commit_graph *read_commit_graph_one(struct repository *repo, + const char *dir); struct repo_settings; @@ -171,11 +180,17 @@ struct commit_graph_opts { * and a negative value on failure. Note that if the repository * is not compatible with the commit-graph feature, then the * methods will return 0 without writing a commit-graph. + * + * The object directory `dir` may be given as a relative path; it is + * canonicalized internally so that it compares equal to the directory of + * graphs that have already been loaded. */ -int write_commit_graph_reachable(struct odb_source *source, +int write_commit_graph_reachable(struct repository *repo, + const char *dir, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts); -int write_commit_graph(struct odb_source *source, +int write_commit_graph(struct repository *r, + const char *dir, const struct string_list *pack_indexes, struct oidset *commits, enum commit_graph_write_flags flags, diff --git a/t/helper/test-read-graph.c b/t/helper/test-read-graph.c index 9f07b9c25a66e3..a75c817e472c7a 100644 --- a/t/helper/test-read-graph.c +++ b/t/helper/test-read-graph.c @@ -4,6 +4,7 @@ #include "commit-graph.h" #include "repository.h" #include "odb.h" +#include "odb/source.h" #include "bloom.h" #include "setup.h" @@ -81,7 +82,7 @@ int cmd__read_graph(int argc, const char **argv) prepare_repo_settings(the_repository); - graph = read_commit_graph_one(source); + graph = read_commit_graph_one(the_repository, source->path); if (!graph) { ret = 1; goto done; diff --git a/t/t4216-log-bloom.sh b/t/t4216-log-bloom.sh index ad2686669d4b30..f57a3d66215c3f 100755 --- a/t/t4216-log-bloom.sh +++ b/t/t4216-log-bloom.sh @@ -755,7 +755,7 @@ test_expect_success PERL_TEST_HELPERS 'Bloom reader notices too-small data chunk test_expect_success PERL_TEST_HELPERS 'Bloom reader notices out-of-bounds filter offsets' ' check_corrupt_graph BIDX 12 FFFFFFFF && # use grep to avoid depending on exact chunk size - test_grep "warning: ignoring out-of-range offset (4294967295) for changed-path filter at pos 3 of .git/objects/info/commit-graph" err + test_grep "warning: ignoring out-of-range offset (4294967295) for changed-path filter at pos 3 of $(pwd)/.git/objects/info/commit-graph" err ' test_expect_success PERL_TEST_HELPERS 'Bloom reader notices too-small index chunk' ' @@ -773,7 +773,7 @@ test_expect_success PERL_TEST_HELPERS 'Bloom reader notices out-of-order index o # actually reading from the bogus offsets anyway. corrupt_graph BIDX 4 0000000c00000005 && echo "warning: ignoring decreasing changed-path index offsets" \ - "(12 > 5) for positions 1 and 2 of .git/objects/info/commit-graph" >expect.err && + "(12 > 5) for positions 1 and 2 of $(pwd)/.git/objects/info/commit-graph" >expect.err && git -c core.commitGraph=false log -- A/B/file2 >expect.out && git -c core.commitGraph=true log -- A/B/file2 >out 2>err && test_cmp expect.out out && From e7662ea1aa21983aa29677152bed51af71333580 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:53 +0200 Subject: [PATCH 13/29] odb/source-files: introduce `struct odb_files_dir` The "files" object database source consists of two separate backends for storing loose and packed objects. These are managed as somewhat separate entities even though they derive from the same object directory, whether it's the primary object directory or that one from an alternate. In a subsequent commit we'll move the handling of alternates into the "files" backend completely so that it becomes another implementation detail thereof. As part of this, we'll want to keep track of both of these sub-sources as a single entity derived from their respective object directory. Prepare for this change by introducing a new `struct odb_files_dir` that encapsulates them. For now, every "files" source has exactly one such directory. In a subsequent commit though, we'll make it a linked list of directories so that we can manage multiple such directories in a single "files" source. Assisted-by: GPT-6.1 Sol Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano Signed-off-by: Johannes Schindelin --- builtin/fast-import.c | 6 +-- builtin/index-pack.c | 2 +- builtin/multi-pack-index.c | 14 +++---- builtin/pack-objects.c | 18 ++++----- builtin/repack.c | 4 +- commit-graph.c | 2 +- gvfs-helper-client.c | 3 +- http-walker.c | 2 +- http.c | 6 +-- loose.c | 6 +-- midx.c | 26 ++++++------- odb/source-files.c | 77 +++++++++++++++++++++++++------------- odb/source-files.h | 27 ++++++++++++- odb/source-loose.c | 2 +- pack-bitmap.c | 2 +- packfile.c | 8 ++-- packfile.h | 4 +- repack-geometry.c | 2 +- repack-midx.c | 6 +-- repack.c | 2 +- t/helper/test-read-midx.c | 2 +- 21 files changed, 134 insertions(+), 87 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 921f9a6a6c2865..32cc302f130461 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -921,7 +921,7 @@ static void end_packfile(void) idx_name = keep_pack(create_index()); /* Register the packfile with core git's machinery. */ - new_p = packfile_store_load_pack(files->packed, idx_name, 1); + new_p = packfile_store_load_pack(files->dirs->packed, idx_name, 1); if (!new_p) die(_("core Git rejected index %s"), idx_name); all_packs[pack_id] = new_p; @@ -1005,7 +1005,7 @@ static int store_object( for (source = the_repository->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid)) + if (!packfile_list_find_oid(packfile_store_get_packs(files->dirs->packed), &oid)) continue; e->type = type; e->pack_id = MAX_PACK_ID; @@ -1212,7 +1212,7 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) for (source = the_repository->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid)) + if (!packfile_list_find_oid(packfile_store_get_packs(files->dirs->packed), &oid)) continue; e->type = OBJ_BLOB; e->pack_id = MAX_PACK_ID; diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 5028a92f069877..83855817712d84 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1641,7 +1641,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name, if (do_fsck_object && startup_info->have_repository) { struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); - packfile_store_load_pack(files->packed, final_index_name, 0); + packfile_store_load_pack(files->dirs->packed, final_index_name, 0); } if (!from_stdin) { diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 753bd53a70fccc..a170ec80b99cff 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -213,7 +213,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, read_packs_from_stdin(&packs); - ret = write_midx_file_only(source->packed, &packs, + ret = write_midx_file_only(source->dirs->packed, &packs, opts.preferred_pack, opts.refs_snapshot, opts.incremental_base, opts.flags); @@ -225,7 +225,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } - ret = write_midx_file(source->packed, opts.preferred_pack, + ret = write_midx_file(source->dirs->packed, opts.preferred_pack, opts.refs_snapshot, opts.flags); free(opts.refs_snapshot); @@ -284,7 +284,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, FREE_AND_NULL(options); - m = get_multi_pack_index(source->packed); + m = get_multi_pack_index(source->dirs->packed); for (cur = m; cur && !(from_midx && to_midx); cur = cur->base_midx) { const char *midx_csum = midx_get_checksum_hex(cur); @@ -307,7 +307,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, die(_("MIDX %s must be an ancestor of %s"), argv[0], argv[1]); } - ret = write_midx_file_compact(source->packed, from_midx, to_midx, + ret = write_midx_file_compact(source->dirs->packed, from_midx, to_midx, opts.incremental_base, opts.flags); return ret; @@ -339,7 +339,7 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, FREE_AND_NULL(options); - return verify_midx_file(source->packed, opts.flags); + return verify_midx_file(source->dirs->packed, opts.flags); } static int cmd_multi_pack_index_expire(int argc, const char **argv, @@ -368,7 +368,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, FREE_AND_NULL(options); - return expire_midx_packs(source->packed, opts.flags); + return expire_midx_packs(source->dirs->packed, opts.flags); } static int cmd_multi_pack_index_repack(int argc, const char **argv, @@ -400,7 +400,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, FREE_AND_NULL(options); - return midx_repack(source->packed, (size_t)opts.batch_size, opts.flags); + return midx_repack(source->dirs->packed, (size_t)opts.batch_size, opts.flags); } int cmd_multi_pack_index(int argc, diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 438bdb359bdddf..61a82536b4a436 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1564,7 +1564,7 @@ static int want_cruft_object_mtime(struct repository *r, for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - struct packed_git **cache = packfile_store_get_kept_pack_cache(files->packed, flags); + struct packed_git **cache = packfile_store_get_kept_pack_cache(files->dirs->packed, flags); for (; *cache; cache++) { struct packed_git *p = *cache; @@ -1759,7 +1759,7 @@ static int want_object_in_pack_mtime(const struct object_id *oid, struct odb_source *source = the_repository->objects->sources->next; for (; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0, NULL)) + if (!odb_source_read_object_info(&files->dirs->loose->base, oid, NULL, 0, NULL)) return 0; } } @@ -1781,7 +1781,7 @@ static int want_object_in_pack_mtime(const struct object_id *oid, for (source = the_repository->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - struct multi_pack_index *m = get_multi_pack_index(files->packed); + struct multi_pack_index *m = get_multi_pack_index(files->dirs->packed); struct pack_entry e; if (m && midx_fill_entry(m, oid, &e, NULL) == MIDX_FILL_HIT) { @@ -1794,11 +1794,11 @@ static int want_object_in_pack_mtime(const struct object_id *oid, for (source = the_repository->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - for (e = files->packed->packs.head; e; e = e->next) { + for (e = files->dirs->packed->packs.head; e; e = e->next) { struct packed_git *p = e->pack; want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime); if (!exclude && want > 0) - packfile_list_prepend(&files->packed->packs, p); + packfile_list_prepend(&files->dirs->packed->packs, p); if (want != -1) return want; } @@ -4160,7 +4160,7 @@ static void add_cruft_object_entry(const struct object_id *oid, enum object_type for (; !found && source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0, NULL)) + if (!odb_source_read_object_info(&files->dirs->loose->base, oid, NULL, 0, NULL)) found = 1; } @@ -4517,7 +4517,7 @@ static void add_objects_in_unpacked_packs(void) if (!source->local) continue; - if (odb_source_for_each_object(&files->packed->base, &oi, + if (odb_source_for_each_object(&files->dirs->packed->base, &oi, add_object_in_unpacked_pack, NULL, &opts)) die(_("cannot open pack index")); } @@ -4627,7 +4627,7 @@ static int force_object_loose(struct odb_source *source, for (struct odb_source *s = source->odb->sources; s; s = s->next) { struct odb_source_files *files = odb_source_files_downcast(s); - if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0, NULL)) + if (!odb_source_read_object_info(&files->dirs->loose->base, oid, NULL, 0, NULL)) return 0; } @@ -4649,7 +4649,7 @@ static int force_object_loose(struct odb_source *source, compat_oid_p = &compat_oid; } - ret = odb_source_write_object(&files->loose->base, buf, len, type, oid, + ret = odb_source_write_object(&files->dirs->loose->base, buf, len, type, oid, compat_oid_p, mtime, 0); out: diff --git a/builtin/repack.c b/builtin/repack.c index a8652e2d6d447e..4686fe12006125 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -623,7 +623,7 @@ int cmd_repack(int argc, * midx_has_unknown_packs() will make the decision for * us. */ - if (!get_multi_pack_index(files->packed)) + if (!get_multi_pack_index(files->dirs->packed)) midx_must_contain_cruft = 1; } @@ -781,7 +781,7 @@ int cmd_repack(int argc, if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(files->packed, NULL, NULL, flags); + write_midx_file(files->dirs->packed, NULL, NULL, flags); } cleanup: diff --git a/commit-graph.c b/commit-graph.c index 2cd81419d575d4..7bcb1aa845848c 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -2020,7 +2020,7 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) for (source = ctx->r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_for_each_object(&files->packed->base, &oi, add_packed_commits_oi, + odb_source_for_each_object(&files->dirs->packed->base, &oi, add_packed_commits_oi, ctx, &opts); } diff --git a/gvfs-helper-client.c b/gvfs-helper-client.c index 79731321386944..0d88cd65230f30 100644 --- a/gvfs-helper-client.c +++ b/gvfs-helper-client.c @@ -272,7 +272,8 @@ static int gh_client__objects__receive_response( if (ghc & GHC__CREATED__PACKFILE) { struct odb_source_files *files = odb_source_files_downcast(gh_client__chosen_odb); - odb_source_prepare(&files->packed->base, ODB_PREPARE_FLUSH_CACHES); + odb_source_prepare(&files->dirs->packed->base, + ODB_PREPARE_FLUSH_CACHES); } *p_ghc = ghc; diff --git a/http-walker.c b/http-walker.c index abafca84d65441..0a6c99f47190ba 100644 --- a/http-walker.c +++ b/http-walker.c @@ -542,7 +542,7 @@ static int fetch_object(struct walker *walker, const struct object_id *oid) } else if (req->rename < 0) { struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); struct strbuf buf = STRBUF_INIT; - odb_loose_path(files->loose, &buf, &req->oid); + odb_loose_path(files->dirs->loose, &buf, &req->oid); ret = error("unable to write sha1 filename %s", buf.buf); strbuf_release(&buf); } diff --git a/http.c b/http.c index 67dcf67e8372f4..c433921fba5ca3 100644 --- a/http.c +++ b/http.c @@ -2852,7 +2852,7 @@ void http_install_packfile(struct packed_git *p, { struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); packfile_list_remove(list_to_remove_from, p); - packfile_store_add_pack(files->packed, p); + packfile_store_add_pack(files->dirs->packed, p); } struct http_pack_request *new_http_pack_request( @@ -2993,7 +2993,7 @@ struct http_object_request *new_http_object_request(const char *base_url, oidcpy(&freq->oid, oid); freq->localfile = -1; - odb_loose_path(files->loose, &filename, oid); + odb_loose_path(files->dirs->loose, &filename, oid); strbuf_addf(&freq->tmpfile, "%s.temp", filename.buf); strbuf_addf(&prevfile, "%s.prev", filename.buf); @@ -3146,7 +3146,7 @@ int finish_http_object_request(struct http_object_request *freq) unlink_or_warn(freq->tmpfile.buf); return -1; } - odb_loose_path(files->loose, &filename, &freq->oid); + odb_loose_path(files->dirs->loose, &filename, &freq->oid); freq->rename = finalize_object_file(the_repository, freq->tmpfile.buf, filename.buf); strbuf_release(&filename); diff --git a/loose.c b/loose.c index c159d29d2d2035..957bf83e6b09fd 100644 --- a/loose.c +++ b/loose.c @@ -117,7 +117,7 @@ int repo_read_loose_object_map(struct repository *repo) for (source = repo->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (loose_object_map_load(files->loose) < 0) + if (loose_object_map_load(files->dirs->loose) < 0) return -1; } @@ -127,7 +127,7 @@ int repo_read_loose_object_map(struct repository *repo) int repo_write_loose_object_map(struct repository *repo) { struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources); - kh_oid_map_t *map = files->loose->map->to_compat; + kh_oid_map_t *map = files->dirs->loose->map->to_compat; struct lock_file lock; int fd; khiter_t iter; @@ -237,7 +237,7 @@ int repo_loose_object_map_oid(struct repository *repo, for (source = repo->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - struct loose_object_map *loose_map = files->loose->map; + struct loose_object_map *loose_map = files->dirs->loose->map; if (!loose_map) continue; map = (to == repo->compat_hash_algo) ? diff --git a/midx.c b/midx.c index 6d1c548e3dae8e..c0f82c41633bc0 100644 --- a/midx.c +++ b/midx.c @@ -837,20 +837,20 @@ void clear_midx_file(struct repository *r) for (source = r->objects->sources; source; source = source->next) { files = odb_source_files_downcast(source); - if (files->packed->midx) - close_midx(files->packed->midx); - files->packed->midx = NULL; + if (files->dirs->packed->midx) + close_midx(files->dirs->packed->midx); + files->dirs->packed->midx = NULL; } } files = odb_source_files_downcast(r->objects->sources); - get_midx_filename(files->packed, &midx); + get_midx_filename(files->dirs->packed, &midx); if (remove_path(midx.buf)) die(_("failed to clear multi-pack-index at %s"), midx.buf); - clear_midx_files_ext(files->packed, MIDX_EXT_BITMAP, NULL); - clear_midx_files_ext(files->packed, MIDX_EXT_REV, NULL); + clear_midx_files_ext(files->dirs->packed, MIDX_EXT_BITMAP, NULL); + clear_midx_files_ext(files->dirs->packed, MIDX_EXT_REV, NULL); strbuf_release(&midx); } @@ -864,21 +864,21 @@ void clear_incremental_midx_files(struct repository *r, for (source = r->objects->sources; source; source = source->next) { files = odb_source_files_downcast(source); - if (files->packed->midx) - close_midx(files->packed->midx); - files->packed->midx = NULL; + if (files->dirs->packed->midx) + close_midx(files->dirs->packed->midx); + files->dirs->packed->midx = NULL; } files = odb_source_files_downcast(r->objects->sources); - get_midx_chain_filename(files->packed, &chain); + get_midx_chain_filename(files->dirs->packed, &chain); if (!keep_hashes && remove_path(chain.buf)) die(_("failed to clear multi-pack-index chain at %s"), chain.buf); - clear_incremental_midx_files_ext(files->packed, MIDX_EXT_BITMAP, keep_hashes); - clear_incremental_midx_files_ext(files->packed, MIDX_EXT_REV, keep_hashes); - clear_incremental_midx_files_ext(files->packed, MIDX_EXT_MIDX, keep_hashes); + clear_incremental_midx_files_ext(files->dirs->packed, MIDX_EXT_BITMAP, keep_hashes); + clear_incremental_midx_files_ext(files->dirs->packed, MIDX_EXT_REV, keep_hashes); + clear_incremental_midx_files_ext(files->dirs->packed, MIDX_EXT_MIDX, keep_hashes); strbuf_release(&chain); } diff --git a/odb/source-files.c b/odb/source-files.c index 68575e9fdf2958..6fc4f6c7e62d1e 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -24,6 +24,30 @@ #include "tree.h" #include "write-or-die.h" +struct odb_files_dir *odb_files_dir_new(struct object_database *odb, + const char *path, bool local) +{ + struct odb_files_dir *dir; + + CALLOC_ARRAY(dir, 1); + dir->abspath = absolute_pathdup(path); + dir->local = local; + dir->loose = odb_source_loose_new(odb, path, local); + dir->packed = odb_source_packed_new(odb, path, local); + + return dir; +} + +void odb_files_dir_free(struct odb_files_dir *dir) +{ + if (!dir) + return; + odb_source_free(&dir->loose->base); + odb_source_free(&dir->packed->base); + free(dir->abspath); + free(dir); +} + static void odb_source_files_reparent(const char *old_cwd, const char *new_cwd, void *cb_data) @@ -31,6 +55,7 @@ static void odb_source_files_reparent(const char *old_cwd, struct odb_source_files *files = cb_data; char *path = reparent_relative_path(old_cwd, new_cwd, files->base.path); + free(files->base.path); files->base.path = path; } @@ -39,8 +64,7 @@ static void odb_source_files_free(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); chdir_notify_unregister(odb_source_files_reparent, files); - odb_source_free(&files->loose->base); - odb_source_free(&files->packed->base); + odb_files_dir_free(files->dirs); odb_source_release(&files->base); free(files); } @@ -48,8 +72,8 @@ static void odb_source_files_free(struct odb_source *source) static void odb_source_files_close(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_close(&files->loose->base); - odb_source_close(&files->packed->base); + odb_source_close(&files->dirs->loose->base); + odb_source_close(&files->dirs->packed->base); } static int odb_source_files_create_on_disk(struct odb_source *source, @@ -144,8 +168,8 @@ static void odb_source_files_prepare(struct odb_source *source, enum odb_prepare_flags flags) { struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_prepare(&files->loose->base, flags); - odb_source_prepare(&files->packed->base, flags); + odb_source_prepare(&files->dirs->loose->base, flags); + odb_source_prepare(&files->dirs->packed->base, flags); } static enum odb_read_status odb_source_files_read_object_info(struct odb_source *source, @@ -157,12 +181,12 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source struct odb_source_files *files = odb_source_files_downcast(source); enum odb_read_status ret_packed, ret_loose; - ret_packed = odb_source_read_object_info(&files->packed->base, oid, oi, + ret_packed = odb_source_read_object_info(&files->dirs->packed->base, oid, oi, flags, errmsg); if (!ret_packed) return 0; - ret_loose = odb_source_read_object_info(&files->loose->base, oid, oi, flags, + ret_loose = odb_source_read_object_info(&files->dirs->loose->base, oid, oi, flags, ret_packed == ODB_READ_NOT_FOUND ? errmsg : NULL); if (!ret_loose) return 0; @@ -184,8 +208,8 @@ static int odb_source_files_read_object_stream(struct odb_stream **out, const struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_stream(out, &files->packed->base, oid) || - !odb_source_read_object_stream(out, &files->loose->base, oid)) + if (!odb_source_read_object_stream(out, &files->dirs->packed->base, oid) || + !odb_source_read_object_stream(out, &files->dirs->loose->base, oid)) return 0; return -1; } @@ -200,12 +224,12 @@ static int odb_source_files_for_each_object(struct odb_source *source, int ret; if (!(opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) { - ret = odb_source_for_each_object(&files->loose->base, request, cb, cb_data, opts); + ret = odb_source_for_each_object(&files->dirs->loose->base, request, cb, cb_data, opts); if (ret) return ret; } - ret = odb_source_for_each_object(&files->packed->base, request, cb, cb_data, opts); + ret = odb_source_for_each_object(&files->dirs->packed->base, request, cb, cb_data, opts); if (ret) return ret; @@ -220,14 +244,14 @@ static int odb_source_files_count_objects(struct odb_source *source, unsigned long count; int ret; - ret = odb_source_count_objects(&files->packed->base, flags, &count); + ret = odb_source_count_objects(&files->dirs->packed->base, flags, &count); if (ret < 0) goto out; if (!(flags & ODB_COUNT_OBJECTS_APPROXIMATE)) { unsigned long loose_count; - ret = odb_source_count_objects(&files->loose->base, flags, &loose_count); + ret = odb_source_count_objects(&files->dirs->loose->base, flags, &loose_count); if (ret < 0) goto out; @@ -250,11 +274,11 @@ static int odb_source_files_find_abbrev_len(struct odb_source *source, unsigned len = min_len; int ret; - ret = odb_source_find_abbrev_len(&files->packed->base, oid, len, &len); + ret = odb_source_find_abbrev_len(&files->dirs->packed->base, oid, len, &len); if (ret < 0) goto out; - ret = odb_source_find_abbrev_len(&files->loose->base, oid, len, &len); + ret = odb_source_find_abbrev_len(&files->dirs->loose->base, oid, len, &len); if (ret < 0) goto out; @@ -271,9 +295,9 @@ static int odb_source_files_freshen_object(struct odb_source *source, int skip_virtualized_objects) { struct odb_source_files *files = odb_source_files_downcast(source); - if (odb_source_freshen_object(&files->packed->base, oid, mtime, + if (odb_source_freshen_object(&files->dirs->packed->base, oid, mtime, skip_virtualized_objects) || - odb_source_freshen_object(&files->loose->base, oid, mtime, + odb_source_freshen_object(&files->dirs->loose->base, oid, mtime, skip_virtualized_objects)) return 1; return 0; @@ -288,7 +312,7 @@ static int odb_source_files_write_object(struct odb_source *source, enum odb_write_object_flags flags) { struct odb_source_files *files = odb_source_files_downcast(source); - return odb_source_write_object(&files->loose->base, buf, len, type, + return odb_source_write_object(&files->dirs->loose->base, buf, len, type, oid, compat_oid, mtime, flags); } @@ -297,7 +321,7 @@ static int odb_source_files_write_object_stream(struct odb_source *source, struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - return odb_source_write_object_stream(&files->loose->base, stream, oid); + return odb_source_write_object_stream(&files->dirs->loose->base, stream, oid); } static int odb_source_files_begin_transaction(struct odb_source *source, @@ -333,7 +357,7 @@ static int too_many_loose_objects(struct odb_source_files *files, int limit) if (limit <= 0) return 0; - if (odb_source_count_objects(&files->loose->base, ODB_COUNT_OBJECTS_APPROXIMATE, + if (odb_source_count_objects(&files->dirs->loose->base, ODB_COUNT_OBJECTS_APPROXIMATE, &loose_count) < 0) return 0; @@ -352,7 +376,7 @@ static struct packed_git *find_base_packs(struct odb_source_files *files, struct packfile_list_entry *e; struct packed_git *base = NULL; - for (e = packfile_store_get_packs(files->packed); e; e = e->next) { + for (e = packfile_store_get_packs(files->dirs->packed); e; e = e->next) { if (e->pack->is_cruft) continue; if (limit) { @@ -377,7 +401,7 @@ static int too_many_packs(struct odb_source_files *files, int gc_auto_pack_limit if (gc_auto_pack_limit <= 0) return 0; - for (e = packfile_store_get_packs(files->packed); e; e = e->next) { + for (e = packfile_store_get_packs(files->dirs->packed); e; e = e->next) { if (e->pack->pack_keep) continue; /* @@ -942,8 +966,8 @@ static int odb_source_files_fsck(struct odb_source *source, if (!(opts->flags & ODB_FSCK_FULL) && !source->local) return 0; - ret |= odb_source_fsck(&files->loose->base, opts); - ret |= odb_source_fsck(&files->packed->base, opts); + ret |= odb_source_fsck(&files->dirs->loose->base, opts); + ret |= odb_source_fsck(&files->dirs->packed->base, opts); return ret; } @@ -956,8 +980,7 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, CALLOC_ARRAY(files, 1); odb_source_init(&files->base, odb, ODB_SOURCE_FILES, path, local); - files->loose = odb_source_loose_new(odb, path, local); - files->packed = odb_source_packed_new(odb, path, local); + files->dirs = odb_files_dir_new(odb, path, local); files->base.free = odb_source_files_free; files->base.close = odb_source_files_close; diff --git a/odb/source-files.h b/odb/source-files.h index 9630b5f962e15f..7f465853b1120c 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -6,14 +6,37 @@ struct odb_source_loose; struct odb_source_packed; +/* + * A single object directory that encapsulates access to both the loose and + * packed backend. This can either be the primary or an alternate object + * directory. + */ +struct odb_files_dir { + /* Absolute path to the object directory. */ + char *abspath; + + /* The two sources derived from this object directory. */ + struct odb_source_loose *loose; + struct odb_source_packed *packed; + + /* + * Whether this is the local object directory of the owning + * repository. Directories added via alternates are not local. + */ + bool local; +}; + +struct odb_files_dir *odb_files_dir_new(struct object_database *odb, + const char *path, bool local); +void odb_files_dir_free(struct odb_files_dir *dir); + /* * The files object database source uses a combination of loose objects and * packfiles. It is the default backend used by Git to store objects. */ struct odb_source_files { struct odb_source base; - struct odb_source_loose *loose; - struct odb_source_packed *packed; + struct odb_files_dir *dirs; }; /* Allocate and initialize a new object source. */ diff --git a/odb/source-loose.c b/odb/source-loose.c index 6806840c237410..e95c96591e38bf 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -1010,7 +1010,7 @@ void odb_source_loose_cache_add_new_oid(struct odb_source *source, switch (source->type) { case ODB_SOURCE_FILES: - loose = odb_source_files_downcast(source)->loose; + loose = odb_source_files_downcast(source)->dirs->loose; break; case ODB_SOURCE_LOOSE: loose = odb_source_loose_downcast(source); diff --git a/pack-bitmap.c b/pack-bitmap.c index 54cba883228c3f..d1d6964f029f4f 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -720,7 +720,7 @@ static int open_bitmap(struct repository *r, for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!open_bitmap_for_source(files->packed, bitmap_git)) + if (!open_bitmap_for_source(files->dirs->packed, bitmap_git)) found = true; /* diff --git a/packfile.c b/packfile.c index 702cf33622054a..0aaf1331699587 100644 --- a/packfile.c +++ b/packfile.c @@ -280,7 +280,7 @@ static int unuse_one_window(struct object_database *odb) for (source = odb->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - for (e = files->packed->packs.head; e; e = e->next) + for (e = files->dirs->packed->packs.head; e; e = e->next) scan_windows(e->pack, &lru_p, &lru_w, &lru_l); } @@ -458,7 +458,7 @@ static int close_one_pack(struct repository *r) for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - for (e = files->packed->packs.head; e; e = e->next) { + for (e = files->dirs->packed->packs.head; e; e = e->next) { if (e->pack->pack_fd == -1) continue; find_lru_pack(e->pack, &lru_p, &mru_w, &accept_windows_inuse); @@ -1934,7 +1934,7 @@ int has_object_pack(struct repository *r, const struct object_id *oid) for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->packed->base, oid, NULL, 0, NULL)) + if (!odb_source_read_object_info(&files->dirs->packed->base, oid, NULL, 0, NULL)) return 1; } @@ -1951,7 +1951,7 @@ int has_object_kept_pack(struct repository *r, const struct object_id *oid, struct odb_source_files *files = odb_source_files_downcast(source); struct packed_git **cache; - cache = packfile_store_get_kept_pack_cache(files->packed, flags); + cache = packfile_store_get_kept_pack_cache(files->dirs->packed, flags); for (; *cache; cache++) { struct packed_git *p = *cache; diff --git a/packfile.h b/packfile.h index b3c66273f6272c..cbb851d43d8c9e 100644 --- a/packfile.h +++ b/packfile.h @@ -79,7 +79,7 @@ static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct for (struct odb_source *source = repo->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); + struct packfile_list_entry *entry = packfile_store_get_packs(files->dirs->packed); if (!entry) continue; data.source = source; @@ -100,7 +100,7 @@ static inline void repo_for_each_pack_data_next(struct repo_for_each_pack_data * for (source = data->source->next; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); + struct packfile_list_entry *entry = packfile_store_get_packs(files->dirs->packed); if (!entry) continue; data->source = source; diff --git a/repack-geometry.c b/repack-geometry.c index 15b34129506d2a..b541c34af5f094 100644 --- a/repack-geometry.c +++ b/repack-geometry.c @@ -33,7 +33,7 @@ void pack_geometry_init(struct pack_geometry *geometry, struct packed_git *p; struct strbuf buf = STRBUF_INIT; struct odb_source_files *files = odb_source_files_downcast(existing->source); - struct multi_pack_index *m = get_multi_pack_index(files->packed); + struct multi_pack_index *m = get_multi_pack_index(files->dirs->packed); repo_for_each_pack(existing->repo, p) { if (geometry->midx_layer_threshold_set && m && diff --git a/repack-midx.c b/repack-midx.c index 64c7f8d0f42372..25140569bd9d2b 100644 --- a/repack-midx.c +++ b/repack-midx.c @@ -564,7 +564,7 @@ static void repack_make_midx_append_plan(struct repack_write_midx_opts *opts, size_t steps_nr = 0, steps_alloc = 0; odb_reprepare(opts->existing->repo->objects); - m = get_multi_pack_index(files->packed); + m = get_multi_pack_index(files->dirs->packed); if (opts->names->nr) { struct strbuf buf = STRBUF_INIT; @@ -620,7 +620,7 @@ static int repack_make_midx_compaction_plan(struct repack_write_midx_opts *opts, opts->existing->repo); odb_reprepare(opts->existing->repo->objects); - m = get_multi_pack_index(files->packed); + m = get_multi_pack_index(files->dirs->packed); for (i = 0; m && i < m->num_packs + m->num_packs_in_base; i++) { if (prepare_midx_pack(m, i)) { @@ -949,7 +949,7 @@ static int write_midx_incremental(struct repack_write_midx_opts *opts) size_t i; int ret = 0; - get_midx_chain_filename(files->packed, &lock_name); + get_midx_chain_filename(files->dirs->packed, &lock_name); if (safe_create_leading_directories(opts->existing->repo, lock_name.buf)) die_errno(_("unable to create leading directories of %s"), diff --git a/repack.c b/repack.c index d2aa58e13484fd..e20431690c96ba 100644 --- a/repack.c +++ b/repack.c @@ -60,7 +60,7 @@ void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, { struct strbuf buf = STRBUF_INIT; struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources); - struct multi_pack_index *m = get_multi_pack_index(files->packed); + struct multi_pack_index *m = get_multi_pack_index(files->dirs->packed); strbuf_addf(&buf, "%s.pack", base_name); if (m && files->base.local && midx_contains_pack(m, buf.buf)) { clear_midx_file(repo); diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index 83b07c6236b167..3f4bafff6191dc 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -22,7 +22,7 @@ static struct multi_pack_index *setup_midx(const char *object_dir, source = odb_find_source(the_repository->objects, object_dir); if (source) { - packed = odb_source_files_downcast(source)->packed; + packed = odb_source_files_downcast(source)->dirs->packed; } else { packed = odb_source_packed_new(the_repository->objects, object_dir, false); From 4a3fcd7fea2beb088474cfa341cac215df3734b5 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:54 +0200 Subject: [PATCH 14/29] odb: refactor `odb_for_each_alternate()` to yield dirs The function `odb_for_each_alternate()` iterates through all alternates of an object database. This is becoming an implementation detail of the "files" backend though, where each alternate will be represented by one `struct odb_files_dir`. Adapt `odb_for_each_alternate()` to already iterate through these structs instead of iterating through sources. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/count-objects.c | 4 ++-- builtin/submodule--helper.c | 7 ++++--- diagnose.c | 8 ++++---- odb.c | 18 +++++++++++------- odb.h | 3 ++- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/builtin/count-objects.c b/builtin/count-objects.c index 18f6e33b6f913f..f2abfaccec1c6f 100644 --- a/builtin/count-objects.c +++ b/builtin/count-objects.c @@ -81,10 +81,10 @@ static int count_cruft(const char *basename UNUSED, const char *path, return 0; } -static int print_alternate(struct odb_source *alternate, void *data UNUSED) +static int print_alternate(struct odb_files_dir *alternate, void *data UNUSED) { printf("alternate: "); - quote_c_style(alternate->path, NULL, stdout, 0); + quote_c_style(alternate->abspath, NULL, stdout, 0); putchar('\n'); return 0; } diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 2a29d649b8c486..af6bed2662d0f0 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -30,6 +30,7 @@ #include "object-name.h" #include "odb.h" #include "odb/source.h" +#include "odb/source-files.h" #include "advice.h" #include "branch.h" #include "list-objects-filter-options.h" @@ -1770,7 +1771,7 @@ static const char alternate_error_advice[] = N_( ); static int add_possible_reference_from_superproject( - struct odb_source *alt_odb, void *sas_cb) + struct odb_files_dir *alt_odb, void *sas_cb) { struct submodule_alternate_setup *sas = sas_cb; size_t len; @@ -1779,12 +1780,12 @@ static int add_possible_reference_from_superproject( * If the alternate object store is another repository, try the * standard layout with .git/(modules/)+/objects */ - if (strip_suffix(alt_odb->path, "/objects", &len)) { + if (strip_suffix(alt_odb->abspath, "/objects", &len)) { struct repository alternate; char *sm_alternate; struct strbuf sb = STRBUF_INIT; struct strbuf err = STRBUF_INIT; - strbuf_add(&sb, alt_odb->path, len); + strbuf_add(&sb, alt_odb->abspath, len); if (repo_init(&alternate, sb.buf, NULL) < 0) die(_("could not get a repository handle for gitdir '%s'"), diff --git a/diagnose.c b/diagnose.c index 9d51ded427a265..9e675eb0d5abd9 100644 --- a/diagnose.c +++ b/diagnose.c @@ -60,13 +60,13 @@ static void dir_file_stats_objects(const char *full_path, (uintmax_t)st.st_size); } -static int dir_file_stats(struct odb_source *source, void *data) +static int dir_file_stats(struct odb_files_dir *source, void *data) { struct strbuf *buf = data; - strbuf_addf(buf, "Contents of %s:\n", source->path); + strbuf_addf(buf, "Contents of %s:\n", source->abspath); - for_each_file_in_pack_dir(source->path, dir_file_stats_objects, + for_each_file_in_pack_dir(source->abspath, dir_file_stats_objects, data); return 0; @@ -270,7 +270,7 @@ int create_diagnostics_archive(struct repository *r, strbuf_reset(&buf); strbuf_addstr(&buf, "--add-virtual-file=packs-local.txt:"); - dir_file_stats(r->objects->sources, &buf); + dir_file_stats(odb_source_files_downcast(r->objects->sources)->dirs, &buf); odb_for_each_alternate(r->objects, dir_file_stats, &buf); strvec_push(&archiver_args, buf.buf); diff --git a/odb.c b/odb.c index 689fe527a6cdf2..5bb2a447556832 100644 --- a/odb.c +++ b/odb.c @@ -17,6 +17,7 @@ #include "object-name.h" #include "odb.h" #include "odb/source-inmemory.h" +#include "odb/source-files.h" #include "path.h" #include "pkt-line.h" #include "promisor-remote.h" @@ -498,18 +499,19 @@ static void read_alternate_refs(struct repository *repo, } struct alternate_refs_data { + struct repository *repo; odb_for_each_alternate_ref_fn *fn; void *payload; }; -static int refs_from_alternate_cb(struct odb_source *alternate, +static int refs_from_alternate_cb(struct odb_files_dir *alternate, void *payload) { struct strbuf path = STRBUF_INIT; size_t base_len; struct alternate_refs_data *cb = payload; - if (!strbuf_realpath(&path, alternate->path, 0)) + if (!strbuf_realpath(&path, alternate->abspath, 0)) goto out; if (!strbuf_strip_suffix(&path, "/objects")) goto out; @@ -521,7 +523,7 @@ static int refs_from_alternate_cb(struct odb_source *alternate, goto out; strbuf_setlen(&path, base_len); - read_alternate_refs(alternate->odb->repo, path.buf, cb->fn, cb->payload); + read_alternate_refs(cb->repo, path.buf, cb->fn, cb->payload); out: strbuf_release(&path); @@ -531,9 +533,11 @@ static int refs_from_alternate_cb(struct odb_source *alternate, void odb_for_each_alternate_ref(struct object_database *odb, odb_for_each_alternate_ref_fn cb, void *payload) { - struct alternate_refs_data data; - data.fn = cb; - data.payload = payload; + struct alternate_refs_data data = { + .fn = cb, + .payload = payload, + .repo = odb->repo, + }; odb_for_each_alternate(odb, refs_from_alternate_cb, &data); } @@ -544,7 +548,7 @@ int odb_for_each_alternate(struct object_database *odb, int r = 0; for (alternate = odb->sources->next; alternate; alternate = alternate->next) { - r = cb(alternate, payload); + r = cb(odb_source_files_downcast(alternate)->dirs, payload); if (r) break; } diff --git a/odb.h b/odb.h index cecdf38ecf6b7f..9c4ff49aa5b7de 100644 --- a/odb.h +++ b/odb.h @@ -12,6 +12,7 @@ struct cached_object_entry; struct list_objects_filter_options; struct odb_source_inmemory; +struct odb_files_dir; struct packed_git; struct repository; struct strbuf; @@ -257,7 +258,7 @@ void odb_restore_primary_source(struct object_database *odb, * function returns a non-zero value, in which case the value is bubbled up * from the callback. */ -typedef int odb_for_each_alternate_fn(struct odb_source *, void *); +typedef int odb_for_each_alternate_fn(struct odb_files_dir *, void *); int odb_for_each_alternate(struct object_database *odb, odb_for_each_alternate_fn cb, void *payload); From ad8afd3882d0ca6f36586d10a0edf48833032a8f Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:55 +0200 Subject: [PATCH 15/29] odb: refactor `odb_find_source()` to yield dirs Similar as in the preceding commit, `odb_find_source()` can be used to yield a single source by its object directory. This function is also specific to the "files" backend once alternates are an implementation detail thereof. Refactor it to be specific to the "files" backend and return an `struct odb_files_dir` to prepare for moving alternates into the "files" source. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/commit-graph.c | 26 ++++++++++++++--------- builtin/multi-pack-index.c | 42 +++++++++++++++++++------------------- odb.c | 26 ----------------------- odb.h | 9 -------- odb/source-files.c | 26 +++++++++++++++++++++++ odb/source-files.h | 6 ++++++ t/helper/test-read-midx.c | 8 ++++---- 7 files changed, 73 insertions(+), 70 deletions(-) diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index a986f08a947580..ff244c866fea4f 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -9,6 +9,7 @@ #include "commit-graph.h" #include "odb.h" #include "odb/source.h" +#include "odb/source-files.h" #include "progress.h" #include "replace-object.h" #include "strbuf.h" @@ -68,7 +69,7 @@ static int graph_verify(int argc, const char **argv, const char *prefix, struct repository *repo UNUSED) { struct commit_graph *graph = NULL; - struct odb_source *source = NULL; + struct odb_files_dir *dir = NULL; char *graph_name; char *chain_name; enum { OPENED_NONE, OPENED_GRAPH, OPENED_CHAIN } opened = OPENED_NONE; @@ -103,9 +104,12 @@ static int graph_verify(int argc, const char **argv, const char *prefix, if (opts.progress) flags |= COMMIT_GRAPH_WRITE_PROGRESS; - source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); - graph_name = get_commit_graph_filename(source->path); - chain_name = get_commit_graph_chain_filename(source->path); + dir = odb_source_files_find_dir(the_repository->objects, opts.obj_dir); + if (!dir) + die(_("could not find object directory matching %s"), opts.obj_dir); + + graph_name = get_commit_graph_filename(dir->abspath); + chain_name = get_commit_graph_chain_filename(dir->abspath); if (open_commit_graph(graph_name, &fd, &st)) opened = OPENED_GRAPH; else if (errno != ENOENT) @@ -123,7 +127,7 @@ static int graph_verify(int argc, const char **argv, const char *prefix, if (opened == OPENED_NONE) return 0; else if (opened == OPENED_GRAPH) - graph = load_commit_graph_one_fd_st(the_repository, source->path, fd, &st); + graph = load_commit_graph_one_fd_st(the_repository, dir->abspath, fd, &st); else graph = load_commit_graph_chain_fd_st(the_repository->objects, fd, &st, &incomplete_chain); @@ -226,7 +230,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, struct string_list pack_indexes = STRING_LIST_INIT_DUP; struct strbuf buf = STRBUF_INIT; struct oidset commits = OIDSET_INIT; - struct odb_source *source = NULL; + struct odb_files_dir *dir = NULL; int result = 0; enum commit_graph_write_flags flags = 0; struct progress *progress = NULL; @@ -294,10 +298,12 @@ static int graph_write(int argc, const char **argv, const char *prefix, git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags |= COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); + dir = odb_source_files_find_dir(the_repository->objects, opts.obj_dir); + if (!dir) + die(_("could not find object directory matching %s"), opts.obj_dir); if (opts.reachable) { - if (write_commit_graph_reachable(the_repository, source->path, flags, &write_opts)) + if (write_commit_graph_reachable(the_repository, dir->abspath, flags, &write_opts)) result = 1; goto cleanup; } @@ -306,7 +312,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, struct strbuf packname = STRBUF_INIT; size_t dirlen; - strbuf_addf(&packname, "%s/pack/", source->path); + strbuf_addf(&packname, "%s/pack/", dir->abspath); dirlen = packname.len; while (strbuf_getline(&buf, stdin) != EOF) { @@ -334,7 +340,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, stop_progress(&progress); } - if (write_commit_graph(the_repository, source->path, + if (write_commit_graph(the_repository, dir->abspath, opts.stdin_packs ? &pack_indexes : NULL, opts.stdin_commits ? &commits : NULL, flags, diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index a170ec80b99cff..fc8b494996159a 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -86,13 +86,13 @@ static int parse_object_dir(const struct option *opt, const char *arg, return 0; } -static struct odb_source_files *handle_object_dir_option(struct repository *repo) +static struct odb_source_packed *handle_object_dir_option(struct repository *repo) { - struct odb_source *source = odb_find_source(repo->objects, opts.object_dir); - if (!source) + struct odb_files_dir *dir = odb_source_files_find_dir(repo->objects, opts.object_dir); + if (!dir) die(_("object directory is not an alternate of the current repository: '%s'"), opts.object_dir); - return odb_source_files_downcast(source); + return dir->packed; } static struct option common_opts[] = { @@ -169,7 +169,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, N_("refs snapshot for selecting bitmap commits")), OPT_END(), }; - struct odb_source_files *source; + struct odb_source_packed *packed_source; int ret; opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE; @@ -204,7 +204,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, options); } - source = handle_object_dir_option(repo); + packed_source = handle_object_dir_option(repo); FREE_AND_NULL(options); @@ -213,7 +213,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, read_packs_from_stdin(&packs); - ret = write_midx_file_only(source->dirs->packed, &packs, + ret = write_midx_file_only(packed_source, &packs, opts.preferred_pack, opts.refs_snapshot, opts.incremental_base, opts.flags); @@ -225,7 +225,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } - ret = write_midx_file(source->dirs->packed, opts.preferred_pack, + ret = write_midx_file(packed_source, opts.preferred_pack, opts.refs_snapshot, opts.flags); free(opts.refs_snapshot); @@ -239,7 +239,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, struct multi_pack_index *m, *cur; struct multi_pack_index *from_midx = NULL; struct multi_pack_index *to_midx = NULL; - struct odb_source_files *source; + struct odb_source_packed *packed_source; int ret; struct option *options; @@ -280,11 +280,11 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, options); } - source = handle_object_dir_option(the_repository); + packed_source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - m = get_multi_pack_index(source->dirs->packed); + m = get_multi_pack_index(packed_source); for (cur = m; cur && !(from_midx && to_midx); cur = cur->base_midx) { const char *midx_csum = midx_get_checksum_hex(cur); @@ -307,7 +307,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, die(_("MIDX %s must be an ancestor of %s"), argv[0], argv[1]); } - ret = write_midx_file_compact(source->dirs->packed, from_midx, to_midx, + ret = write_midx_file_compact(packed_source, from_midx, to_midx, opts.incremental_base, opts.flags); return ret; @@ -321,7 +321,7 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, static struct option builtin_multi_pack_index_verify_options[] = { OPT_END(), }; - struct odb_source_files *source; + struct odb_source_packed *packed_source; options = add_common_options(builtin_multi_pack_index_verify_options); @@ -335,11 +335,11 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_verify_usage, options); - source = handle_object_dir_option(the_repository); + packed_source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return verify_midx_file(source->dirs->packed, opts.flags); + return verify_midx_file(packed_source, opts.flags); } static int cmd_multi_pack_index_expire(int argc, const char **argv, @@ -350,7 +350,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, static struct option builtin_multi_pack_index_expire_options[] = { OPT_END(), }; - struct odb_source_files *source; + struct odb_source_packed *packed_source; options = add_common_options(builtin_multi_pack_index_expire_options); @@ -364,11 +364,11 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_expire_usage, options); - source = handle_object_dir_option(the_repository); + packed_source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return expire_midx_packs(source->dirs->packed, opts.flags); + return expire_midx_packs(packed_source, opts.flags); } static int cmd_multi_pack_index_repack(int argc, const char **argv, @@ -381,7 +381,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, N_("during repack, collect pack-files of smaller size into a batch that is larger than this size")), OPT_END(), }; - struct odb_source_files *source; + struct odb_source_packed *packed_source; options = add_common_options(builtin_multi_pack_index_repack_options); @@ -396,11 +396,11 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_repack_usage, options); - source = handle_object_dir_option(the_repository); + packed_source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return midx_repack(source->dirs->packed, (size_t)opts.batch_size, opts.flags); + return midx_repack(packed_source, (size_t)opts.batch_size, opts.flags); } int cmd_multi_pack_index(int argc, diff --git a/odb.c b/odb.c index 5bb2a447556832..4fec73ced94694 100644 --- a/odb.c +++ b/odb.c @@ -411,32 +411,6 @@ char *compute_alternate_path(const char *path, struct strbuf *err) return ref_git; } -struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir) -{ - struct odb_source *source; - char *obj_dir_real = real_pathdup(obj_dir, 1); - struct strbuf odb_path_real = STRBUF_INIT; - - for (source = odb->sources; source; source = source->next) { - strbuf_realpath(&odb_path_real, source->path, 1); - if (!strcmp(obj_dir_real, odb_path_real.buf)) - break; - } - - free(obj_dir_real); - strbuf_release(&odb_path_real); - - return source; -} - -struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir) -{ - struct odb_source *source = odb_find_source(odb, obj_dir); - if (!source) - die(_("could not find object directory matching %s"), obj_dir); - return source; -} - static void fill_alternate_refs_command(struct repository *repo, struct child_process *cmd, const char *repo_path) diff --git a/odb.h b/odb.h index 9c4ff49aa5b7de..c7aa38c63e19b2 100644 --- a/odb.h +++ b/odb.h @@ -226,15 +226,6 @@ struct odb_fsck_options { */ int odb_fsck(struct object_database *odb, struct odb_fsck_options *opts); -/* - * Find source by its object directory path. Returns a `NULL` pointer in case - * the source could not be found. - */ -struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir); - -/* Same as `odb_find_source()`, but dies in case the source doesn't exist. */ -struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir); - /* * Replace the current writable object directory with the specified temporary * object directory and return the newly installed primary source. The former diff --git a/odb/source-files.c b/odb/source-files.c index 6fc4f6c7e62d1e..74d9eee0553fcd 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -972,6 +972,32 @@ static int odb_source_files_fsck(struct odb_source *source, return ret; } +struct odb_files_dir *odb_source_files_find_dir(struct object_database *odb, const char *obj_dir) +{ + char *obj_dir_real = real_pathdup(obj_dir, 1); + struct strbuf odb_path_real = STRBUF_INIT; + struct odb_files_dir *dir = NULL; + struct odb_source *source; + + for (source = odb->sources; source; source = source->next) { + struct odb_source_files *files; + + if (source->type != ODB_SOURCE_FILES) + continue; + files = odb_source_files_downcast(source); + + strbuf_realpath(&odb_path_real, files->dirs->abspath, 1); + if (!strcmp(obj_dir_real, odb_path_real.buf)) { + dir = files->dirs; + break; + } + } + + free(obj_dir_real); + strbuf_release(&odb_path_real); + return dir; +} + struct odb_source_files *odb_source_files_new(struct object_database *odb, const char *path, bool local) diff --git a/odb/source-files.h b/odb/source-files.h index 7f465853b1120c..77f4d842e0f24e 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -72,4 +72,10 @@ static inline struct odb_source_files *odb_source_files_downcast(struct odb_sour return container_of(source, struct odb_source_files, base); } +/* + * Find "files" directory by its object directory path. Returns a `NULL` + * pointer in case the object directory could not be found. + */ +struct odb_files_dir *odb_source_files_find_dir(struct object_database *odb, const char *obj_dir); + #endif diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index 3f4bafff6191dc..88a79fdfc8d0d5 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -16,13 +16,13 @@ static struct multi_pack_index *setup_midx(const char *object_dir, struct odb_source_packed **out) { struct odb_source_packed *packed; - struct odb_source *source; + struct odb_files_dir *dir; setup_git_directory(the_repository); - source = odb_find_source(the_repository->objects, object_dir); - if (source) { - packed = odb_source_files_downcast(source)->dirs->packed; + dir = odb_source_files_find_dir(the_repository->objects, object_dir); + if (dir) { + packed = dir->packed; } else { packed = odb_source_packed_new(the_repository->objects, object_dir, false); From c5d78ba8a288ec6e017a8d891d00c754dc029425 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:56 +0200 Subject: [PATCH 16/29] odb/source-files: add the ability to have multiple object dirs In a subsequent commit we'll migrate handling of alternates into the "files" backend. As part of that, the backend will need to grow the ability to handle multiple object directories for alternates and for transactions. Introduce a `struct odb_files_dir::next` pointer so that we can have multiple directories. Adapt the backend to loop through this list as necessary. In general: - For reading paths we will loop through all object directories until we have found the object. - For writing paths we will typically write to the first of our object directories. This mechanism isn't used yet as we still only ever have a single object directory at the current point in time. But that will change over subsequent commits. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-files.c | 151 ++++++++++++++++++++++++++++----------------- odb/source-files.h | 9 +++ 2 files changed, 102 insertions(+), 58 deletions(-) diff --git a/odb/source-files.c b/odb/source-files.c index 74d9eee0553fcd..bdf1c628684e28 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -63,8 +63,15 @@ static void odb_source_files_reparent(const char *old_cwd, static void odb_source_files_free(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); + chdir_notify_unregister(odb_source_files_reparent, files); - odb_files_dir_free(files->dirs); + + while (files->dirs) { + struct odb_files_dir *next = files->dirs->next; + odb_files_dir_free(files->dirs); + files->dirs = next; + } + odb_source_release(&files->base); free(files); } @@ -72,8 +79,11 @@ static void odb_source_files_free(struct odb_source *source) static void odb_source_files_close(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_close(&files->dirs->loose->base); - odb_source_close(&files->dirs->packed->base); + + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + odb_source_close(&dir->loose->base); + odb_source_close(&dir->packed->base); + } } static int odb_source_files_create_on_disk(struct odb_source *source, @@ -168,8 +178,11 @@ static void odb_source_files_prepare(struct odb_source *source, enum odb_prepare_flags flags) { struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_prepare(&files->dirs->loose->base, flags); - odb_source_prepare(&files->dirs->packed->base, flags); + + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + odb_source_prepare(&dir->loose->base, flags); + odb_source_prepare(&dir->packed->base, flags); + } } static enum odb_read_status odb_source_files_read_object_info(struct odb_source *source, @@ -179,28 +192,33 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source struct strbuf *errmsg) { struct odb_source_files *files = odb_source_files_downcast(source); - enum odb_read_status ret_packed, ret_loose; - - ret_packed = odb_source_read_object_info(&files->dirs->packed->base, oid, oi, - flags, errmsg); - if (!ret_packed) - return 0; - - ret_loose = odb_source_read_object_info(&files->dirs->loose->base, oid, oi, flags, - ret_packed == ODB_READ_NOT_FOUND ? errmsg : NULL); - if (!ret_loose) - return 0; + enum odb_read_status status = ODB_READ_NOT_FOUND; /* - * Reading the packed object may have failed even though the object - * exists, for example because it is corrupt. Report this failure to - * the caller in case neither of the sources was able to read the - * object, and prefer the error of the packed source in case both - * reads have failed. + * Reading an object may fail even though the object exists, for + * example because it is corrupt. Report this failure to the caller in + * case none of the directories was able to read the object, and + * prefer the first such error in case multiple reads have failed. */ - if (ret_packed != ODB_READ_NOT_FOUND) - return ret_packed; - return ret_loose; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + enum odb_read_status ret; + + ret = odb_source_read_object_info(&dir->packed->base, oid, oi, flags, + status == ODB_READ_NOT_FOUND ? errmsg : NULL); + if (!ret) + return 0; + if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND) + status = ret; + + ret = odb_source_read_object_info(&dir->loose->base, oid, oi, flags, + status == ODB_READ_NOT_FOUND ? errmsg : NULL); + if (!ret) + return 0; + if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND) + status = ret; + } + + return status; } static int odb_source_files_read_object_stream(struct odb_stream **out, @@ -208,9 +226,12 @@ static int odb_source_files_read_object_stream(struct odb_stream **out, const struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_stream(out, &files->dirs->packed->base, oid) || - !odb_source_read_object_stream(out, &files->dirs->loose->base, oid)) - return 0; + + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) + if (!odb_source_read_object_stream(out, &dir->packed->base, oid) || + !odb_source_read_object_stream(out, &dir->loose->base, oid)) + return 0; + return -1; } @@ -223,16 +244,21 @@ static int odb_source_files_for_each_object(struct odb_source *source, struct odb_source_files *files = odb_source_files_downcast(source); int ret; - if (!(opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) { - ret = odb_source_for_each_object(&files->dirs->loose->base, request, cb, cb_data, opts); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !dir->local) + continue; + + if (!(opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) { + ret = odb_source_for_each_object(&dir->loose->base, request, cb, cb_data, opts); + if (ret) + return ret; + } + + ret = odb_source_for_each_object(&dir->packed->base, request, cb, cb_data, opts); if (ret) return ret; } - ret = odb_source_for_each_object(&files->dirs->packed->base, request, cb, cb_data, opts); - if (ret) - return ret; - return 0; } @@ -241,21 +267,23 @@ static int odb_source_files_count_objects(struct odb_source *source, unsigned long *out) { struct odb_source_files *files = odb_source_files_downcast(source); - unsigned long count; + unsigned long count = 0; int ret; - ret = odb_source_count_objects(&files->dirs->packed->base, flags, &count); - if (ret < 0) - goto out; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + unsigned long dir_count; - if (!(flags & ODB_COUNT_OBJECTS_APPROXIMATE)) { - unsigned long loose_count; - - ret = odb_source_count_objects(&files->dirs->loose->base, flags, &loose_count); + ret = odb_source_count_objects(&dir->packed->base, flags, &dir_count); if (ret < 0) goto out; + count += dir_count; - count += loose_count; + if (!(flags & ODB_COUNT_OBJECTS_APPROXIMATE)) { + ret = odb_source_count_objects(&dir->loose->base, flags, &dir_count); + if (ret < 0) + goto out; + count += dir_count; + } } *out = count; @@ -272,15 +300,17 @@ static int odb_source_files_find_abbrev_len(struct odb_source *source, { struct odb_source_files *files = odb_source_files_downcast(source); unsigned len = min_len; - int ret; + int ret = 0; - ret = odb_source_find_abbrev_len(&files->dirs->packed->base, oid, len, &len); - if (ret < 0) - goto out; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + ret = odb_source_find_abbrev_len(&dir->packed->base, oid, len, &len); + if (ret < 0) + goto out; - ret = odb_source_find_abbrev_len(&files->dirs->loose->base, oid, len, &len); - if (ret < 0) - goto out; + ret = odb_source_find_abbrev_len(&dir->loose->base, oid, len, &len); + if (ret < 0) + goto out; + } *out = len; ret = 0; @@ -295,11 +325,14 @@ static int odb_source_files_freshen_object(struct odb_source *source, int skip_virtualized_objects) { struct odb_source_files *files = odb_source_files_downcast(source); - if (odb_source_freshen_object(&files->dirs->packed->base, oid, mtime, - skip_virtualized_objects) || - odb_source_freshen_object(&files->dirs->loose->base, oid, mtime, - skip_virtualized_objects)) - return 1; + + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) + if (odb_source_freshen_object(&dir->packed->base, oid, mtime, + skip_virtualized_objects) || + odb_source_freshen_object(&dir->loose->base, oid, mtime, + skip_virtualized_objects)) + return 1; + return 0; } @@ -963,11 +996,13 @@ static int odb_source_files_fsck(struct odb_source *source, struct odb_source_files *files = odb_source_files_downcast(source); int ret = 0; - if (!(opts->flags & ODB_FSCK_FULL) && !source->local) - return 0; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!(opts->flags & ODB_FSCK_FULL) && !dir->local) + continue; - ret |= odb_source_fsck(&files->dirs->loose->base, opts); - ret |= odb_source_fsck(&files->dirs->packed->base, opts); + ret |= odb_source_fsck(&dir->loose->base, opts); + ret |= odb_source_fsck(&dir->packed->base, opts); + } return ret; } diff --git a/odb/source-files.h b/odb/source-files.h index 77f4d842e0f24e..36af0c1b8ba059 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -15,6 +15,9 @@ struct odb_files_dir { /* Absolute path to the object directory. */ char *abspath; + /* List of alternate object directories. */ + struct odb_files_dir *next; + /* The two sources derived from this object directory. */ struct odb_source_loose *loose; struct odb_source_packed *packed; @@ -36,6 +39,12 @@ void odb_files_dir_free(struct odb_files_dir *dir); */ struct odb_source_files { struct odb_source base; + + /* + * List of all object directories; the main directory is first (and + * cannot be NULL after initialization). Subsequent directories are + * alternates. + */ struct odb_files_dir *dirs; }; From 616c9409cce7171beeec36979c58ac234b70074e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:57 +0200 Subject: [PATCH 17/29] tmp-objdir: absorb logic to set and restore primary sources The functions `odb_set_temporary_primary_source()` and `odb_restore_primary_source()` can be used to adapt the primary object database source. Nowadays though we only have a single user of this subsystem left, which is the "tmp-objdir" subsystem. Despite that, this functionality is also becoming less useful overall as alternates are becoming an implementation detail of the "files" backend. And with that change, there will only ever be a single source attached to the object database anyway. Move the logic into the "tmp-objdir" subsystem accordingly. Assisted-by: GPT-6.1 Sol Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano Signed-off-by: Johannes Schindelin --- builtin/gc.c | 36 ++++++++++++++---------------------- odb.c | 45 --------------------------------------------- odb.h | 17 ----------------- tmp-objdir.c | 38 ++++++++++++++++++++++++++++++++++---- 4 files changed, 48 insertions(+), 88 deletions(-) diff --git a/builtin/gc.c b/builtin/gc.c index bd122b1e878074..7e249b64a68fe7 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1042,28 +1042,24 @@ static int pack_loose(struct maintenance_run_opts *opts) int result = 0; struct write_loose_object_data data; struct child_process pack_proc = CHILD_PROCESS_INIT; - struct odb_source *prev_source = NULL; - const char *object_dir = r->objects->sources->path; + struct odb_source *source = r->objects->sources; + struct odb_files_dir *shared = NULL; /* If set, use the shared object directory. */ if (shared_object_dir) { - odb_set_temporary_primary_source(r->objects, shared_object_dir, - 0, &prev_source); - object_dir = shared_object_dir; + shared = odb_files_dir_new(r->objects, shared_object_dir, + false); + source = &shared->loose->base; } /* * Do not start pack-objects process * if there are no loose objects. */ - if (!for_each_loose_file_in_source(r->objects->sources, + if (!for_each_loose_file_in_source(source, bail_on_loose, - NULL, NULL, NULL)) { - if (shared_object_dir) - odb_restore_primary_source(r->objects, prev_source, - shared_object_dir); - return 0; - } + NULL, NULL, NULL)) + goto out; pack_proc.git_cmd = 1; @@ -1072,7 +1068,7 @@ static int pack_loose(struct maintenance_run_opts *opts) strvec_push(&pack_proc.args, "--quiet"); else strvec_push(&pack_proc.args, "--no-quiet"); - strvec_pushf(&pack_proc.args, "%s/pack/loose", object_dir); + strvec_pushf(&pack_proc.args, "%s/pack/loose", source->path); pack_proc.in = -1; @@ -1084,10 +1080,8 @@ static int pack_loose(struct maintenance_run_opts *opts) if (start_command(&pack_proc)) { error(_("failed to start 'git pack-objects' process")); - if (shared_object_dir) - odb_restore_primary_source(r->objects, prev_source, - shared_object_dir); - return 1; + result = 1; + goto out; } data.in = xfdopen(pack_proc.in, "w"); @@ -1103,7 +1097,7 @@ static int pack_loose(struct maintenance_run_opts *opts) else if (data.batch_size > 0) data.batch_size--; /* Decrease for equality on limit. */ - for_each_loose_file_in_source(r->objects->sources, + for_each_loose_file_in_source(source, write_loose_object_to_stdin, NULL, NULL, &data); @@ -1114,10 +1108,8 @@ static int pack_loose(struct maintenance_run_opts *opts) result = 1; } - if (shared_object_dir) - odb_restore_primary_source(r->objects, prev_source, - shared_object_dir); - +out: + odb_files_dir_free(shared); return result; } diff --git a/odb.c b/odb.c index 4fec73ced94694..38ef5e21083858 100644 --- a/odb.c +++ b/odb.c @@ -302,51 +302,6 @@ struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, return odb_add_alternate_recursively(odb, dir, 0); } -struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, - const char *dir, int will_destroy, - struct odb_source **prev_source) -{ - struct odb_source *source; - - /* - * Make a new primary odb and link the old primary ODB in as an - * alternate - */ - source = odb_source_new(odb, dir, false); - - /* - * Disable ref updates while a temporary odb is active, since - * the objects in the database may roll back. - */ - odb->repo->disable_ref_updates = true; - source->will_destroy = will_destroy; - source->next = odb->sources; - odb->sources = source; - - if (prev_source) - *prev_source = source->next; - - return source; -} - -void odb_restore_primary_source(struct object_database *odb, - struct odb_source *restore_source, - const char *old_path) -{ - struct odb_source *cur_source = odb->sources; - - if (strcmp(old_path, cur_source->path)) - BUG("expected %s as primary object store; found %s", - old_path, cur_source->path); - - if (cur_source->next != restore_source) - BUG("we expect the old primary object store to be the first alternate"); - - odb->repo->disable_ref_updates = false; - odb->sources = restore_source; - odb_source_free(cur_source); -} - char *compute_alternate_path(const char *path, struct strbuf *err) { char *ref_git = NULL; diff --git a/odb.h b/odb.h index c7aa38c63e19b2..364f512b998a2f 100644 --- a/odb.h +++ b/odb.h @@ -226,23 +226,6 @@ struct odb_fsck_options { */ int odb_fsck(struct object_database *odb, struct odb_fsck_options *opts); -/* - * Replace the current writable object directory with the specified temporary - * object directory and return the newly installed primary source. The former - * primary source is reported via `prev_source` when non-NULL. - */ -struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, - const char *dir, int will_destroy, - struct odb_source **prev_source); - -/* - * Restore the primary source that was previously replaced by - * `odb_set_temporary_primary_source()`. - */ -void odb_restore_primary_source(struct object_database *odb, - struct odb_source *restore_source, - const char *old_path); - /* * Iterate through all alternates of the database and execute the provided * callback function for each of them. Stop iterating once the callback diff --git a/tmp-objdir.c b/tmp-objdir.c index deaaf6ba2ea315..31a7920be7daea 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -51,6 +51,26 @@ static void tmp_objdir_reparent(const char *old_cwd, free(path); } +/* + * Restore the primary source that was previously replaced by + * `tmp_objdir_replace_primary_odb()`. + */ +static void tmp_objdir_restore_source(struct tmp_objdir *t) +{ + struct odb_source *cur_source = t->repo->objects->sources; + + if (strcmp(t->path.buf, cur_source->path)) + BUG("expected %s as primary object store; found %s", + t->path.buf, cur_source->path); + + if (cur_source->next != t->prev_source) + BUG("we expect the old primary object store to be the first alternate"); + + t->repo->disable_ref_updates = false; + t->repo->objects->sources = t->prev_source; + odb_source_free(cur_source); +} + int tmp_objdir_destroy(struct tmp_objdir *t) { int err; @@ -62,7 +82,7 @@ int tmp_objdir_destroy(struct tmp_objdir *t) the_tmp_objdir = NULL; if (t->prev_source) - odb_restore_primary_source(t->repo->objects, t->prev_source, t->path.buf); + tmp_objdir_restore_source(t); err = remove_dir_recursively(&t->path, 0); @@ -298,7 +318,7 @@ int tmp_objdir_migrate(struct tmp_objdir *t) if (t->prev_source) { if (t->repo->objects->sources->will_destroy) BUG("migrating an ODB that was marked for destruction"); - odb_restore_primary_source(t->repo->objects, t->prev_source, t->path.buf); + tmp_objdir_restore_source(t); t->prev_source = NULL; } @@ -328,6 +348,16 @@ struct odb_source *tmp_objdir_replace_primary_odb(struct tmp_objdir *t, BUG("the primary object database is already replaced"); t->will_destroy = will_destroy; - return odb_set_temporary_primary_source(t->repo->objects, t->path.buf, - will_destroy, &t->prev_source); + /* + * Make a new primary source and link the old primary source in as an + * alternate. Disable ref updates while a temporary source is active, + * since the objects in the database may roll back. + */ + t->prev_source = t->repo->objects->sources; + t->repo->objects->sources = odb_source_new(t->repo->objects, t->path.buf, false); + t->repo->objects->sources->next = t->prev_source; + t->repo->objects->sources->will_destroy = will_destroy; + t->repo->disable_ref_updates = true; + + return t->repo->objects->sources; } From 0e73273d089a906d2b8a4e063e6ec6f9771d53e7 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:58 +0200 Subject: [PATCH 18/29] tmp-objdir: manage quarantine as an object directory When creating a quarantine directory via the "tmp-objdir" subsystem we create a new "files" backend that new objects part of the transaction can be written to. In a future commit though we'll move handling of alternates into the "files" backend, and as part of that it will no longer be possible for us to have multiple sources attached to a single object database. In a preceding commit, we have prepared the "files" backend to be able to handle multiple object directories. We don't use that mechanism for alternates yet, but will start doing so in a subsequent commit. But with that infrastructure ready we can already migrate tmp-objdirs over to use this new mechanism. Adapt the subsystem so we create a `struct odb_files_dir` instead of a new "files" source. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- object-file.c | 4 ++-- tmp-objdir.c | 48 +++++++++++++++++++++++++++++------------------- tmp-objdir.h | 4 ++-- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/object-file.c b/object-file.c index 68751dadc3575d..9ffcfdbbcaae79 100644 --- a/object-file.c +++ b/object-file.c @@ -488,7 +488,7 @@ struct odb_transaction_files { enum odb_transaction_flags flags; struct tmp_objdir *objdir; - struct odb_source *quarantine; + struct odb_files_dir *quarantine; struct transaction_packfile packfile; const char *prefix; @@ -1443,7 +1443,7 @@ static int odb_transaction_files_write_pack(struct odb_transaction *base, return -1; } - odb_source_prepare(transaction->quarantine, + odb_source_prepare(&transaction->quarantine->packed->base, ODB_PREPARE_FLUSH_CACHES); } diff --git a/tmp-objdir.c b/tmp-objdir.c index 31a7920be7daea..a3903b6e6dbda8 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -12,13 +12,17 @@ #include "quote.h" #include "odb.h" #include "odb/source.h" +#include "odb/source-files.h" +#include "odb/source-loose.h" +#include "odb/source-packed.h" #include "repository.h" struct tmp_objdir { struct repository *repo; struct strbuf path; struct strvec env; - struct odb_source *prev_source; + struct odb_files_dir *temp_dir; + struct odb_files_dir *orig_dir; int will_destroy; }; @@ -57,18 +61,19 @@ static void tmp_objdir_reparent(const char *old_cwd, */ static void tmp_objdir_restore_source(struct tmp_objdir *t) { - struct odb_source *cur_source = t->repo->objects->sources; + struct odb_source_files *files = odb_source_files_downcast(t->repo->objects->sources); + struct odb_files_dir *cur_dir = files->dirs; - if (strcmp(t->path.buf, cur_source->path)) + if (t->temp_dir != files->dirs) BUG("expected %s as primary object store; found %s", - t->path.buf, cur_source->path); + t->temp_dir->abspath, cur_dir->abspath); - if (cur_source->next != t->prev_source) + if (t->temp_dir->next != t->orig_dir) BUG("we expect the old primary object store to be the first alternate"); t->repo->disable_ref_updates = false; - t->repo->objects->sources = t->prev_source; - odb_source_free(cur_source); + files->dirs = t->orig_dir; + odb_files_dir_free(cur_dir); } int tmp_objdir_destroy(struct tmp_objdir *t) @@ -81,7 +86,7 @@ int tmp_objdir_destroy(struct tmp_objdir *t) if (t == the_tmp_objdir) the_tmp_objdir = NULL; - if (t->prev_source) + if (t->orig_dir) tmp_objdir_restore_source(t); err = remove_dir_recursively(&t->path, 0); @@ -315,11 +320,11 @@ int tmp_objdir_migrate(struct tmp_objdir *t) if (!t) return 0; - if (t->prev_source) { - if (t->repo->objects->sources->will_destroy) + if (t->orig_dir) { + if (t->will_destroy) BUG("migrating an ODB that was marked for destruction"); tmp_objdir_restore_source(t); - t->prev_source = NULL; + t->orig_dir = NULL; } strbuf_addbuf(&src, &t->path); @@ -341,10 +346,12 @@ const char **tmp_objdir_env(const struct tmp_objdir *t) return t->env.v; } -struct odb_source *tmp_objdir_replace_primary_odb(struct tmp_objdir *t, - int will_destroy) +struct odb_files_dir *tmp_objdir_replace_primary_odb(struct tmp_objdir *t, + int will_destroy) { - if (t->prev_source) + struct odb_source_files *files = odb_source_files_downcast(t->repo->objects->sources); + + if (t->temp_dir) BUG("the primary object database is already replaced"); t->will_destroy = will_destroy; @@ -353,11 +360,14 @@ struct odb_source *tmp_objdir_replace_primary_odb(struct tmp_objdir *t, * alternate. Disable ref updates while a temporary source is active, * since the objects in the database may roll back. */ - t->prev_source = t->repo->objects->sources; - t->repo->objects->sources = odb_source_new(t->repo->objects, t->path.buf, false); - t->repo->objects->sources->next = t->prev_source; - t->repo->objects->sources->will_destroy = will_destroy; + t->temp_dir = odb_files_dir_new(t->repo->objects, t->path.buf, false); + t->temp_dir->loose->base.will_destroy = will_destroy; + t->temp_dir->packed->base.will_destroy = will_destroy; + t->temp_dir->next = files->dirs; + + t->orig_dir = files->dirs; + files->dirs = t->temp_dir; t->repo->disable_ref_updates = true; - return t->repo->objects->sources; + return t->temp_dir; } diff --git a/tmp-objdir.h b/tmp-objdir.h index 05f0d08d1065c4..5ed0db1ee03754 100644 --- a/tmp-objdir.h +++ b/tmp-objdir.h @@ -61,7 +61,7 @@ void tmp_objdir_discard_objects(struct tmp_objdir *); * If will_destroy is nonzero, the object directory may not be migrated. Returns * the newly installed primary source. */ -struct odb_source *tmp_objdir_replace_primary_odb(struct tmp_objdir *, - int will_destroy); +struct odb_files_dir *tmp_objdir_replace_primary_odb(struct tmp_objdir *, + int will_destroy); #endif /* TMP_OBJDIR_H */ From c3bda4b2d9a8c19dab5f37819277788486562349 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:35:59 +0200 Subject: [PATCH 19/29] tmp-objdir: replace primary source at creation time When creating transactions via the "tmp-objdir" subsystem callers are expected to first call `tmp_objdir_create()` to create the quarantine directory and then `tmp_objdir_replace_primary_odb()` to activate that quarantine directory as the primary object database source so that all newly written objects are written into it. This dance is performed by all users of temporary object directories, so that makes the setup unnecessarily involved. Merge the logic to replace the primary object database source into `tmp_objdir_create()` to simplify the dance. Note that with the quarantine directory being installed by `tmp_objdir_create()`, users of this function no longer get a pointer to the underlying quarantine directory. So instead of repreparing only that directory, we now reprepare the whole source instead, and as we know that the source has wired up the quarantine directory already, this will cause us to reprepare the quarantine directory, too. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- log-tree.c | 3 +-- object-file.c | 8 ++------ tmp-objdir.c | 47 +++++++++++++++++++---------------------------- tmp-objdir.h | 18 +++++------------- 4 files changed, 27 insertions(+), 49 deletions(-) diff --git a/log-tree.c b/log-tree.c index 83a3c4bf9b16b9..361dfadbb368f0 100644 --- a/log-tree.c +++ b/log-tree.c @@ -1043,10 +1043,9 @@ static int do_remerge_diff(struct rev_info *opt, * into the alternative object store list as the primary. */ if (opt->remerge_diff && !opt->remerge_objdir) { - opt->remerge_objdir = tmp_objdir_create(the_repository, "remerge-diff"); + opt->remerge_objdir = tmp_objdir_create(the_repository, "remerge-diff", 1); if (!opt->remerge_objdir) return error(_("unable to create temporary object directory")); - tmp_objdir_replace_primary_odb(opt->remerge_objdir, 1); } /* Setup merge options */ diff --git a/object-file.c b/object-file.c index 9ffcfdbbcaae79..231e16c3d23033 100644 --- a/object-file.c +++ b/object-file.c @@ -488,7 +488,6 @@ struct odb_transaction_files { enum odb_transaction_flags flags; struct tmp_objdir *objdir; - struct odb_files_dir *quarantine; struct transaction_packfile packfile; const char *prefix; @@ -511,12 +510,10 @@ int odb_transaction_files_prepare(struct odb_transaction *base) if (!transaction || transaction->objdir) return 0; - transaction->objdir = tmp_objdir_create(base->source->odb->repo, transaction->prefix); + transaction->objdir = tmp_objdir_create(base->source->odb->repo, transaction->prefix, 0); if (!transaction->objdir) return error(_("unable to create temporary object directory")); - transaction->quarantine = tmp_objdir_replace_primary_odb(transaction->objdir, 0); - return 0; } @@ -1443,8 +1440,7 @@ static int odb_transaction_files_write_pack(struct odb_transaction *base, return -1; } - odb_source_prepare(&transaction->quarantine->packed->base, - ODB_PREPARE_FLUSH_CACHES); + odb_source_prepare(base->source, ODB_PREPARE_FLUSH_CACHES); } return 0; diff --git a/tmp-objdir.c b/tmp-objdir.c index a3903b6e6dbda8..719b55e580d9e7 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -57,7 +57,7 @@ static void tmp_objdir_reparent(const char *old_cwd, /* * Restore the primary source that was previously replaced by - * `tmp_objdir_replace_primary_odb()`. + * `tmp_objdir_create()`. */ static void tmp_objdir_restore_source(struct tmp_objdir *t) { @@ -157,8 +157,10 @@ static int setup_tmp_objdir(const char *root) } struct tmp_objdir *tmp_objdir_create(struct repository *r, - const char *prefix) + const char *prefix, + int will_destroy) { + struct odb_source_files *files = odb_source_files_downcast(r->objects->sources); static int installed_handlers; struct tmp_objdir *t; @@ -167,6 +169,7 @@ struct tmp_objdir *tmp_objdir_create(struct repository *r, t = xcalloc(1, sizeof(*t)); t->repo = r; + t->will_destroy = will_destroy; strbuf_init(&t->path, 0); strvec_init(&t->env); @@ -204,6 +207,20 @@ struct tmp_objdir *tmp_objdir_create(struct repository *r, env_replace(&t->env, GIT_QUARANTINE_ENVIRONMENT, absolute_path(t->path.buf)); + /* + * Make a new primary source and link the old primary source in as an + * alternate. Disable ref updates while a temporary source is active, + * since the objects in the database may roll back. + */ + t->temp_dir = odb_files_dir_new(t->repo->objects, t->path.buf, false); + t->temp_dir->loose->base.will_destroy = will_destroy; + t->temp_dir->packed->base.will_destroy = will_destroy; + t->temp_dir->next = files->dirs; + + t->orig_dir = files->dirs; + files->dirs = t->temp_dir; + t->repo->disable_ref_updates = true; + return t; } @@ -345,29 +362,3 @@ const char **tmp_objdir_env(const struct tmp_objdir *t) return NULL; return t->env.v; } - -struct odb_files_dir *tmp_objdir_replace_primary_odb(struct tmp_objdir *t, - int will_destroy) -{ - struct odb_source_files *files = odb_source_files_downcast(t->repo->objects->sources); - - if (t->temp_dir) - BUG("the primary object database is already replaced"); - t->will_destroy = will_destroy; - - /* - * Make a new primary source and link the old primary source in as an - * alternate. Disable ref updates while a temporary source is active, - * since the objects in the database may roll back. - */ - t->temp_dir = odb_files_dir_new(t->repo->objects, t->path.buf, false); - t->temp_dir->loose->base.will_destroy = will_destroy; - t->temp_dir->packed->base.will_destroy = will_destroy; - t->temp_dir->next = files->dirs; - - t->orig_dir = files->dirs; - files->dirs = t->temp_dir; - t->repo->disable_ref_updates = true; - - return t->temp_dir; -} diff --git a/tmp-objdir.h b/tmp-objdir.h index 5ed0db1ee03754..a301b01d84cb84 100644 --- a/tmp-objdir.h +++ b/tmp-objdir.h @@ -11,7 +11,7 @@ * Example: * * struct child_process child = CHILD_PROCESS_INIT; - * struct tmp_objdir *t = tmp_objdir_create(repo, "incoming"); + * struct tmp_objdir *t = tmp_objdir_create(repo, "incoming", 0); * strvec_push(&child.args, cmd); * strvec_pushv(&child.env, tmp_objdir_env(t)); * if (!run_command(&child)) && !tmp_objdir_migrate(t)) @@ -25,10 +25,11 @@ struct repository; struct tmp_objdir; /* - * Create a new temporary object directory with the specified prefix; - * returns NULL on failure. + * Create a new temporary object directory with the specified prefix and + * install the directory as the primary write target; returns NULL on failure. */ -struct tmp_objdir *tmp_objdir_create(struct repository *r, const char *prefix); +struct tmp_objdir *tmp_objdir_create(struct repository *r, const char *prefix, + int will_destroy); /* * Return a list of environment strings, suitable for use with @@ -55,13 +56,4 @@ int tmp_objdir_destroy(struct tmp_objdir *); */ void tmp_objdir_discard_objects(struct tmp_objdir *); -/* - * Replaces the writable object store in the current process with the temporary - * object directory and makes the former main object store an alternate. - * If will_destroy is nonzero, the object directory may not be migrated. Returns - * the newly installed primary source. - */ -struct odb_files_dir *tmp_objdir_replace_primary_odb(struct tmp_objdir *, - int will_destroy); - #endif /* TMP_OBJDIR_H */ From 0a51605794045681f99a4a6811ac8d5d9a2da350 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:36:00 +0200 Subject: [PATCH 20/29] odb/source: make `will_destroy` an implementation detail The `struct odb_source::will_destroy` flag tracks whether a source is part of a transaction that we know we'll destruct anyway. If so, the backend can optimize for that particular case, for example by not flushing any data to disk. While the intent is sensible, it assumes that transactions are backed by a separate source that's being linked into the object database. But that may or may not be true, as backends may have significantly better ways to achieve the same. So the assumption doesn't make much sense in the first place, as we're now tracking backend-specific details on the generic `struct odb_source` level. Move the field from the generic source into the "loose" source, as this is the only source that ever makes use of it anyway. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-loose.c | 2 +- odb/source-loose.h | 3 +++ odb/source.h | 5 ----- tmp-objdir.c | 4 +--- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/odb/source-loose.c b/odb/source-loose.c index e95c96591e38bf..1b350f375c88e1 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -610,7 +610,7 @@ static int odb_source_loose_freshen_object(struct odb_source *source, static void close_loose_object(struct odb_source_loose *loose, int fd, const char *filename) { - if (loose->base.will_destroy) + if (loose->will_destroy) goto out; if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT)) diff --git a/odb/source-loose.h b/odb/source-loose.h index 3cf2e1f8f1d5e2..7c7e845fe348f4 100644 --- a/odb/source-loose.h +++ b/odb/source-loose.h @@ -28,6 +28,9 @@ struct odb_source_loose { /* Map between object IDs for loose objects. */ struct loose_object_map *map; + + /* Whether this is a source that will never be committed to disk. */ + int will_destroy; }; struct odb_source_loose *odb_source_loose_new(struct object_database *odb, diff --git a/odb/source.h b/odb/source.h index 5a4e502a265462..973fd19ecd1316 100644 --- a/odb/source.h +++ b/odb/source.h @@ -81,11 +81,6 @@ struct odb_source { */ bool local; - /* - * This object store is ephemeral, so there is no need to fsync. - */ - int will_destroy; - /* * Path to the source. If this is a relative path, it is relative to * the current working directory. diff --git a/tmp-objdir.c b/tmp-objdir.c index 719b55e580d9e7..2f2ffbbc7d06a6 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -14,7 +14,6 @@ #include "odb/source.h" #include "odb/source-files.h" #include "odb/source-loose.h" -#include "odb/source-packed.h" #include "repository.h" struct tmp_objdir { @@ -213,8 +212,7 @@ struct tmp_objdir *tmp_objdir_create(struct repository *r, * since the objects in the database may roll back. */ t->temp_dir = odb_files_dir_new(t->repo->objects, t->path.buf, false); - t->temp_dir->loose->base.will_destroy = will_destroy; - t->temp_dir->packed->base.will_destroy = will_destroy; + t->temp_dir->loose->will_destroy = will_destroy; t->temp_dir->next = files->dirs; t->orig_dir = files->dirs; From 8340ce28bf170bed1126b0b694436e43c37842b9 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:36:01 +0200 Subject: [PATCH 21/29] odb/source-files: extract reading alternates In the next commit we'll add a new callsite that wants to read alternates without having a proper object database source for a given alternate available to it. Prepare for this by extracting the logic into a separate function that only requires an object directory path as input. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-files.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/odb/source-files.c b/odb/source-files.c index bdf1c628684e28..5725a812211d62 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -174,6 +174,24 @@ static int odb_source_files_create_on_disk(struct odb_source *source, return ret; } +static int read_alternates(const char *object_dir, struct strvec *out) +{ + struct strbuf buf = STRBUF_INIT; + char *path; + + path = xstrfmt("%s/info/alternates", object_dir); + if (strbuf_read_file(&buf, path, 1024) < 0) { + warn_on_fopen_errors(path); + free(path); + return 0; + } + parse_alternates(buf.buf, '\n', object_dir, out); + + strbuf_release(&buf); + free(path); + return 0; +} + static void odb_source_files_prepare(struct odb_source *source, enum odb_prepare_flags flags) { @@ -367,20 +385,7 @@ static int odb_source_files_begin_transaction(struct odb_source *source, static int odb_source_files_read_alternates(struct odb_source *source, struct strvec *out) { - struct strbuf buf = STRBUF_INIT; - char *path; - - path = xstrfmt("%s/info/alternates", source->path); - if (strbuf_read_file(&buf, path, 1024) < 0) { - warn_on_fopen_errors(path); - free(path); - return 0; - } - parse_alternates(buf.buf, '\n', source->path, out); - - strbuf_release(&buf); - free(path); - return 0; + return read_alternates(source->path, out); } static int too_many_loose_objects(struct odb_source_files *files, int limit) From 4b3ffd48f9e2fa0c034d0626a026a7b452c27cd7 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:36:02 +0200 Subject: [PATCH 22/29] odb/source-files: move alternates into the backend Originally, when designing pluggable object databases the goal was that the object database can have multiple sources, and every source attached to it could use a different backend. This would have allowed for quite a lot of flexibility, as you could trivially mix and match different kinds of object storages in whatever way you like. But while well-intentioned, this design led to a bunch of conceptual problems: - We're now trying to read objects in source order, whereas we previously tried to read objects via packfiles before trying to read them via loose objects. This led to a performance regression when using alternates or when using a quarantine directory. - Some data structures are supposed to only ever exist once, like for example bitmaps and commit graphs. At the same time, those data structures also span across the union of all objects, so they may cross sources. - It is unclear how we can extend GIT_OBJECT_DIRECTORY or GIT_ALTERNATE_OBJECT_DIRECTORIES to become backend-agnostic in a backwards-compatible way. In general, introducing an object storage extension into the current status quo where alternates may have to be extended to become generic was proving to be painful. - Some mechanisms of alternates assume way too much about how exactly their backends work. Alternate refs for example assume that the alternate is backed by a filesystem path, and that this filesystem path may also allow us to read references. This is not a given though, as backends may not even have local data at all. In short, there are a bunch of conceptual mismatches when we have alternates and pluggable object databases coexist. So while the original idea was nice, it does not result in a system that is easy to reason about. Correct course by moving alternates into the "files" source itself so that it becomes an implementation detail thereof so that we can avoid all of these shortcomings. While it's unfortunate that we cannot easily mix and match sources now, that ability doesn't go away. It's still very much feasible to introduce a new backend that allows for exactly that use case, and such a backend may also be a lot more flexible as we can now add new logic to determine which objects should be stored where. So the original motivation for having per-source backends can still be realized with the new architecture. Note that as part of this move, we also handle the GIT_OBJECT_DIRECTORY and GIT_ALTERNATE_OBJECT_DIRECTORIES environment variables in the "files" backend. This may be surprising at first, but object directories are very much a concept of that backend, too. So these variables would have bad interactions with other backends, and they create a bit of a mismatch with the eventual object storage extension that we plan to introduce. Note that this commit is way larger than I'd like it to be. I'm sorry, I couldn't find a way to split it up further. That being said, most of the changes are straight-forward conversions that go from iterating over sources to iterating over object directories. The more involved changes are moving the infrastructure to track individual sources from "odb.c" and moving them into "odb/source-files.c". This also serves to show a bit of a who's-who of commands that don't work properly with pluggable object databases. Almost all of these commands are related to housekeeping though -- some of them will be converted eventually, like for example commit graphs or MIDXs. But many of them will stay incompatible going forward as they are simply too specific to the "files" backend. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/count-objects.c | 2 +- builtin/fast-import.c | 18 +- builtin/fetch.c | 2 +- builtin/fsck.c | 6 +- builtin/gc.c | 16 +- builtin/index-pack.c | 2 +- builtin/multi-pack-index.c | 6 +- builtin/pack-objects.c | 64 +++--- builtin/prune.c | 2 +- bundle.c | 2 +- commit-graph.c | 28 ++- diagnose.c | 2 +- fetch-pack.c | 2 +- gvfs-helper-client.c | 19 +- http-walker.c | 2 +- http.c | 6 +- loose.c | 16 +- midx.c | 29 +-- odb.c | 440 ++++--------------------------------- odb.h | 36 +-- odb/source-files.c | 312 ++++++++++++++++++++++++-- odb/source-files.h | 23 +- odb/source.c | 5 +- odb/source.h | 25 +-- odb/streaming.c | 8 +- odb/transaction.c | 2 +- pack-bitmap.c | 8 +- packfile.c | 28 +-- packfile.h | 21 +- path.c | 2 +- prune-packed.c | 2 +- repack.c | 4 +- repository.c | 4 +- setup.c | 2 +- t/helper/test-read-graph.c | 2 +- tmp-objdir.c | 4 +- 36 files changed, 506 insertions(+), 646 deletions(-) diff --git a/builtin/count-objects.c b/builtin/count-objects.c index f2abfaccec1c6f..3f7b20fde531a3 100644 --- a/builtin/count-objects.c +++ b/builtin/count-objects.c @@ -118,7 +118,7 @@ int cmd_count_objects(int argc, report_linked_checkout_garbage(the_repository); } - for_each_loose_file_in_source(the_repository->objects->sources, + for_each_loose_file_in_source(the_repository->objects->source, count_loose, count_cruft, NULL, NULL); if (verbose) { diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 32cc302f130461..b93122d2eb3c27 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -895,7 +895,7 @@ static void end_packfile(void) running = 1; clear_delta_base_cache(); if (object_count) { - struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->source); struct packed_git *new_p; struct object_id cur_pack_oid; char *idx_name; @@ -975,7 +975,7 @@ static int store_object( struct object_id *oidout, uintmax_t mark) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); void *out, *delta; struct object_entry *e; unsigned char hdr[96]; @@ -1002,10 +1002,8 @@ static int store_object( return 1; } - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - if (!packfile_list_find_oid(packfile_store_get_packs(files->dirs->packed), &oid)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!packfile_list_find_oid(packfile_store_get_packs(dir->packed), &oid)) continue; e->type = type; e->pack_id = MAX_PACK_ID; @@ -1122,10 +1120,10 @@ static void truncate_pack(struct hashfile_checkpoint *checkpoint) static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) { + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); size_t in_sz = 64 * 1024, out_sz = 64 * 1024; unsigned char *in_buf = xmalloc(in_sz); unsigned char *out_buf = xmalloc(out_sz); - struct odb_source *source; struct object_entry *e; struct object_id oid; unsigned long hdrlen; @@ -1209,10 +1207,8 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) goto out; } - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - if (!packfile_list_find_oid(packfile_store_get_packs(files->dirs->packed), &oid)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!packfile_list_find_oid(packfile_store_get_packs(dir->packed), &oid)) continue; e->type = OBJ_BLOB; e->pack_id = MAX_PACK_ID; diff --git a/builtin/fetch.c b/builtin/fetch.c index c02370a9eb03af..1d35ba2955913b 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -2921,7 +2921,7 @@ int cmd_fetch(int argc, trace2_region_enter("fetch", "write-commit-graph", the_repository); write_commit_graph_reachable(the_repository, - the_repository->objects->sources->path, + the_repository->objects->source->path, commit_graph_flags, NULL); trace2_region_leave("fetch", "write-commit-graph", the_repository); diff --git a/builtin/fsck.c b/builtin/fsck.c index 4a2da5f34a20d8..8cbbaaa486dcf8 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -868,7 +868,6 @@ int cmd_fsck(int argc, OPT_BOOL(0, "references", &check_references, N_("check reference database consistency")), OPT_END(), }; - struct odb_source *source; struct snapshot snap = { .nr = 0, .alloc = 0, @@ -983,13 +982,14 @@ int cmd_fsck(int argc, check_connectivity(repo); if (repo->settings.core_commit_graph) { + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); struct child_process commit_graph_verify = CHILD_PROCESS_INIT; - for (source = repo->objects->sources; source; source = source->next) { + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { child_process_init(&commit_graph_verify); commit_graph_verify.git_cmd = 1; strvec_pushl(&commit_graph_verify.args, "commit-graph", - "verify", "--object-dir", source->path, NULL); + "verify", "--object-dir", dir->abspath, NULL); if (show_progress) strvec_push(&commit_graph_verify.args, "--progress"); else diff --git a/builtin/gc.c b/builtin/gc.c index 7e249b64a68fe7..117055fd239772 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -746,7 +746,7 @@ int cmd_gc(int argc, if (the_repository->settings.gc_write_commit_graph == 1) write_commit_graph_reachable(the_repository, - the_repository->objects->sources->path, + the_repository->objects->source->path, !opts.quiet && !daemonized ? COMMIT_GRAPH_WRITE_PROGRESS : 0, NULL); @@ -1010,7 +1010,7 @@ static int loose_object_auto_condition(struct gc_config *cfg UNUSED) if (loose_object_auto_limit < 0) return 1; - return for_each_loose_file_in_source(the_repository->objects->sources, + return for_each_loose_file_in_source(the_repository->objects->source, loose_object_count, NULL, NULL, &count); } @@ -1042,7 +1042,7 @@ static int pack_loose(struct maintenance_run_opts *opts) int result = 0; struct write_loose_object_data data; struct child_process pack_proc = CHILD_PROCESS_INIT; - struct odb_source *source = r->objects->sources; + struct odb_source *source = r->objects->source; struct odb_files_dir *shared = NULL; /* If set, use the shared object directory. */ @@ -1452,15 +1452,15 @@ static int maintenance_task_cache_local_objs(UNUSED struct maintenance_run_opts return 0; /* If the dest is the same as the local odb path then we do nothing. */ - if (!fspathcmp(r->objects->sources->path, shared_object_dir)) + if (!fspathcmp(r->objects->source->path, shared_object_dir)) goto cleanup; strbuf_addf(&dstdir, "%s/pack", shared_object_dir); - for_each_file_in_pack_dir(r->objects->sources->path, move_pack_to_shared_cache, + for_each_file_in_pack_dir(r->objects->source->path, move_pack_to_shared_cache, dstdir.buf); - ret = for_each_loose_file_in_source(r->objects->sources, + ret = for_each_loose_file_in_source(r->objects->source, move_loose_object_to_shared_cache, NULL, NULL, NULL); @@ -1608,7 +1608,7 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts, int result = 0; struct lock_file lk; struct repository *r = the_repository; - char *lock_path = xstrfmt("%s/maintenance", r->objects->sources->path); + char *lock_path = xstrfmt("%s/maintenance", r->objects->source->path); enum auto_gc_hook_result auto_gc_hook_result = AUTO_GC_HOOK_UNDECIDED; if (repo_hold_lock_file_for_update(r, &lk, lock_path, LOCK_NO_DEREF) < 0) { @@ -3200,7 +3200,7 @@ static int update_background_schedule(const struct maintenance_start_opts *opts, unsigned int i; int result = 0; struct lock_file lk; - char *lock_path = xstrfmt("%s/schedule", the_repository->objects->sources->path); + char *lock_path = xstrfmt("%s/schedule", the_repository->objects->source->path); if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) { if (errno == EEXIST) diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 83855817712d84..f674c8863d2ef1 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1640,7 +1640,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name, if (do_fsck_object && startup_info->have_repository) { struct odb_source_files *files = - odb_source_files_downcast(the_repository->objects->sources); + odb_source_files_downcast(the_repository->objects->source); packfile_store_load_pack(files->dirs->packed, final_index_name, 0); } diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index fc8b494996159a..c48212290cf81b 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -80,7 +80,7 @@ static int parse_object_dir(const struct option *opt, const char *arg, char **value = opt->value; free(*value); if (unset) - *value = xstrdup(the_repository->objects->sources->path); + *value = xstrdup(the_repository->objects->source->path); else *value = real_pathdup(arg, 1); return 0; @@ -426,8 +426,8 @@ int cmd_multi_pack_index(int argc, if (the_repository && the_repository->objects && - the_repository->objects->sources) - opts.object_dir = xstrdup(the_repository->objects->sources->path); + the_repository->objects->source) + opts.object_dir = xstrdup(the_repository->objects->source->path); argc = parse_options(argc, argv, prefix, options, builtin_multi_pack_index_usage, 0); diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 61a82536b4a436..27236c348c8f2e 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1560,11 +1560,10 @@ static int want_cruft_object_mtime(struct repository *r, const struct object_id *oid, unsigned flags, uint32_t mtime) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct packed_git **cache = packfile_store_get_kept_pack_cache(files->dirs->packed, flags); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + struct packed_git **cache = packfile_store_get_kept_pack_cache(dir->packed, flags); for (; *cache; cache++) { struct packed_git *p = *cache; @@ -1747,21 +1746,19 @@ static int want_object_in_pack_mtime(const struct object_id *oid, off_t *found_offset, uint32_t found_mtime) { + struct odb_source_files *files = + odb_source_files_downcast(the_repository->objects->source); int want; struct packfile_list_entry *e; - struct odb_source *source; if (!exclude && local) { /* - * Note that we start iterating at `sources->next` so that we - * skip the local object source. + * Note that we start iterating at `dirs->next` so that we + * skip the local object directory. */ - struct odb_source *source = the_repository->objects->sources->next; - for (; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->dirs->loose->base, oid, NULL, 0, NULL)) + for (struct odb_files_dir *dir = files->dirs->next; dir; dir = dir->next) + if (!odb_source_read_object_info(&dir->loose->base, oid, NULL, 0, NULL)) return 0; - } } /* @@ -1779,9 +1776,8 @@ static int want_object_in_pack_mtime(const struct object_id *oid, *found_offset = 0; } - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct multi_pack_index *m = get_multi_pack_index(files->dirs->packed); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + struct multi_pack_index *m = get_multi_pack_index(dir->packed); struct pack_entry e; if (m && midx_fill_entry(m, oid, &e, NULL) == MIDX_FILL_HIT) { @@ -1791,14 +1787,12 @@ static int want_object_in_pack_mtime(const struct object_id *oid, } } - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - for (e = files->dirs->packed->packs.head; e; e = e->next) { + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + for (e = dir->packed->packs.head; e; e = e->next) { struct packed_git *p = e->pack; want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime); if (!exclude && want > 0) - packfile_list_prepend(&files->dirs->packed->packs, p); + packfile_list_prepend(&dir->packed->packs, p); if (want != -1) return want; } @@ -4155,14 +4149,13 @@ static void add_cruft_object_entry(const struct object_id *oid, enum object_type if (!want_object_in_pack_mtime(oid, 0, &pack, &offset, mtime)) return; if (!pack && type == OBJ_BLOB) { - struct odb_source *source = the_repository->objects->sources; + struct odb_source_files *files = + odb_source_files_downcast(the_repository->objects->source); int found = 0; - for (; !found && source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->dirs->loose->base, oid, NULL, 0, NULL)) + for (struct odb_files_dir *dir = files->dirs; !found && dir; dir = dir->next) + if (!odb_source_read_object_info(&dir->loose->base, oid, NULL, 0, NULL)) found = 1; - } /* * If a traversed tree has a missing blob then we want @@ -4497,7 +4490,8 @@ static int add_object_in_unpacked_pack(const struct object_id *oid, static void add_objects_in_unpacked_packs(void) { - struct odb_source *source; + struct odb_source_files *files = + odb_source_files_downcast(to_pack.repo->objects->source); time_t mtime; struct odb_for_each_object_options opts = { .flags = ODB_FOR_EACH_OBJECT_PACK_ORDER | @@ -4511,13 +4505,11 @@ static void add_objects_in_unpacked_packs(void) .source_infop = &source_info, }; - for (source = to_pack.repo->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - if (!source->local) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!dir->local) continue; - if (odb_source_for_each_object(&files->dirs->packed->base, &oi, + if (odb_source_for_each_object(&dir->packed->base, &oi, add_object_in_unpacked_pack, NULL, &opts)) die(_("cannot open pack index")); } @@ -4561,7 +4553,7 @@ static int add_loose_object(const struct object_id *oid, const char *path, */ static void add_unreachable_loose_objects(struct rev_info *revs) { - for_each_loose_file_in_source(the_repository->objects->sources, + for_each_loose_file_in_source(the_repository->objects->source, add_loose_object, NULL, NULL, revs); } @@ -4625,11 +4617,9 @@ static int force_object_loose(struct odb_source *source, size_t len; int ret; - for (struct odb_source *s = source->odb->sources; s; s = s->next) { - struct odb_source_files *files = odb_source_files_downcast(s); - if (!odb_source_read_object_info(&files->dirs->loose->base, oid, NULL, 0, NULL)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) + if (!odb_source_read_object_info(&dir->loose->base, oid, NULL, 0, NULL)) return 0; - } oi.typep = &type; oi.sizep = &len; @@ -4676,7 +4666,7 @@ static void loosen_unused_packed_objects(void) if (!packlist_find(&to_pack, &oid) && !has_sha1_pack_kept_or_nonlocal(&oid) && !loosened_object_can_be_discarded(&oid, p->mtime)) { - if (force_object_loose(the_repository->objects->sources, + if (force_object_loose(the_repository->objects->source, &oid, &p->mtime)) die(_("unable to force loose object")); loosened_objects_nr++; diff --git a/builtin/prune.c b/builtin/prune.c index a7e4678d117072..97f4c1418bc58e 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -198,7 +198,7 @@ int cmd_prune(int argc, revs.exclude_promisor_objects = 1; } - for_each_loose_file_in_source(repo->objects->sources, + for_each_loose_file_in_source(repo->objects->source, prune_object, prune_cruft, prune_subdir, &revs); prune_packed_objects(show_only ? PRUNE_PACKED_DRY_RUN : 0); diff --git a/bundle.c b/bundle.c index d371fd7a43101d..ef127891db7c24 100644 --- a/bundle.c +++ b/bundle.c @@ -239,7 +239,7 @@ int verify_bundle(struct repository *r, .quiet = 1, }; - if (!r || !r->objects || !r->objects->sources) + if (!r || !r->objects || !r->objects->source) return error(_("need a repository to verify a bundle")); for (i = 0; i < p->nr; i++) { diff --git a/commit-graph.c b/commit-graph.c index 7bcb1aa845848c..2e6844e1bdf21e 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -15,6 +15,7 @@ #include "hash-lookup.h" #include "commit-graph.h" #include "odb.h" +#include "odb/source-files.h" #include "oid-array.h" #include "path.h" #include "alloc.h" @@ -38,7 +39,7 @@ void git_test_write_commit_graph_or_die(struct repository *repo) if (git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags = COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - if (write_commit_graph_reachable(repo, repo->objects->sources->path, flags, NULL)) + if (write_commit_graph_reachable(repo, repo->objects->source->path, flags, NULL)) die("failed to write commit-graph under GIT_TEST_COMMIT_GRAPH"); } @@ -657,7 +658,7 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, CALLOC_ARRAY(oids, count); for (i = 0; i < count; i++) { - struct odb_source *source; + struct odb_source_files *files; if (strbuf_getline_lf(&line, fp) == EOF) break; @@ -670,9 +671,11 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, } valid = 0; - for (source = odb->sources; source; source = source->next) { - char *graph_name = get_split_graph_filename(source->path, line.buf); - struct commit_graph *g = load_commit_graph_one(odb->repo, source->path, graph_name); + + files = odb_source_files_downcast(odb->source); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + char *graph_name = get_split_graph_filename(dir->abspath, line.buf); + struct commit_graph *g = load_commit_graph_one(odb->repo, dir->abspath, graph_name); free(graph_name); @@ -742,7 +745,7 @@ struct commit_graph *read_commit_graph_one(struct repository *repo, */ static struct commit_graph *prepare_commit_graph(struct repository *r) { - struct odb_source *source; + struct odb_source_files *files; /* * Early return if there is no object database or if the commit graph is @@ -773,8 +776,9 @@ static struct commit_graph *prepare_commit_graph(struct repository *r) if (!commit_graph_compatible(r)) return NULL; - for (source = r->objects->sources; source; source = source->next) { - r->objects->commit_graph = read_commit_graph_one(r, source->path); + files = odb_source_files_downcast(r->objects->source); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + r->objects->commit_graph = read_commit_graph_one(r, dir->abspath); if (r->objects->commit_graph) break; } @@ -2003,7 +2007,7 @@ static int fill_oids_from_commits(struct write_commit_graph_context *ctx, static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) { - struct odb_source *source; + struct odb_source_files *files; enum object_type type; struct odb_for_each_object_options opts = { .flags = ODB_FOR_EACH_OBJECT_PACK_ORDER, @@ -2018,9 +2022,9 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) _("Finding commits for commit graph among packed objects"), ctx->approx_nr_objects); - for (source = ctx->r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_for_each_object(&files->dirs->packed->base, &oi, add_packed_commits_oi, + files = odb_source_files_downcast(ctx->r->objects->source); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + odb_source_for_each_object(&dir->packed->base, &oi, add_packed_commits_oi, ctx, &opts); } diff --git a/diagnose.c b/diagnose.c index 9e675eb0d5abd9..2381130c2d5cda 100644 --- a/diagnose.c +++ b/diagnose.c @@ -270,7 +270,7 @@ int create_diagnostics_archive(struct repository *r, strbuf_reset(&buf); strbuf_addstr(&buf, "--add-virtual-file=packs-local.txt:"); - dir_file_stats(odb_source_files_downcast(r->objects->sources)->dirs, &buf); + dir_file_stats(odb_source_files_downcast(r->objects->source)->dirs, &buf); odb_for_each_alternate(r->objects, dir_file_stats, &buf); strvec_push(&archiver_args, buf.buf); diff --git a/fetch-pack.c b/fetch-pack.c index ea97eed0bcdc48..058de0fb976653 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1077,7 +1077,7 @@ static int get_pack(struct fetch_pack_args *args, die(_("fetch-pack: unable to fork off %s"), cmd_name); if (do_keep && (pack_lockfiles || fsck_objects)) { int is_well_formed; - char *pack_lockfile = index_pack_lockfile(the_repository->objects->sources, + char *pack_lockfile = index_pack_lockfile(the_repository->objects->source, cmd.out, &is_well_formed); diff --git a/gvfs-helper-client.c b/gvfs-helper-client.c index 0d88cd65230f30..1458060c4378e5 100644 --- a/gvfs-helper-client.c +++ b/gvfs-helper-client.c @@ -26,7 +26,7 @@ struct gh_server__process { static int gh_server__subprocess_map_initialized; static struct hashmap gh_server__subprocess_map; -static struct odb_source *gh_client__chosen_odb; +static struct odb_files_dir *gh_client__chosen_odb; /* * The "objects" capability has verbs: "get" and "post" and "prefetch". @@ -188,7 +188,8 @@ static void gh_client__update_loose_cache(const char *line) if (get_oid_hex(v1_oid, &oid)) BUG("update_loose_cache: invalid line '%s'", line); - odb_source_loose_cache_add_new_oid(gh_client__chosen_odb, &oid); + odb_source_loose_cache_add_new_oid(&gh_client__chosen_odb->loose->base, + &oid); } /* @@ -271,8 +272,7 @@ static int gh_client__objects__receive_response( } if (ghc & GHC__CREATED__PACKFILE) { - struct odb_source_files *files = odb_source_files_downcast(gh_client__chosen_odb); - odb_source_prepare(&files->dirs->packed->base, + odb_source_prepare(&gh_client__chosen_odb->packed->base, ODB_PREPARE_FLUSH_CACHES); } @@ -292,19 +292,20 @@ static int gh_client__objects__receive_response( */ static void gh_client__choose_odb(void) { - struct odb_source *odb; + struct odb_files_dir *odb; if (gh_client__chosen_odb) return; odb_prepare(the_repository->objects, ODB_PREPARE_FLUSH_CACHES); - gh_client__chosen_odb = the_repository->objects->sources; + gh_client__chosen_odb = + odb_source_files_downcast(the_repository->objects->source)->dirs; if (!gvfs_shared_cache_pathname.len) return; - for (odb = the_repository->objects->sources->next; odb; odb = odb->next) { - if (!fspathcmp(odb->path, gvfs_shared_cache_pathname.buf)) { + for (odb = gh_client__chosen_odb->next; odb; odb = odb->next) { + if (!fspathcmp(odb->abspath, gvfs_shared_cache_pathname.buf)) { gh_client__chosen_odb = odb; return; } @@ -357,7 +358,7 @@ static struct gh_server__process *gh_client__find_long_running_process( strvec_push(&argv, "gvfs-helper"); strvec_push(&argv, "--cache-server=trust"); strvec_pushf(&argv, "--shared-cache=%s", - gh_client__chosen_odb->path); + gh_client__chosen_odb->abspath); /* If gvfs.fallback=false, then don't add --fallback. */ if (!repo_config_get_bool(the_repository, "gvfs.fallback", &fallback) && diff --git a/http-walker.c b/http-walker.c index 0a6c99f47190ba..4f50dd9b8d02a4 100644 --- a/http-walker.c +++ b/http-walker.c @@ -540,7 +540,7 @@ static int fetch_object(struct walker *walker, const struct object_id *oid) } else if (!oideq(&obj_req->oid, &req->real_oid)) { ret = error("File %s has bad hash", hex); } else if (req->rename < 0) { - struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); struct strbuf buf = STRBUF_INIT; odb_loose_path(files->dirs->loose, &buf, &req->oid); ret = error("unable to write sha1 filename %s", buf.buf); diff --git a/http.c b/http.c index c433921fba5ca3..b2068a1e7eb074 100644 --- a/http.c +++ b/http.c @@ -2850,7 +2850,7 @@ int finish_http_pack_request(struct http_pack_request *preq) void http_install_packfile(struct packed_git *p, struct packfile_list *list_to_remove_from) { - struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); packfile_list_remove(list_to_remove_from, p); packfile_store_add_pack(files->dirs->packed, p); } @@ -2978,7 +2978,7 @@ static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb, struct http_object_request *new_http_object_request(const char *base_url, const struct object_id *oid) { - struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); char *hex = oid_to_hex(oid); struct strbuf filename = STRBUF_INIT; struct strbuf prevfile = STRBUF_INIT; @@ -3119,7 +3119,7 @@ void process_http_object_request(struct http_object_request *freq) int finish_http_object_request(struct http_object_request *freq) { - struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); struct stat st; struct strbuf filename = STRBUF_INIT; diff --git a/loose.c b/loose.c index 957bf83e6b09fd..d6a2bb5ba3c877 100644 --- a/loose.c +++ b/loose.c @@ -113,11 +113,10 @@ int loose_object_map_load(struct odb_source_loose *loose) int repo_read_loose_object_map(struct repository *repo) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); - for (source = repo->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - if (loose_object_map_load(files->dirs->loose) < 0) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (loose_object_map_load(dir->loose) < 0) return -1; } @@ -126,7 +125,7 @@ int repo_read_loose_object_map(struct repository *repo) int repo_write_loose_object_map(struct repository *repo) { - struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); kh_oid_map_t *map = files->dirs->loose->map->to_compat; struct lock_file lock; int fd; @@ -231,13 +230,12 @@ int repo_loose_object_map_oid(struct repository *repo, const struct git_hash_algo *to, struct object_id *dest) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); kh_oid_map_t *map; khiter_t pos; - for (source = repo->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct loose_object_map *loose_map = files->dirs->loose->map; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + struct loose_object_map *loose_map = dir->loose->map; if (!loose_map) continue; map = (to == repo->compat_hash_algo) ? diff --git a/midx.c b/midx.c index c0f82c41633bc0..8638ddf0be7f00 100644 --- a/midx.c +++ b/midx.c @@ -829,21 +829,15 @@ void clear_incremental_midx_files_ext(struct odb_source_packed *source, const ch void clear_midx_file(struct repository *r) { - struct odb_source_files *files; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); struct strbuf midx = STRBUF_INIT; - if (r->objects) { - struct odb_source *source; - - for (source = r->objects->sources; source; source = source->next) { - files = odb_source_files_downcast(source); - if (files->dirs->packed->midx) - close_midx(files->dirs->packed->midx); - files->dirs->packed->midx = NULL; - } + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (dir->packed->midx) + close_midx(dir->packed->midx); + dir->packed->midx = NULL; } - files = odb_source_files_downcast(r->objects->sources); get_midx_filename(files->dirs->packed, &midx); if (remove_path(midx.buf)) @@ -858,18 +852,15 @@ void clear_midx_file(struct repository *r) void clear_incremental_midx_files(struct repository *r, const struct strvec *keep_hashes) { - struct odb_source_files *files; - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); struct strbuf chain = STRBUF_INIT; - for (source = r->objects->sources; source; source = source->next) { - files = odb_source_files_downcast(source); - if (files->dirs->packed->midx) - close_midx(files->dirs->packed->midx); - files->dirs->packed->midx = NULL; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (dir->packed->midx) + close_midx(dir->packed->midx); + dir->packed->midx = NULL; } - files = odb_source_files_downcast(r->objects->sources); get_midx_chain_filename(files->dirs->packed, &chain); if (!keep_hashes && remove_path(chain.buf)) diff --git a/odb.c b/odb.c index 38ef5e21083858..9cd831acc764c1 100644 --- a/odb.c +++ b/odb.c @@ -16,12 +16,11 @@ #include "object-file.h" #include "object-name.h" #include "odb.h" -#include "odb/source-inmemory.h" #include "odb/source-files.h" +#include "odb/source-inmemory.h" #include "path.h" #include "pkt-line.h" #include "promisor-remote.h" -#include "quote.h" #include "replace-object.h" #include "run-command.h" #include "setup.h" @@ -36,48 +35,6 @@ #include "trace2.h" #include "write-or-die.h" -/* - * NEEDSWORK: we're using "core.ignoreCase" to deduplicate alternates that - * _may_ be the same. This requires quite a bit of boilerplate for dubious - * benefit: - * - * - Duplicating alternates should really only lead to regressed performance. - * - * - We don't properly resolve symlinks or mointpoints, so we may still end - * up duplicating alternates. - * - * - The value may be lying, in which case we might deduplicate alternates - * that are in fact not mapping to the same directory. - * - * We should investigate whether we can remove this whole mechanism outright. - */ -static int odb_source_paths_cmp(struct object_database *o, - const char *a, const char *b) -{ - if (o->source_paths_icase < 0) { - int icase = 0; - repo_config_get_bool(o->repo, "core.ignorecase", &icase); - o->source_paths_icase = icase; - } - - return o->source_paths_icase ? strcasecmp(a, b) : strcmp(a, b); -} - -static int odb_source_by_path_cmp(const void *cb_data, - const struct hashmap_entry *entry, - const struct hashmap_entry *entry_or_key, - const void *keydata) -{ - struct object_database *o = (struct object_database *)cb_data; - const struct odb_source *source = container_of(entry, const struct odb_source, by_path_entry); - const char *path = keydata; - - if (!path) - path = container_of(entry_or_key, const struct odb_source, by_path_entry)->path; - - return odb_source_paths_cmp(o, source->path, path); -} - int odb_mkstemp(struct object_database *odb, struct strbuf *temp_filename, const char *pattern) { @@ -99,209 +56,6 @@ int odb_mkstemp(struct object_database *odb, return xmkstemp_mode(temp_filename->buf, mode); } -static int gvfs_matched_shared_cache_to_alternate; - -/* - * Return non-zero iff the path is usable as an alternate object database. - */ -static bool odb_is_source_usable(struct object_database *o, const char *path) -{ - extern struct strbuf gvfs_shared_cache_pathname; - struct strbuf normalized_objdir = STRBUF_INIT; - struct hashmap_entry key; - bool usable = false; - - strbuf_realpath(&normalized_objdir, o->sources->path, 1); - - if (!strcmp(path, gvfs_shared_cache_pathname.buf)) { - /* - * `gvfs.sharedCache` is the preferred alternate that we - * will use with `gvfs-helper.exe` to dynamically fetch - * missing objects. It is set during git_default_config(). - * - * Make sure the directory exists on disk before we let the - * stock code discredit it. - */ - struct strbuf buf_pack_foo = STRBUF_INIT; - enum scld_error scld; - - /* - * Force create the "" and "/pack" directories, if - * not present on disk. Append an extra bogus directory to - * get safe_create_leading_directories() to see "/pack" - * as a leading directory of something deeper (which it - * won't create). - */ - strbuf_addf(&buf_pack_foo, "%s/pack/foo", path); - - scld = safe_create_leading_directories(o->repo, buf_pack_foo.buf); - if (scld != SCLD_OK && scld != SCLD_EXISTS) { - error_errno(_("could not create shared-cache ODB '%s'"), - gvfs_shared_cache_pathname.buf); - - strbuf_release(&buf_pack_foo); - - /* - * Pretend no shared-cache was requested and - * effectively fallback to ".git/objects" for - * fetching missing objects. - */ - strbuf_release(&gvfs_shared_cache_pathname); - return 0; - } - - /* - * We know that there is an alternate (either from - * .git/objects/info/alternates or from a memory-only - * entry) associated with the shared-cache directory. - */ - gvfs_matched_shared_cache_to_alternate++; - strbuf_release(&buf_pack_foo); - } - - /* Detect cases where alternate disappeared */ - if (!is_directory(path)) { - warning(_("object directory %s does not exist; " - "check .git/objects/info/alternates"), - path); - goto out; - } - - /* - * Prevent the common mistake of listing the same - * thing twice, or object directory itself. - */ - if (!hashmap_get_size(&o->source_by_path)) { - assert(!o->sources->next); - hashmap_entry_init(&o->sources->by_path_entry, - strihash(o->sources->path)); - hashmap_add(&o->source_by_path, &o->sources->by_path_entry); - } - - if (!odb_source_paths_cmp(o, path, normalized_objdir.buf)) - goto out; - - hashmap_entry_init(&key, strihash(path)); - if (hashmap_get(&o->source_by_path, &key, path)) - goto out; - - usable = true; - -out: - strbuf_release(&normalized_objdir); - return usable; -} - -void parse_alternates(const char *string, - int sep, - const char *relative_base, - struct strvec *out) -{ - struct strbuf pathbuf = STRBUF_INIT; - struct strbuf buf = STRBUF_INIT; - - if (!string || !*string) - return; - - while (*string) { - const char *end; - - strbuf_reset(&buf); - strbuf_reset(&pathbuf); - - if (*string == '#') { - /* comment; consume up to next separator */ - end = strchrnul(string, sep); - } else if (*string == '"' && !unquote_c_style(&buf, string, &end)) { - /* - * quoted path; unquote_c_style has copied the - * data for us and set "end". Broken quoting (e.g., - * an entry that doesn't end with a quote) falls - * back to the unquoted case below. - */ - } else { - /* normal, unquoted path */ - end = strchrnul(string, sep); - strbuf_add(&buf, string, end - string); - } - - if (*end) - end++; - string = end; - - if (!buf.len) - continue; - - if (!is_absolute_path(buf.buf) && relative_base) { - strbuf_realpath(&pathbuf, relative_base, 1); - strbuf_addch(&pathbuf, '/'); - } - strbuf_addbuf(&pathbuf, &buf); - - strbuf_reset(&buf); - if (!strbuf_realpath(&buf, pathbuf.buf, 0)) { - warning(_("unable to normalize alternate object " - "path: %s"), pathbuf.buf); - continue; - } - - /* - * The trailing slash after the directory name is given by - * this function at the end. Remove duplicates. - */ - while (buf.len && buf.buf[buf.len - 1] == '/') - strbuf_setlen(&buf, buf.len - 1); - - strvec_push(out, buf.buf); - } - - strbuf_release(&pathbuf); - strbuf_release(&buf); -} - -static struct odb_source *odb_add_alternate_recursively(struct object_database *odb, - const char *source, - int depth) -{ - struct odb_source *alternate = NULL; - struct strvec sources = STRVEC_INIT; - - if (!odb_is_source_usable(odb, source)) - goto error; - - alternate = odb_source_new(odb, source, false); - - /* add the alternate entry */ - *odb->sources_tail = alternate; - odb->sources_tail = &(alternate->next); - - hashmap_entry_init(&alternate->by_path_entry, strihash(alternate->path)); - if (hashmap_get(&odb->source_by_path, &alternate->by_path_entry, - alternate->path)) - BUG("source must not yet exist"); - hashmap_add(&odb->source_by_path, &alternate->by_path_entry); - - /* recursively add alternates */ - odb_source_read_alternates(alternate, &sources); - if (sources.nr && depth + 1 > 5) { - error(_("%s: ignoring alternate object stores, nesting too deep"), - source); - } else { - for (size_t i = 0; i < sources.nr; i++) - odb_add_alternate_recursively(odb, sources.v[i], depth + 1); - } - - error: - strvec_clear(&sources); - return alternate; -} - -struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, - const char *dir) -{ - return odb_add_alternate_recursively(odb, dir, 0); -} - char *compute_alternate_path(const char *path, struct strbuf *err) { char *ref_git = NULL; @@ -473,64 +227,22 @@ void odb_for_each_alternate_ref(struct object_database *odb, int odb_for_each_alternate(struct object_database *odb, odb_for_each_alternate_fn cb, void *payload) { - struct odb_source *alternate; + struct odb_source_files *files = odb_source_files_downcast(odb->source); int r = 0; - for (alternate = odb->sources->next; alternate; alternate = alternate->next) { - r = cb(odb_source_files_downcast(alternate)->dirs, payload); + for (struct odb_files_dir *dir = files->dirs->next; dir; dir = dir->next) { + r = cb(dir, payload); if (r) break; } return r; } -static void odb_prepare_alternates(struct object_database *odb, - const char *alternate_db) -{ - extern struct strbuf gvfs_shared_cache_pathname; - struct strvec sources = STRVEC_INIT; - - parse_alternates(alternate_db, PATH_SEP, NULL, &sources); - odb_source_read_alternates(odb->sources, &sources); - - for (size_t i = 0; i < sources.nr; i++) - odb_add_alternate_recursively(odb, sources.v[i], 0); - - if (gvfs_shared_cache_pathname.len && - !gvfs_matched_shared_cache_to_alternate) { - /* - * There is no entry in .git/objects/info/alternates for - * the requested shared-cache directory. Therefore, the - * odb-list does not contain this directory. - * - * Force this directory into the odb-list as an in-memory - * alternate. Implicitly create the directory on disk, if - * necessary. - * - * See GIT_ALTERNATE_OBJECT_DIRECTORIES for another example - * of this kind of usage. - * - * Note: This has the net-effect of allowing Git to treat - * `gvfs.sharedCache` as an unofficial alternate. This - * usage should be discouraged for compatbility reasons - * with other tools in the overall Git ecosystem (that - * won't know about this trick). It would be much better - * for us to update .git/objects/info/alternates instead. - * The code here is considered a backstop. - */ - parse_alternates(gvfs_shared_cache_pathname.buf, '\n', NULL, &sources); - odb_source_read_alternates(odb->sources, &sources); - for (size_t i = 0; i < sources.nr; i++) - odb_add_alternate_recursively(odb, sources.v[i], 0); - - } - - strvec_clear(&sources); -} - int odb_has_alternates(struct object_database *odb) { - return !!odb->sources->next; + if (odb->source->type != ODB_SOURCE_FILES) + return 0; + return !!odb_source_files_downcast(odb->source)->dirs->next; } #define CAP_GET (1u<<0) @@ -700,16 +412,13 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * while (1) { extern int core_use_gvfs_helper; - struct odb_source *source; - for (source = odb->sources; source; source = source->next) { - ret = odb_source_read_object_info(source, real, oi, flags, - corrupt_err.len ? NULL : &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; - } + ret = odb_source_read_object_info(odb->source, real, oi, flags, + corrupt_err.len ? NULL : &corrupt_err); + if (!ret) + goto out; + if (ret != ODB_READ_NOT_FOUND) + corrupt = true; if (core_use_gvfs_helper && !tried_gvfs_helper && !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { @@ -739,15 +448,13 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * * caches or reload on-disk state. */ if (!(flags & OBJECT_INFO_QUICK)) { - for (source = odb->sources; source; source = source->next) { - ret = odb_source_read_object_info(source, real, oi, - flags | OBJECT_INFO_SECOND_READ, - corrupt_err.len ? NULL : &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; - } + ret = odb_source_read_object_info(odb->source, real, oi, + flags | OBJECT_INFO_SECOND_READ, + corrupt_err.len ? NULL : &corrupt_err); + if (!ret) + goto out; + if (ret != ODB_READ_NOT_FOUND) + corrupt = true; if (gvfs_virtualize_objects(odb->repo) && !tried_hook && !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { // TODO Assert or at least trace2 if gvfs-helper @@ -1003,12 +710,8 @@ int odb_freshen_object(struct object_database *odb, const struct object_id *oid, int skip_virtualized_objects) { - struct odb_source *source; - for (source = odb->sources; source; source = source->next) - if (odb_source_freshen_object(source, oid, NULL, - skip_virtualized_objects)) - return 1; - return 0; + return odb_source_freshen_object(odb->source, oid, NULL, + skip_virtualized_objects); } int odb_for_each_object_ext(struct object_database *odb, @@ -1017,18 +720,7 @@ int odb_for_each_object_ext(struct object_database *odb, void *cb_data, const struct odb_for_each_object_options *opts) { - int ret; - - for (struct odb_source *source = odb->sources; source; source = source->next) { - if (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local) - continue; - - ret = odb_source_for_each_object(source, request, cb, cb_data, opts); - if (ret) - return ret; - } - - return 0; + return odb_source_for_each_object(odb->source, request, cb, cb_data, opts); } int odb_for_each_object(struct object_database *odb, @@ -1047,7 +739,6 @@ int odb_count_objects(struct object_database *odb, enum odb_count_objects_flags flags, unsigned long *out) { - struct odb_source *source; unsigned long count = 0; int ret; @@ -1056,15 +747,9 @@ int odb_count_objects(struct object_database *odb, return 0; } - for (source = odb->sources; source; source = source->next) { - unsigned long c; - - ret = odb_source_count_objects(source, flags, &c); - if (ret < 0) - goto out; - - count += c; - } + ret = odb_source_count_objects(odb->source, flags, &count); + if (ret < 0) + goto out; odb->object_count = count; odb->object_count_valid = 1; @@ -1135,13 +820,7 @@ int odb_find_abbrev_len(struct object_database *odb, goto out; } - for (struct odb_source *source = odb->sources; source; source = source->next) { - ret = odb_source_find_abbrev_len(source, oid, len, &len); - if (ret) - goto out; - } - - ret = 0; + ret = odb_source_find_abbrev_len(odb->source, oid, len, &len); *out = len; out: @@ -1197,7 +876,7 @@ int odb_write_object_ext(struct object_database *odb, compat_oid_p = &compat_oid; } - return odb_source_write_object(odb->sources, buf, len, type, + return odb_source_write_object(odb->source, buf, len, type, oid, compat_oid_p, NULL, flags); } @@ -1205,19 +884,19 @@ int odb_write_object_stream(struct object_database *odb, struct odb_stream *stream, struct object_id *oid) { - return odb_source_write_object_stream(odb->sources, stream, oid); + return odb_source_write_object_stream(odb->source, stream, oid); } int odb_optimize(struct object_database *odb, const struct odb_optimize_options *opts) { - return odb_source_optimize(odb->sources, opts); + return odb_source_optimize(odb->source, opts); } bool odb_optimize_required(struct object_database *odb, const struct odb_optimize_options *opts) { - return odb_source_optimize_required(odb->sources, opts); + return odb_source_optimize_required(odb->source, opts); } void odb_generate_pack_options_release(struct odb_generate_pack_options *opts) @@ -1231,9 +910,9 @@ int odb_generate_pack(struct object_database *odb, struct odb_pack_generator **out, const struct odb_generate_pack_options *opts) { - if (!odb->sources->generate_pack) + if (!odb->source->generate_pack) return error(_("primary object source does not support generating packfiles")); - return odb_source_generate_pack(odb->sources, out, opts); + return odb_source_generate_pack(odb->source, out, opts); } int odb_pack_generator_finish(struct odb_pack_generator *generator) @@ -1244,55 +923,29 @@ int odb_pack_generator_finish(struct odb_pack_generator *generator) struct object_database *odb_new(struct repository *repo, enum odb_new_flags flags) { - char *primary_source = NULL, *secondary_sources = NULL; struct object_database *o; CALLOC_ARRAY(o, 1); o->repo = repo; pthread_mutex_init(&o->replace_mutex, NULL); - hashmap_init(&o->source_by_path, odb_source_by_path_cmp, o, 0); - o->source_paths_icase = -1; - - if (flags & ODB_NEW_HONOR_ENV) { - primary_source = xstrdup_or_null(getenv(DB_ENVIRONMENT)); - secondary_sources = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); - } - if (!primary_source) - primary_source = xstrfmt("%s/objects", repo->commondir); - o->sources = odb_source_new(o, primary_source, true); - o->sources_tail = &o->sources->next; + o->source = odb_source_new(o, flags); o->inmemory_objects = &odb_source_inmemory_new(o)->base; - odb_prepare_alternates(o, secondary_sources); - - free(secondary_sources); - free(primary_source); return o; } void odb_close(struct object_database *o) { - struct odb_source *source; - for (source = o->sources; source; source = source->next) - odb_source_close(source); + odb_source_close(o->source); close_commit_graph(o); } static void odb_free_sources(struct object_database *o) { - while (o->sources) { - struct odb_source *next; - - next = o->sources->next; - odb_source_free(o->sources); - o->sources = next; - } - + odb_source_free(o->source); odb_source_free(o->inmemory_objects); o->inmemory_objects = NULL; - - hashmap_clear(&o->source_by_path); } void odb_free(struct object_database *o) @@ -1311,24 +964,12 @@ void odb_free(struct object_database *o) void odb_prepare(struct object_database *o, enum odb_prepare_flags flags) { - struct odb_source *source; - obj_read_lock(); - /* - * Reprepare alt odbs, in case the alternates file was modified - * during the course of this process. This only _adds_ odbs to - * the linked list, so existing odbs will continue to exist for - * the lifetime of the process. Consequently, we don't have to - * reprocess GIT_ALTERNATE_OBJECT_DIRECTORIES here. - */ - if (flags & ODB_PREPARE_FLUSH_CACHES) { - odb_prepare_alternates(o, NULL); + if (flags & ODB_PREPARE_FLUSH_CACHES) o->object_count_valid = 0; - } - for (source = o->sources; source; source = source->next) - odb_source_prepare(source, flags); + odb_source_prepare(o->source, flags); obj_read_unlock(); } @@ -1340,8 +981,5 @@ void odb_reprepare(struct object_database *o) int odb_fsck(struct object_database *odb, struct odb_fsck_options *options) { - int ret = 0; - for (struct odb_source *source = odb->sources; source; source = source->next) - ret |= odb_source_fsck(source, options); - return ret; + return odb_source_fsck(odb->source, options); } diff --git a/odb.h b/odb.h index 364f512b998a2f..484f93ba72a66d 100644 --- a/odb.h +++ b/odb.h @@ -28,13 +28,16 @@ char *compute_alternate_path(const char *path, struct strbuf *err); /* * The object database encapsulates access to objects in a repository. It - * manages one or more sources that store the actual objects which are - * configured via alternates. + * manages the object source as well as auxiliary data structures required to + * manage objects. */ struct object_database { /* Repository that owns this database. */ struct repository *repo; + /* The source backing this object database. */ + struct odb_source *source; + /* * State of current object database transaction. Only one * transaction may be pending at a time. Is NULL when no transaction is @@ -42,27 +45,6 @@ struct object_database { */ struct odb_transaction *transaction; - /* - * Set of all object directories; the main directory is first (and - * cannot be NULL after initialization). Subsequent directories are - * alternates. - */ - struct odb_source *sources; - struct odb_source **sources_tail; - - /* - * Map of object database sources, keyed by their respective paths. - * This map is used to detect the case where the same source is - * registered multiple times. - */ - struct hashmap source_by_path; - - /* - * Whether source paths shall be compared case-insensitively, as - * determined by "core.ignoreCase". - */ - int source_paths_icase; - /* * Objects that should be substituted by other objects * (see git-replace(1)). @@ -264,8 +246,7 @@ int odb_has_alternates(struct object_database *odb); * recursive alternates it points to), but do not modify the on-disk alternates * file. */ -struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, - const char *dir); +void odb_add_to_alternates_memory(struct object_database *odb, const char *dir); /* * Read an object from the database. Returns the object data and assigns object @@ -834,11 +815,6 @@ int odb_generate_pack(struct object_database *odb, */ int odb_pack_generator_finish(struct odb_pack_generator *generator); -void parse_alternates(const char *string, - int sep, - const char *relative_base, - struct strvec *out); - int read_object_process(struct repository *r, const struct object_id *oid); #endif /* ODB_H */ diff --git a/odb/source-files.c b/odb/source-files.c index 5725a812211d62..90bacc12eaa157 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -15,6 +15,7 @@ #include "packfile.h" #include "path.h" #include "promisor-remote.h" +#include "quote.h" #include "repack.h" #include "run-command.h" #include "strbuf.h" @@ -71,6 +72,7 @@ static void odb_source_files_free(struct odb_source *source) odb_files_dir_free(files->dirs); files->dirs = next; } + hashmap_clear(&files->dirs_by_path); odb_source_release(&files->base); free(files); @@ -174,6 +176,193 @@ static int odb_source_files_create_on_disk(struct odb_source *source, return ret; } +/* + * NEEDSWORK: we're using "core.ignoreCase" to deduplicate alternates that + * _may_ be the same. This requires quite a bit of boilerplate for dubious + * benefit: + * + * - Duplicating alternates should really only lead to regressed performance. + * + * - We don't properly resolve symlinks or mointpoints, so we may still end + * up duplicating alternates. + * + * - The value may be lying, in which case we might deduplicate alternates + * that are in fact not mapping to the same directory. + * + * We should investigate whether we can remove this whole mechanism outright. + */ +static int odb_files_dir_paths_cmp(struct odb_source_files *files, + const char *a, const char *b) +{ + if (files->dirs_paths_icase < 0) { + int icase = 0; + repo_config_get_bool(files->base.odb->repo, "core.ignorecase", &icase); + files->dirs_paths_icase = icase; + } + + return files->dirs_paths_icase ? strcasecmp(a, b) : strcmp(a, b); +} + +static int odb_files_dir_by_path_cmp(const void *cb_data, + const struct hashmap_entry *entry, + const struct hashmap_entry *entry_or_key, + const void *keydata) +{ + struct odb_source_files *files = (struct odb_source_files *)cb_data; + const struct odb_files_dir *dir = container_of(entry, const struct odb_files_dir, by_path_entry); + const char *path = keydata; + + if (!path) + path = container_of(entry_or_key, const struct odb_files_dir, by_path_entry)->abspath; + + return odb_files_dir_paths_cmp(files, dir->abspath, path); +} + +static int gvfs_matched_shared_cache_to_alternate; + +/* + * Return non-zero iff the path is usable as an alternate object directory. + */ +static bool odb_files_dir_is_usable(struct odb_source_files *files, + const char *path) +{ + extern struct strbuf gvfs_shared_cache_pathname; + struct strbuf normalized_objdir = STRBUF_INIT; + struct hashmap_entry key; + bool usable = false; + + strbuf_realpath(&normalized_objdir, files->dirs->abspath, 1); + + if (!strcmp(path, gvfs_shared_cache_pathname.buf)) { + /* + * `gvfs.sharedCache` is the preferred alternate that we + * will use with `gvfs-helper.exe` to dynamically fetch + * missing objects. It is set during git_default_config(). + * + * Make sure the directory exists on disk before we let the + * stock code discredit it. + */ + struct strbuf buf_pack_foo = STRBUF_INIT; + enum scld_error scld; + + /* + * Append a bogus directory so that the leading-directory + * helper creates both the object directory and its pack/. + */ + strbuf_addf(&buf_pack_foo, "%s/pack/foo", path); + scld = safe_create_leading_directories(files->base.odb->repo, + buf_pack_foo.buf); + strbuf_release(&buf_pack_foo); + if (scld != SCLD_OK && scld != SCLD_EXISTS) { + error_errno(_("could not create shared-cache ODB '%s'"), + gvfs_shared_cache_pathname.buf); + strbuf_release(&gvfs_shared_cache_pathname); + goto out; + } + + gvfs_matched_shared_cache_to_alternate++; + } + + /* Detect cases where alternate disappeared */ + if (!is_directory(path)) { + warning(_("object directory %s does not exist; " + "check .git/objects/info/alternates"), + path); + goto out; + } + + /* + * Prevent the common mistake of listing the same + * thing twice, or object directory itself. + */ + if (!hashmap_get_size(&files->dirs_by_path)) { + assert(!files->dirs->next); + hashmap_entry_init(&files->dirs->by_path_entry, + strihash(files->dirs->abspath)); + hashmap_add(&files->dirs_by_path, &files->dirs->by_path_entry); + } + + if (!odb_files_dir_paths_cmp(files, path, normalized_objdir.buf)) + goto out; + + hashmap_entry_init(&key, strihash(path)); + if (hashmap_get(&files->dirs_by_path, &key, path)) + goto out; + + usable = true; + +out: + strbuf_release(&normalized_objdir); + return usable; +} + +static void parse_alternates(const char *string, + int sep, + const char *relative_base, + struct strvec *out) +{ + struct strbuf pathbuf = STRBUF_INIT; + struct strbuf buf = STRBUF_INIT; + + if (!string || !*string) + return; + + while (*string) { + const char *end; + + strbuf_reset(&buf); + strbuf_reset(&pathbuf); + + if (*string == '#') { + /* comment; consume up to next separator */ + end = strchrnul(string, sep); + } else if (*string == '"' && !unquote_c_style(&buf, string, &end)) { + /* + * quoted path; unquote_c_style has copied the + * data for us and set "end". Broken quoting (e.g., + * an entry that doesn't end with a quote) falls + * back to the unquoted case below. + */ + } else { + /* normal, unquoted path */ + end = strchrnul(string, sep); + strbuf_add(&buf, string, end - string); + } + + if (*end) + end++; + string = end; + + if (!buf.len) + continue; + + if (!is_absolute_path(buf.buf) && relative_base) { + strbuf_realpath(&pathbuf, relative_base, 1); + strbuf_addch(&pathbuf, '/'); + } + strbuf_addbuf(&pathbuf, &buf); + + strbuf_reset(&buf); + if (!strbuf_realpath(&buf, pathbuf.buf, 0)) { + warning(_("unable to normalize alternate object " + "path: %s"), pathbuf.buf); + continue; + } + + /* + * The trailing slash after the directory name is given by + * this function at the end. Remove duplicates. + */ + while (buf.len && buf.buf[buf.len - 1] == '/') + strbuf_setlen(&buf, buf.len - 1); + + strvec_push(out, buf.buf); + } + + strbuf_release(&pathbuf); + strbuf_release(&buf); +} + static int read_alternates(const char *object_dir, struct strvec *out) { struct strbuf buf = STRBUF_INIT; @@ -192,11 +381,89 @@ static int read_alternates(const char *object_dir, struct strvec *out) return 0; } +static void odb_add_alternate_recursively(struct odb_source_files *files, + const char *path, + int depth) +{ + struct odb_files_dir *alternate; + struct strvec alternates = STRVEC_INIT; + + if (!odb_files_dir_is_usable(files, path)) + goto out; + + alternate = odb_files_dir_new(files->base.odb, path, false); + + /* add the alternate entry */ + *files->dirs_tail = alternate; + files->dirs_tail = &(alternate->next); + + hashmap_entry_init(&alternate->by_path_entry, strihash(alternate->abspath)); + if (hashmap_get(&files->dirs_by_path, &alternate->by_path_entry, + alternate->abspath)) + BUG("object directory must not yet exist"); + hashmap_add(&files->dirs_by_path, &alternate->by_path_entry); + + /* recursively add alternates */ + read_alternates(alternate->abspath, &alternates); + if (alternates.nr && depth + 1 > 5) { + error(_("%s: ignoring alternate object stores, nesting too deep"), + path); + } else { + for (size_t i = 0; i < alternates.nr; i++) + odb_add_alternate_recursively(files, alternates.v[i], depth + 1); + } + + out: + strvec_clear(&alternates); +} + +void odb_add_to_alternates_memory(struct object_database *odb, const char *dir) +{ + odb_add_alternate_recursively(odb_source_files_downcast(odb->source), + dir, 0); +} + +static void odb_prepare_alternates(struct odb_source_files *files, + const char *alternate_db) +{ + extern struct strbuf gvfs_shared_cache_pathname; + struct strvec alternates = STRVEC_INIT; + + parse_alternates(alternate_db, PATH_SEP, NULL, &alternates); + read_alternates(files->dirs->abspath, &alternates); + + for (size_t i = 0; i < alternates.nr; i++) + odb_add_alternate_recursively(files, alternates.v[i], 0); + + if (gvfs_shared_cache_pathname.len && + !gvfs_matched_shared_cache_to_alternate) { + /* + * Keep gvfs.sharedCache available as an in-memory alternate + * when the on-disk alternates file does not list it. + */ + parse_alternates(gvfs_shared_cache_pathname.buf, '\n', NULL, + &alternates); + for (size_t i = 0; i < alternates.nr; i++) + odb_add_alternate_recursively(files, alternates.v[i], 0); + } + + strvec_clear(&alternates); +} + static void odb_source_files_prepare(struct odb_source *source, enum odb_prepare_flags flags) { struct odb_source_files *files = odb_source_files_downcast(source); + /* + * Reprepare alternates, in case the alternates file was modified + * during the course of this process. This only _adds_ directories to + * the linked list, so existing directories will continue to exist + * for the lifetime of the process. + */ + if (flags & ODB_PREPARE_FLUSH_CACHES) + odb_prepare_alternates(files, NULL); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { odb_source_prepare(&dir->loose->base, flags); odb_source_prepare(&dir->packed->base, flags); @@ -1014,23 +1281,15 @@ static int odb_source_files_fsck(struct odb_source *source, struct odb_files_dir *odb_source_files_find_dir(struct object_database *odb, const char *obj_dir) { + struct odb_source_files *files = odb_source_files_downcast(odb->source); char *obj_dir_real = real_pathdup(obj_dir, 1); struct strbuf odb_path_real = STRBUF_INIT; - struct odb_files_dir *dir = NULL; - struct odb_source *source; - - for (source = odb->sources; source; source = source->next) { - struct odb_source_files *files; - - if (source->type != ODB_SOURCE_FILES) - continue; - files = odb_source_files_downcast(source); + struct odb_files_dir *dir; - strbuf_realpath(&odb_path_real, files->dirs->abspath, 1); - if (!strcmp(obj_dir_real, odb_path_real.buf)) { - dir = files->dirs; + for (dir = files->dirs; dir; dir = dir->next) { + strbuf_realpath(&odb_path_real, dir->abspath, 1); + if (!strcmp(obj_dir_real, odb_path_real.buf)) break; - } } free(obj_dir_real); @@ -1039,14 +1298,27 @@ struct odb_files_dir *odb_source_files_find_dir(struct object_database *odb, con } struct odb_source_files *odb_source_files_new(struct object_database *odb, - const char *path, - bool local) + enum odb_new_flags flags) { struct odb_source_files *files; + char *object_dir = NULL; + char *alternates = NULL; + + if (flags & ODB_NEW_HONOR_ENV) { + object_dir = xstrdup_or_null(getenv(DB_ENVIRONMENT)); + alternates = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); + } + if (!object_dir) + object_dir = xstrfmt("%s/objects", odb->repo->commondir); CALLOC_ARRAY(files, 1); - odb_source_init(&files->base, odb, ODB_SOURCE_FILES, path, local); - files->dirs = odb_files_dir_new(odb, path, local); + odb_source_init(&files->base, odb, ODB_SOURCE_FILES, object_dir, true); + + hashmap_init(&files->dirs_by_path, odb_files_dir_by_path_cmp, files, 0); + files->dirs_paths_icase = -1; + + files->dirs = odb_files_dir_new(odb, object_dir, true); + files->dirs_tail = &files->dirs->next; files->base.free = odb_source_files_free; files->base.close = odb_source_files_close; @@ -1072,8 +1344,12 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, * is not (yet) possible though because we access and assume relative * paths in the primary ODB source in some user-facing functionality. */ - if (!is_absolute_path(path)) + if (!is_absolute_path(object_dir)) chdir_notify_register(odb_source_files_reparent, files); + odb_prepare_alternates(files, alternates); + + free(object_dir); + free(alternates); return files; } diff --git a/odb/source-files.h b/odb/source-files.h index 36af0c1b8ba059..837e8903044c2a 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -18,6 +18,12 @@ struct odb_files_dir { /* List of alternate object directories. */ struct odb_files_dir *next; + /* + * Entry in the files source's map of directories, keyed by this + * directory's path. + */ + struct hashmap_entry by_path_entry; + /* The two sources derived from this object directory. */ struct odb_source_loose *loose; struct odb_source_packed *packed; @@ -46,12 +52,25 @@ struct odb_source_files { * alternates. */ struct odb_files_dir *dirs; + struct odb_files_dir **dirs_tail; + + /* + * Map of object directories, keyed by their respective paths. This + * map is used to detect the case where the same directory is + * registered multiple times. + */ + struct hashmap dirs_by_path; + + /* + * Whether directory paths shall be compared case-insensitively, as + * determined by "core.ignoreCase". + */ + int dirs_paths_icase; }; /* Allocate and initialize a new object source. */ struct odb_source_files *odb_source_files_new(struct object_database *odb, - const char *path, - bool local); + enum odb_new_flags flags); /* * Optimize the files object database source by repacking loose objects and diff --git a/odb/source.c b/odb/source.c index 30188b806d524d..b25ef14df83ca8 100644 --- a/odb/source.c +++ b/odb/source.c @@ -24,10 +24,9 @@ const char *odb_source_type_to_name(enum odb_source_type type) } struct odb_source *odb_source_new(struct object_database *odb, - const char *path, - bool local) + enum odb_new_flags flags) { - return &odb_source_files_new(odb, path, local)->base; + return &odb_source_files_new(odb, flags)->base; } void odb_source_init(struct odb_source *source, diff --git a/odb/source.h b/odb/source.h index 973fd19ecd1316..eddee095a047d2 100644 --- a/odb/source.h +++ b/odb/source.h @@ -49,24 +49,9 @@ struct odb_create_on_disk_options { /* * The source is the part of the object database that stores the actual * objects. It thus encapsulates the logic to read and write the specific - * on-disk format. An object database can have multiple sources: - * - * - The primary source, which is typically located in "$GIT_DIR/objects". - * This is where new objects are usually written to. - * - * - Alternate sources, which are configured via "objects/info/alternates" or - * via the GIT_ALTERNATE_OBJECT_DIRECTORIES environment variable. These - * alternate sources are only used to read objects. + * on-disk format. */ struct odb_source { - struct odb_source *next; - - /* - * Entry in the object database's map of sources, keyed by this - * source's path. - */ - struct hashmap_entry by_path_entry; - /* Object database that owns this object source. */ struct object_database *odb; @@ -332,13 +317,11 @@ struct odb_source { }; /* - * Allocate and initialize a new source for the given object database located - * at `path`. `local` indicates whether or not the source is the local and thus - * primary object source of the object database. + * Allocate and initialize a new source for the given object database. The path + * of the source is derived from repository paths. */ struct odb_source *odb_source_new(struct object_database *odb, - const char *path, - bool local); + enum odb_new_flags flags); /* * Initialize the source for the given object database located at `path`. diff --git a/odb/streaming.c b/odb/streaming.c index 282c2412f1e0a2..d4af0f3466885a 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -184,12 +184,8 @@ static int istream_source(struct odb_stream **out, struct object_database *odb, const struct object_id *oid) { - struct odb_source *source; - - for (source = odb->sources; source; source = source->next) - if (!odb_source_read_object_stream(out, source, oid)) - return 0; - + if (!odb_source_read_object_stream(out, odb->source, oid)) + return 0; return open_istream_incore(out, odb, oid); } diff --git a/odb/transaction.c b/odb/transaction.c index f6f20088ec1fde..69824a3921d18f 100644 --- a/odb/transaction.c +++ b/odb/transaction.c @@ -12,7 +12,7 @@ int odb_transaction_begin(struct object_database *odb, if (odb->transaction) return error(_("object database transaction already pending")); - ret = odb_source_begin_transaction(odb->sources, out, flags); + ret = odb_source_begin_transaction(odb->source, out, flags); if (!ret) odb->transaction = *out; diff --git a/pack-bitmap.c b/pack-bitmap.c index d1d6964f029f4f..e77cad81f85073 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -712,15 +712,13 @@ static int open_bitmap_for_source(struct odb_source_packed *source, static int open_bitmap(struct repository *r, struct bitmap_index *bitmap_git) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); bool found = false; assert(!bitmap_git->map); - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - if (!open_bitmap_for_source(files->dirs->packed, bitmap_git)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!open_bitmap_for_source(dir->packed, bitmap_git)) found = true; /* diff --git a/packfile.c b/packfile.c index 0aaf1331699587..35e03601f4d25f 100644 --- a/packfile.c +++ b/packfile.c @@ -273,14 +273,13 @@ static void scan_windows(struct packed_git *p, static int unuse_one_window(struct object_database *odb) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(odb->source); struct packfile_list_entry *e; struct packed_git *lru_p = NULL; struct pack_window *lru_w = NULL, *lru_l = NULL; - for (source = odb->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - for (e = files->dirs->packed->packs.head; e; e = e->next) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + for (e = dir->packed->packs.head; e; e = e->next) scan_windows(e->pack, &lru_p, &lru_w, &lru_l); } @@ -450,15 +449,14 @@ static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struc static int close_one_pack(struct repository *r) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); struct packfile_list_entry *e; struct packed_git *lru_p = NULL; struct pack_window *mru_w = NULL; int accept_windows_inuse = 1; - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - for (e = files->dirs->packed->packs.head; e; e = e->next) { + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + for (e = dir->packed->packs.head; e; e = e->next) { if (e->pack->pack_fd == -1) continue; find_lru_pack(e->pack, &lru_p, &mru_w, &accept_windows_inuse); @@ -1930,11 +1928,10 @@ struct packed_git **packfile_store_get_kept_pack_cache(struct odb_source_packed int has_object_pack(struct repository *r, const struct object_id *oid) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->dirs->packed->base, oid, NULL, 0, NULL)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!odb_source_read_object_info(&dir->packed->base, oid, NULL, 0, NULL)) return 1; } @@ -1944,14 +1941,13 @@ int has_object_pack(struct repository *r, const struct object_id *oid) int has_object_kept_pack(struct repository *r, const struct object_id *oid, unsigned flags) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); struct pack_entry e; - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { struct packed_git **cache; - cache = packfile_store_get_kept_pack_cache(files->dirs->packed, flags); + cache = packfile_store_get_kept_pack_cache(dir->packed, flags); for (; *cache; cache++) { struct packed_git *p = *cache; diff --git a/packfile.h b/packfile.h index cbb851d43d8c9e..b5e9740ef76c7e 100644 --- a/packfile.h +++ b/packfile.h @@ -69,20 +69,20 @@ void packfile_store_add_pack(struct odb_source_packed *store, struct packfile_list_entry *packfile_store_get_packs(struct odb_source_packed *store); struct repo_for_each_pack_data { - struct odb_source *source; + struct odb_files_dir *dir; struct packfile_list_entry *entry; }; static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct repository *repo) { struct repo_for_each_pack_data data = { 0 }; + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); - for (struct odb_source *source = repo->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct packfile_list_entry *entry = packfile_store_get_packs(files->dirs->packed); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + struct packfile_list_entry *entry = packfile_store_get_packs(dir->packed); if (!entry) continue; - data.source = source; + data.dir = dir; data.entry = entry; break; } @@ -92,23 +92,22 @@ static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct static inline void repo_for_each_pack_data_next(struct repo_for_each_pack_data *data) { - struct odb_source *source; + struct odb_files_dir *dir; data->entry = data->entry->next; if (data->entry) return; - for (source = data->source->next; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct packfile_list_entry *entry = packfile_store_get_packs(files->dirs->packed); + for (dir = data->dir->next; dir; dir = dir->next) { + struct packfile_list_entry *entry = packfile_store_get_packs(dir->packed); if (!entry) continue; - data->source = source; + data->dir = dir; data->entry = entry; return; } - data->source = NULL; + data->dir = NULL; data->entry = NULL; } diff --git a/path.c b/path.c index 08da7fb4278758..80f7a8f39e2b40 100644 --- a/path.c +++ b/path.c @@ -396,7 +396,7 @@ static void adjust_git_path(struct repository *repo, strbuf_splice(buf, 0, buf->len, repo->index_file, strlen(repo->index_file)); else if (dir_prefix(base, "objects")) - replace_dir(buf, git_dir_len + 7, repo->objects->sources->path); + replace_dir(buf, git_dir_len + 7, repo->objects->source->path); else if (repo_settings_get_hooks_path(repo) && dir_prefix(base, "hooks")) replace_dir(buf, git_dir_len + 5, repo_settings_get_hooks_path(repo)); else if (repo->different_commondir) diff --git a/prune-packed.c b/prune-packed.c index d49dc11957c7a6..b6749b8d13233d 100644 --- a/prune-packed.c +++ b/prune-packed.c @@ -40,7 +40,7 @@ void prune_packed_objects(int opts) progress = start_delayed_progress(the_repository, _("Removing duplicate objects"), 256); - for_each_loose_file_in_source(the_repository->objects->sources, + for_each_loose_file_in_source(the_repository->objects->source, prune_object, NULL, prune_subdir, &opts); /* Ensure we show 100% before finishing progress */ diff --git a/repack.c b/repack.c index e20431690c96ba..53c34197f3a1c7 100644 --- a/repack.c +++ b/repack.c @@ -59,7 +59,7 @@ void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, bool wrote_incremental_midx) { struct strbuf buf = STRBUF_INIT; - struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); struct multi_pack_index *m = get_multi_pack_index(files->dirs->packed); strbuf_addf(&buf, "%s.pack", base_name); if (m && files->base.local && midx_contains_pack(m, buf.buf)) { @@ -158,7 +158,7 @@ void existing_packs_collect(struct existing_packs *existing, string_list_append(&existing->non_kept_packs, buf.buf); } - existing->source = existing->repo->objects->sources; + existing->source = existing->repo->objects->source; string_list_sort(&existing->kept_packs); string_list_sort(&existing->non_kept_packs); diff --git a/repository.c b/repository.c index bb88f6b5fc6daf..fdb60fd96629bc 100644 --- a/repository.c +++ b/repository.c @@ -126,9 +126,9 @@ const char *repo_get_common_dir(struct repository *repo) const char *repo_get_object_directory(struct repository *repo) { - if (!repo->objects->sources) + if (!repo->objects->source) BUG("repository hasn't been set up"); - return repo->objects->sources->path; + return repo->objects->source->path; } const char *repo_get_index_file(struct repository *repo) diff --git a/setup.c b/setup.c index cb4cb9a5dbe188..f5bba2e084d0a8 100644 --- a/setup.c +++ b/setup.c @@ -2737,7 +2737,7 @@ void create_object_database(struct repository *repo, repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); - if (odb_source_create_on_disk(repo->objects->sources, &opts) < 0) + if (odb_source_create_on_disk(repo->objects->source, &opts) < 0) die(_("failed creating object database")); } diff --git a/t/helper/test-read-graph.c b/t/helper/test-read-graph.c index a75c817e472c7a..62e23840522270 100644 --- a/t/helper/test-read-graph.c +++ b/t/helper/test-read-graph.c @@ -78,7 +78,7 @@ int cmd__read_graph(int argc, const char **argv) int ret = 0; setup_git_directory(the_repository); - source = the_repository->objects->sources; + source = the_repository->objects->source; prepare_repo_settings(the_repository); diff --git a/tmp-objdir.c b/tmp-objdir.c index 2f2ffbbc7d06a6..3debb60270f7b0 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -60,7 +60,7 @@ static void tmp_objdir_reparent(const char *old_cwd, */ static void tmp_objdir_restore_source(struct tmp_objdir *t) { - struct odb_source_files *files = odb_source_files_downcast(t->repo->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(t->repo->objects->source); struct odb_files_dir *cur_dir = files->dirs; if (t->temp_dir != files->dirs) @@ -159,7 +159,7 @@ struct tmp_objdir *tmp_objdir_create(struct repository *r, const char *prefix, int will_destroy) { - struct odb_source_files *files = odb_source_files_downcast(r->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); static int installed_handlers; struct tmp_objdir *t; From 17a9afc637e32d822b2a4356765bd159cf6ebce2 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 8 Oct 2026 10:36:03 +0200 Subject: [PATCH 23/29] odb/source: drop `read_alternates` callback Now that alternates are an implementation detail of the "files" backend there is no reason anymore to expose them anywhere else. The callback to read alternates is thus no longer required. Remove it. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-files.c | 7 ------- odb/source-inmemory.c | 7 ------- odb/source-loose.c | 7 ------- odb/source-packed.c | 7 ------- odb/source.h | 27 --------------------------- 5 files changed, 55 deletions(-) diff --git a/odb/source-files.c b/odb/source-files.c index 90bacc12eaa157..517c448c4cf998 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -649,12 +649,6 @@ static int odb_source_files_begin_transaction(struct odb_source *source, return odb_transaction_files_begin(source, out, flags); } -static int odb_source_files_read_alternates(struct odb_source *source, - struct strvec *out) -{ - return read_alternates(source->path, out); -} - static int too_many_loose_objects(struct odb_source_files *files, int limit) { unsigned long loose_count; @@ -1334,7 +1328,6 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, files->base.write_object = odb_source_files_write_object; files->base.write_object_stream = odb_source_files_write_object_stream; files->base.begin_transaction = odb_source_files_begin_transaction; - files->base.read_alternates = odb_source_files_read_alternates; files->base.optimize = odb_source_files_optimize; files->base.optimize_required = odb_source_files_optimize_required; files->base.generate_pack = odb_source_files_generate_pack; diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 93345666505db3..3f63f216919f83 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -323,12 +323,6 @@ static int odb_source_inmemory_begin_transaction(struct odb_source *source UNUSE return error("in-memory source does not support transactions"); } -static int odb_source_inmemory_read_alternates(struct odb_source *source UNUSED, - struct strvec *out UNUSED) -{ - return 0; -} - static void odb_source_inmemory_close(struct odb_source *source UNUSED) { } @@ -391,7 +385,6 @@ struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb) source->base.write_object_stream = odb_source_inmemory_write_object_stream; source->base.freshen_object = odb_source_inmemory_freshen_object; source->base.begin_transaction = odb_source_inmemory_begin_transaction; - source->base.read_alternates = odb_source_inmemory_read_alternates; return source; } diff --git a/odb/source-loose.c b/odb/source-loose.c index 1b350f375c88e1..cc942013588fe2 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -996,12 +996,6 @@ static int odb_source_loose_begin_transaction(struct odb_source *source UNUSED, return error("loose source does not support transactions"); } -static int odb_source_loose_read_alternates(struct odb_source *source UNUSED, - struct strvec *out UNUSED) -{ - return 0; -} - void odb_source_loose_cache_add_new_oid(struct odb_source *source, const struct object_id *oid) { @@ -1177,7 +1171,6 @@ struct odb_source_loose *odb_source_loose_new(struct object_database *odb, loose->base.write_object = odb_source_loose_write_object; loose->base.write_object_stream = odb_source_loose_write_object_stream; loose->base.begin_transaction = odb_source_loose_begin_transaction; - loose->base.read_alternates = odb_source_loose_read_alternates; if (!is_absolute_path(loose->base.path)) chdir_notify_register(odb_source_loose_reparent, loose); diff --git a/odb/source-packed.c b/odb/source-packed.c index df1418c9e39256..6eb863f99188a3 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -688,12 +688,6 @@ static int odb_source_packed_begin_transaction(struct odb_source *source UNUSED, return error("packed backend cannot begin transactions"); } -static int odb_source_packed_read_alternates(struct odb_source *source UNUSED, - struct strvec *out UNUSED) -{ - return 0; -} - void (*report_garbage)(unsigned seen_bits, const char *path); static void report_helper(const struct string_list *list, @@ -1020,7 +1014,6 @@ struct odb_source_packed *odb_source_packed_new(struct object_database *odb, packed->base.write_object = odb_source_packed_write_object; packed->base.write_object_stream = odb_source_packed_write_object_stream; packed->base.begin_transaction = odb_source_packed_begin_transaction; - packed->base.read_alternates = odb_source_packed_read_alternates; if (!is_absolute_path(path)) chdir_notify_register(odb_source_packed_reparent, packed); diff --git a/odb/source.h b/odb/source.h index eddee095a047d2..59a2f17fd71b88 100644 --- a/odb/source.h +++ b/odb/source.h @@ -259,19 +259,6 @@ struct odb_source { struct odb_transaction **out, enum odb_transaction_flags flags); - /* - * This callback is expected to read the list of alternate object - * database sources connected to it and write them into the `strvec`. - * - * The result is expected to be paths to the alternates. All paths must - * be resolved to absolute paths. - * - * The callback is expected to return 0 on success, a negative error - * code otherwise. - */ - int (*read_alternates)(struct odb_source *source, - struct strvec *out); - /* * This callback is expected to optimize the object database source. * Returns 0 on success, a negative error code otherwise. @@ -510,20 +497,6 @@ static inline int odb_source_write_object_stream(struct odb_source *source, return source->write_object_stream(source, stream, oid); } -/* - * Read the list of alternative object database sources from the given backend - * and populate the `strvec` with them. The listing is not recursive -- that - * is, if any of the yielded alternate sources has alternates itself, those - * will not be yielded as part of this function call. - * - * Return 0 on success, a negative error code otherwise. - */ -static inline int odb_source_read_alternates(struct odb_source *source, - struct strvec *out) -{ - return source->read_alternates(source, out); -} - /* * Create a new transaction that can be used to write objects into a temporary * staging area. The objects will only be persisted when the transaction is From 18e7904e24fecc2a2b62f0ffef6c38270b234aa0 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 15 Oct 2026 07:27:55 +0200 Subject: [PATCH 24/29] odb/source-files: fix performance when reading objects with alternates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the move towards pluggable object databases we reorganized how we access alternates. Before the introduction of pluggability, we were reading objects by first scanning through packfiles from all alternates before scanning through all loose object directories. With the change though, we are now iterating through the sources one by one, checking each of them for whether they contain the object either as a packed or loose object. Checking whether an object exists in packfiles is comparatively cheap because the list of packs is cached, and the pack indices are likely to be in the page cache. In contrast to that, checking whether an object exists in loose format always incurs one stat(3p) syscall. So scanning for objects one alternate at a time led to a performance regression in in the case where most of the objects are contained in a packfile of an alternate, as reported in [1]. We could fix that issue by introducing another flag that tells the backends to only search the optimized storage backend. But that is adding more complexity to the object database layers that is ultimately only really required because of the way the "files" backend stores its data. So for an alternate backend, such new infrastructure would not really be a good fit. But since then we have moved handling of alternates into the "files" backend itself. One of the consequences of this is that this backend is now having full control over all of its sub-sources, and that also makes it way easier for us to optimize for this access pattern again. Do so by first trying to read objects from packfiles located in any of its object directories before trying to read loose objects. The following benchmark demonstrates that this significantly improves performance in some cases. The benchmark uses the Git repository itself and sets up its object directory as an alternate, while the primary object directory is simply an empty object directory. It then reads all commits enumerated via `git rev-list HEAD` with git-cat-file(1): Benchmark 1: git cat-file --batch-check --unordered (rev = HEAD~) Time (mean ± σ): 179.4 ms ± 2.6 ms [User: 114.6 ms, System: 61.4 ms] Range (min … max): 177.1 ms … 187.7 ms 16 runs Benchmark 2: git cat-file --batch-check --unordered (rev = HEAD) Time (mean ± σ): 95.0 ms ± 1.5 ms [User: 79.8 ms, System: 15.0 ms] Range (min … max): 93.0 ms … 97.9 ms 30 runs Summary git cat-file --batch-check --unordered (rev = HEAD) ran 1.89 ± 0.04 times faster than git cat-file --batch-check --unordered (rev = HEAD~) [1]: Reported-by: Wolfgang Kritzinger Helped-by: Tyrie Vella Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-files.c | 5 +++-- t/t5615-alternate-env.sh | 43 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/odb/source-files.c b/odb/source-files.c index 517c448c4cf998..8199e9429437c0 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -478,6 +478,7 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source { struct odb_source_files *files = odb_source_files_downcast(source); enum odb_read_status status = ODB_READ_NOT_FOUND; + enum odb_read_status ret; /* * Reading an object may fail even though the object exists, for @@ -486,15 +487,15 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source * prefer the first such error in case multiple reads have failed. */ for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { - enum odb_read_status ret; - ret = odb_source_read_object_info(&dir->packed->base, oid, oi, flags, status == ODB_READ_NOT_FOUND ? errmsg : NULL); if (!ret) return 0; if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND) status = ret; + } + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { ret = odb_source_read_object_info(&dir->loose->base, oid, oi, flags, status == ODB_READ_NOT_FOUND ? errmsg : NULL); if (!ret) diff --git a/t/t5615-alternate-env.sh b/t/t5615-alternate-env.sh index 1bfeccdeb49958..4b801b93433239 100755 --- a/t/t5615-alternate-env.sh +++ b/t/t5615-alternate-env.sh @@ -88,4 +88,47 @@ test_expect_success !MINGW 'broken quoting falls back to interpreting raw' ' EOF ' +test_expect_success 'packs across sources are checked before loose objects' ' + # Regression test for a performance issue in which an object that + # resides in an alternate as a packed object caused a spurious loose + # object lookup (a filesystem stat) on the main object store before the + # alternate packfile was consulted. Reading such an object must resolve + # to the alternate packfile, never to a loose copy in the main store. + # + # Build an alternate whose object "B" is stored as a delta in a + # packfile. git deltifies successive versions of a tracked file, so the + # older, shorter blob "B" becomes a delta against the newer, longer + # blob "O". A loose object has no delta base, so %(deltabase) tells us + # which store answered the read: the alternate pack (O) or a loose copy + # (the zero oid). + git init alt-src && + test_seq 1 200 >alt-src/file && + git -C alt-src add file && + git -C alt-src commit -q -m base && + B=$(git -C alt-src rev-parse HEAD:file) && + git -C alt-src cat-file blob "$B" >b-content && + test_seq 1 210 >alt-src/file && + git -C alt-src add file && + git -C alt-src commit -q -m more && + O=$(git -C alt-src rev-parse HEAD:file) && + git -C alt-src repack -adf --window=10 --depth=50 && + + # Precondition: in the alternate, B is a delta based on O. + echo "$B" >in && + echo "$O" >expect && + git -C alt-src cat-file --batch-check="%(deltabase)" actual && + test_cmp expect actual && + + # Main repo: write B as a loose object, before any alternate is active. + git init main && + git -C main hash-object -w --stdin /dev/null && + test_path_is_file "main/.git/objects/$(test_oid_to_path "$B")" && + + # With the alternate active, B must resolve to the alternate packfile + # (deltabase O), not to the main store loose copy (deltabase zero oid). + GIT_ALTERNATE_OBJECT_DIRECTORIES="$PWD/alt-src/.git/objects" \ + git -C main cat-file --batch-check="%(deltabase)" actual && + test_cmp expect actual +' + test_done From c8a4c6d325752ae18386068b21500c1896685afa Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 15 Oct 2026 07:27:56 +0200 Subject: [PATCH 25/29] odb/source-files: fix performance when streaming objects with alternates This patch is essentially the equivalent to the preceding patch, but for streaming objects instead of just reading them directly. Exercising performance is a bit more complicated here as streaming only really kicks in for large objects. Furthermore, I expect that the impact is not as big as with the last commit, either, because for large objects the time to stat the file will probably be dwarfed by the time it takes to read the object itself. But even though this is probably not as impactful, it makes sense anyway to do this change to align behaviour across both of these functions. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb/source-files.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/odb/source-files.c b/odb/source-files.c index 8199e9429437c0..79eb8348d4952d 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -514,8 +514,10 @@ static int odb_source_files_read_object_stream(struct odb_stream **out, struct odb_source_files *files = odb_source_files_downcast(source); for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) - if (!odb_source_read_object_stream(out, &dir->packed->base, oid) || - !odb_source_read_object_stream(out, &dir->loose->base, oid)) + if (!odb_source_read_object_stream(out, &dir->packed->base, oid)) + return 0; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) + if (!odb_source_read_object_stream(out, &dir->loose->base, oid)) return 0; return -1; From a268ddde2adb2c312f50c7ce913c5accf16a7e5a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 15 Oct 2026 07:27:57 +0200 Subject: [PATCH 26/29] odb/source-files: make the second read an implementation detail When looking up an object fails we perform a second read in all sources with the `OBJECT_INFO_SECOND_READ` flag set. This flag tells sources that the first lookup has failed and that they should reload any on-disk state that may have changed in the meantime so that the object may be surfaced after all. The "packed" source uses this to reprepare its packfiles, whereas the "loose" source short-circuits the second read because there are no caches to flush. This logic lives in the generic object database layer even though it is highly specific to how the "files" backend handles its reads. But now that alternates are an implementation detail of that source, we can easily move that logic into the backend, as well. Move the logic into the "files" source and drop the flag. Note that the "packed" source also used this flag to figure out whether it should try to surface an object via a packfile that should have the object as determined by a multi-pack index, but where the packfile has vanished. The multi-pack index is stale in that case, and that scenario should really only happen when the process is racing with a concurrent maintenance task. This code is adapted to instead use `OBJECT_INFO_QUICK`, which means that the fallback may trigger on the first read already, but not in a case where callers know to handle false negatives already. In the worst case this could cause another scan through all packfiles covered by the multi-pack index. But that scan should be fast enough anyway, and it only triggers when we hit the race. So the effect thereof should be limited. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 33 ++++++++++----------------------- odb.h | 12 ------------ odb/source-files.c | 13 +++++++++++++ odb/source-loose.c | 10 ---------- odb/source-packed.c | 15 +++------------ odb/source.h | 7 ------- 6 files changed, 26 insertions(+), 64 deletions(-) diff --git a/odb.c b/odb.c index 9cd831acc764c1..fdeb120f3a603b 100644 --- a/odb.c +++ b/odb.c @@ -442,29 +442,16 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * */ } - /* - * When the object hasn't been found we try a second read and - * tell the sources so. This may cause them to invalidate - * caches or reload on-disk state. - */ - if (!(flags & OBJECT_INFO_QUICK)) { - ret = odb_source_read_object_info(odb->source, real, oi, - flags | OBJECT_INFO_SECOND_READ, - corrupt_err.len ? NULL : &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; - if (gvfs_virtualize_objects(odb->repo) && !tried_hook && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { - // TODO Assert or at least trace2 if gvfs-helper - // TODO was tried and failed and then read-object-hook - // TODO is successful at getting this object. - tried_hook = 1; - // TODO BUG? Should 'oid' be 'real' ? - if (!read_object_process(odb->repo, oid)) - goto retry; - } + if (!(flags & OBJECT_INFO_QUICK) && + gvfs_virtualize_objects(odb->repo) && !tried_hook && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + // TODO Assert or at least trace2 if gvfs-helper + // TODO was tried and failed and then read-object-hook + // TODO is successful at getting this object. + tried_hook = 1; + // TODO BUG? Should 'oid' be 'real' ? + if (!read_object_process(odb->repo, oid)) + goto retry; } /* Check if it is a missing object */ diff --git a/odb.h b/odb.h index 484f93ba72a66d..503a9b598c7dcf 100644 --- a/odb.h +++ b/odb.h @@ -390,18 +390,6 @@ enum object_info_flags { /* Die if object corruption (not just an object being missing) was detected. */ OBJECT_INFO_DIE_IF_CORRUPT = (1 << 3), - /* - * We have already tried reading the object, but it couldn't be found - * via any of the attached sources, and are now doing a second read. - * This second read asks the individual sources to also evaluate - * whether any on-disk state may have changed that may have caused the - * object to appear. - * - * This flag is for internal use, only. The second read only occurs - * when `OBJECT_INFO_QUICK` was not passed. - */ - OBJECT_INFO_SECOND_READ = (1 << 4), - /* * This is meant for bulk prefetching of missing blobs in a partial * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK. diff --git a/odb/source-files.c b/odb/source-files.c index 79eb8348d4952d..f5497c41abe890 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -479,7 +479,9 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source struct odb_source_files *files = odb_source_files_downcast(source); enum odb_read_status status = ODB_READ_NOT_FOUND; enum odb_read_status ret; + bool retry = false; +retry: /* * Reading an object may fail even though the object exists, for * example because it is corrupt. Report this failure to the caller in @@ -487,6 +489,9 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source * prefer the first such error in case multiple reads have failed. */ for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (retry) + odb_source_prepare(&dir->packed->base, ODB_PREPARE_FLUSH_CACHES); + ret = odb_source_read_object_info(&dir->packed->base, oid, oi, flags, status == ODB_READ_NOT_FOUND ? errmsg : NULL); if (!ret) @@ -496,6 +501,9 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source } for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (retry) + break; + ret = odb_source_read_object_info(&dir->loose->base, oid, oi, flags, status == ODB_READ_NOT_FOUND ? errmsg : NULL); if (!ret) @@ -504,6 +512,11 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source status = ret; } + if (!retry && !(flags & OBJECT_INFO_QUICK)) { + retry = true; + goto retry; + } + return status; } diff --git a/odb/source-loose.c b/odb/source-loose.c index cc942013588fe2..befdd6b00bed65 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -223,16 +223,6 @@ static enum odb_read_status odb_source_loose_read_object_info(struct odb_source { struct odb_source_loose *loose = odb_source_loose_downcast(source); static struct strbuf buf = STRBUF_INIT; - - /* - * The second read shouldn't cause new loose objects to show up, unless - * there was a race condition with a secondary process. We don't care - * about this case though, so we simply skip reading loose objects a - * second time. - */ - if (flags & OBJECT_INFO_SECOND_READ) - return ODB_READ_NOT_FOUND; - odb_loose_path(loose, &buf, oid); return read_object_info_from_path(loose, buf.buf, oid, oi, flags, errmsg); } diff --git a/odb/source-packed.c b/odb/source-packed.c index 6eb863f99188a3..6cf5be12c01fc4 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -52,12 +52,11 @@ static int find_pack_entry(struct odb_source_packed *store, * packs, and repreparing the on-disk pack set does not reload the * borrowed, cached MIDX, so scan its packs directly for the survivor. * - * Do this only on the second read, by which point repreparing packs has - * already had a chance to find an object merely relocated into a new, - * uncovered pack; only a genuine hidden duplicate reaches here. + * Skip the scan for quick lookups, which are expected to accept false + * negatives in favor of speed. */ if (midx_result == MIDX_FILL_OWNER_UNAVAILABLE && - (flags & OBJECT_INFO_SECOND_READ)) { + !(flags & OBJECT_INFO_QUICK)) { struct multi_pack_index *m = store->midx; uint32_t i; @@ -86,14 +85,6 @@ static enum odb_read_status odb_source_packed_read_object_info(struct odb_source struct pack_entry e; int ret; - /* - * In case the first read didn't surface the object, we have to reload - * packfiles. This may cause us to discover new packfiles that have - * been added since the last time we have prepared the packfile store. - */ - if (flags & OBJECT_INFO_SECOND_READ) - odb_source_prepare(source, ODB_PREPARE_FLUSH_CACHES); - if (!find_pack_entry(packed, oid, &e, flags, &bad_pack)) { /* * The lookup may have failed because the object is known to be diff --git a/odb/source.h b/odb/source.h index 59a2f17fd71b88..c852dd9a25b0a8 100644 --- a/odb/source.h +++ b/odb/source.h @@ -119,13 +119,6 @@ struct odb_source { * - `OBJECT_INFO_QUICK` indicates it is fine to use caches without * re-verifying the data. * - * - `OBJECT_INFO_SECOND_READ` indicates that the initial object - * lookup has failed and that the object sources should check - * whether any of its on-disk state has changed that may have - * caused the object to appear. Sources are free to ignore the - * second read in case they know that the first read would have - * already surfaced the object without reloading any on-disk state. - * * The callback is expected to return an `enum odb_read_status`. Please * refer to the individual values that can be returned. In case reading * the object has failed with a generic error and `errmsg` is non-NULL, From 4e588d61f6869c2baf4bdcb5b820274b9c0eee80 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 15 Oct 2026 07:27:58 +0200 Subject: [PATCH 27/29] odb: remove read loop for found objects Before moving alternates into the "files" backend we had to handle alternates in `do_oid_object_info_extended()`. This made the retry logic somewhat complex, as we had to juggle reloading of the sources, the second read and fetching missing objects. And to handle the potential retries we loop around that logic multiple times. Nowadays though, the reload and retry logic are located in the "files" backend itself. Consequently, using a loop is a bit on the heavy side and makes the code harder to reason about. Simplify the logic by removing the loop. Instead, we explicitly re-read exactly once after having fetched objects from a promisor remote. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 115 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 57 insertions(+), 58 deletions(-) diff --git a/odb.c b/odb.c index fdeb120f3a603b..427da284c3b2e1 100644 --- a/odb.c +++ b/odb.c @@ -392,10 +392,10 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * const struct object_id *oid, struct object_info *oi, unsigned flags) { + extern int core_use_gvfs_helper; struct strbuf corrupt_err = STRBUF_INIT; const struct object_id *real = oid; enum odb_read_status ret; - int already_retried = 0; bool corrupt = false; int tried_hook = 0; int tried_gvfs_helper = 0; @@ -410,74 +410,73 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * if (!odb_source_read_object_info(odb->inmemory_objects, oid, oi, flags, NULL)) return 0; - while (1) { - extern int core_use_gvfs_helper; + ret = odb_source_read_object_info(odb->source, real, oi, flags, + corrupt_err.len ? NULL : &corrupt_err); + if (!ret) + goto out; + if (ret != ODB_READ_NOT_FOUND) + corrupt = true; - ret = odb_source_read_object_info(odb->source, real, oi, flags, - corrupt_err.len ? NULL : &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; + if (core_use_gvfs_helper && !tried_gvfs_helper && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + enum gh_client__created ghc; - if (core_use_gvfs_helper && !tried_gvfs_helper && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { - enum gh_client__created ghc; + gh_client__get_immediate(real, &ghc); + tried_gvfs_helper = 1; - gh_client__get_immediate(real, &ghc); - tried_gvfs_helper = 1; + /* + * Retry the lookup IIF `gvfs-helper` created one + * or more new packfiles or loose objects. + */ + if (ghc != GHC__CREATED__NOTHING) + goto retry; - /* - * Retry the lookup IIF `gvfs-helper` created one - * or more new packfiles or loose objects. - */ - if (ghc != GHC__CREATED__NOTHING) - continue; + /* + * If `gvfs-helper` fails, we just want to return -1. + * But allow the other providers to have a shot at it. + * (At least until we have a chance to consolidate + * them.) + */ + } - /* - * If `gvfs-helper` fails, we just want to return -1. - * But allow the other providers to have a shot at it. - * (At least until we have a chance to consolidate - * them.) - */ - } + if (!(flags & OBJECT_INFO_QUICK) && + gvfs_virtualize_objects(odb->repo) && !tried_hook && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + // TODO Assert or at least trace2 if gvfs-helper + // TODO was tried and failed and then read-object-hook + // TODO is successful at getting this object. + tried_hook = 1; + // TODO BUG? Should 'oid' be 'real' ? + if (!read_object_process(odb->repo, oid)) + goto retry; + } - if (!(flags & OBJECT_INFO_QUICK) && - gvfs_virtualize_objects(odb->repo) && !tried_hook && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { - // TODO Assert or at least trace2 if gvfs-helper - // TODO was tried and failed and then read-object-hook - // TODO is successful at getting this object. - tried_hook = 1; - // TODO BUG? Should 'oid' be 'real' ? - if (!read_object_process(odb->repo, oid)) - goto retry; - } + /* Try to fetch the missing object from a promisor remote. */ + if (odb->repo->fetch_if_missing && repo_has_promisor_remote(odb->repo) && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + promisor_remote_get_direct(odb->repo, real, 1); - /* Check if it is a missing object */ - if (odb->repo->fetch_if_missing && repo_has_promisor_remote(odb->repo) && - !already_retried && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { - promisor_remote_get_direct(odb->repo, real, 1); - already_retried = 1; - continue; - } + ret = odb_source_read_object_info(odb->source, real, oi, flags, + corrupt_err.len ? NULL : &corrupt_err); + if (!ret) + goto out; + if (ret != ODB_READ_NOT_FOUND) + corrupt = true; + } - if (flags & OBJECT_INFO_DIE_IF_CORRUPT) { - if ((flags & OBJECT_INFO_LOOKUP_REPLACE) && !oideq(real, oid)) - die(_("replacement %s not found for %s"), - oid_to_hex(real), oid_to_hex(oid)); - if (corrupt) { - if (corrupt_err.len) - die("%s", corrupt_err.buf); - die(_("object %s is corrupt"), oid_to_hex(real)); - } + if (flags & OBJECT_INFO_DIE_IF_CORRUPT) { + if ((flags & OBJECT_INFO_LOOKUP_REPLACE) && !oideq(real, oid)) + die(_("replacement %s not found for %s"), + oid_to_hex(real), oid_to_hex(oid)); + if (corrupt) { + if (corrupt_err.len) + die("%s", corrupt_err.buf); + die(_("object %s is corrupt"), oid_to_hex(real)); } - - ret = corrupt ? ODB_READ_ERROR : ODB_READ_NOT_FOUND; - goto out; } + ret = corrupt ? ODB_READ_ERROR : ODB_READ_NOT_FOUND; + out: strbuf_release(&corrupt_err); return ret; From 2cc9348017434d63813d8cff56b076219a04794e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 15 Oct 2026 07:27:59 +0200 Subject: [PATCH 28/29] odb: don't try re-fetching corrupt objects When we failed reading an object in a repository with a promisor remote we try to fetch it from the promisor and then re-read it. This logic kicks in not only when the object couldn't be found, but also in the case where it could be found but the object is corrupt. It feels dubious though to just paper over corruption and proceed as if nothing has happened, happily fetching data from the promisor. Adapt the code so that we only fetch objects in case they are missing. Suggested-by: Justin Tobler Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/odb.c b/odb.c index 427da284c3b2e1..68c1cc1bab609b 100644 --- a/odb.c +++ b/odb.c @@ -396,7 +396,6 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * struct strbuf corrupt_err = STRBUF_INIT; const struct object_id *real = oid; enum odb_read_status ret; - bool corrupt = false; int tried_hook = 0; int tried_gvfs_helper = 0; @@ -414,8 +413,6 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * corrupt_err.len ? NULL : &corrupt_err); if (!ret) goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; if (core_use_gvfs_helper && !tried_gvfs_helper && !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { @@ -451,32 +448,27 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * goto retry; } - /* Try to fetch the missing object from a promisor remote. */ - if (odb->repo->fetch_if_missing && repo_has_promisor_remote(odb->repo) && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + if (ret == ODB_READ_NOT_FOUND && + odb->repo->fetch_if_missing && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT) && + repo_has_promisor_remote(odb->repo)) { + /* Try to fetch the missing object from a promisor remote. */ promisor_remote_get_direct(odb->repo, real, 1); - ret = odb_source_read_object_info(odb->source, real, oi, flags, corrupt_err.len ? NULL : &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; } - if (flags & OBJECT_INFO_DIE_IF_CORRUPT) { + if (ret < 0 && (flags & OBJECT_INFO_DIE_IF_CORRUPT)) { if ((flags & OBJECT_INFO_LOOKUP_REPLACE) && !oideq(real, oid)) die(_("replacement %s not found for %s"), oid_to_hex(real), oid_to_hex(oid)); - if (corrupt) { + if (ret != ODB_READ_NOT_FOUND) { if (corrupt_err.len) die("%s", corrupt_err.buf); die(_("object %s is corrupt"), oid_to_hex(real)); } } - ret = corrupt ? ODB_READ_ERROR : ODB_READ_NOT_FOUND; - out: strbuf_release(&corrupt_err); return ret; From f7fcf9846f6033f65584314aa3d4bcc63ebdcc15 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 15 Oct 2026 07:28:00 +0200 Subject: [PATCH 29/29] odb/source: drop `local` flag The `local` flag indicates whether a given `struct odb_source` is a local source or not, where "local" is defined as being the object directory that is directly attached to the current repository. But now that alternates are handled in the "files" backend, this property is not something that even makes sense anymore to have on the generic source. In fact, we already have a partial replacement for this field in `struct odb_files_dir`. The only other site that requires it is the packed object database source, because it propagates the bit into the packfiles it is opening. Drop the flag from `struct odb_source` and instead move it into the packed source. Note that we stop considering the `local` bit when iterating through all loose objects with the `ODB_FOR_EACH_OBJECT_LOCAL_ONLY` flag. This is fine though, as we already handle the flag in the "files" backend itself, where we skip over any object directory that is not marked as local. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- midx.c | 2 +- odb/source-files.c | 4 ++-- odb/source-inmemory.c | 10 +++++++--- odb/source-loose.c | 7 ++----- odb/source-loose.h | 3 +-- odb/source-packed.c | 5 +++-- odb/source-packed.h | 7 +++++++ odb/source.c | 4 +--- odb/source.h | 13 +------------ repack.c | 2 +- 10 files changed, 26 insertions(+), 31 deletions(-) diff --git a/midx.c b/midx.c index 8638ddf0be7f00..fbe0a775053648 100644 --- a/midx.c +++ b/midx.c @@ -470,7 +470,7 @@ int prepare_midx_pack(struct multi_pack_index *m, strbuf_addf(&pack_name, "%s/pack/%s", packed->base.path, m->pack_names[pack_int_id]); p = packfile_store_load_pack(packed, - pack_name.buf, packed->base.local); + pack_name.buf, packed->local); strbuf_release(&pack_name); if (!p) { diff --git a/odb/source-files.c b/odb/source-files.c index f5497c41abe890..211500da4aa9aa 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -33,7 +33,7 @@ struct odb_files_dir *odb_files_dir_new(struct object_database *odb, CALLOC_ARRAY(dir, 1); dir->abspath = absolute_pathdup(path); dir->local = local; - dir->loose = odb_source_loose_new(odb, path, local); + dir->loose = odb_source_loose_new(odb, path); dir->packed = odb_source_packed_new(odb, path, local); return dir; @@ -1322,7 +1322,7 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, object_dir = xstrfmt("%s/objects", odb->repo->commondir); CALLOC_ARRAY(files, 1); - odb_source_init(&files->base, odb, ODB_SOURCE_FILES, object_dir, true); + odb_source_init(&files->base, odb, ODB_SOURCE_FILES, object_dir); hashmap_init(&files->dirs_by_path, odb_files_dir_by_path_cmp, files, 0); files->dirs_paths_icase = -1; diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 3f63f216919f83..b23da3f6788f12 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -161,8 +161,12 @@ static int odb_source_inmemory_for_each_object(struct odb_source *source, }; struct object_id null_oid = { 0 }; - if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY) || - (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local)) + /* + * In-memory objects are neither promisor objects nor do they live in + * the local object directory, so there is nothing to iterate over. + */ + if (opts->flags & (ODB_FOR_EACH_OBJECT_PROMISOR_ONLY | + ODB_FOR_EACH_OBJECT_LOCAL_ONLY)) return 0; if (!inmemory->objects) return 0; @@ -370,7 +374,7 @@ struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb) struct odb_source_inmemory *source; CALLOC_ARRAY(source, 1); - odb_source_init(&source->base, odb, ODB_SOURCE_INMEMORY, "source", false); + odb_source_init(&source->base, odb, ODB_SOURCE_INMEMORY, "source"); source->base.free = odb_source_inmemory_free; source->base.close = odb_source_inmemory_close; diff --git a/odb/source-loose.c b/odb/source-loose.c index befdd6b00bed65..a7ddd3c2d531f2 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -465,8 +465,6 @@ static int odb_source_loose_for_each_object(struct odb_source *source, /* There are no loose promisor objects, so we can return immediately. */ if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) return 0; - if ((opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY) && !source->local) - return 0; if (opts->prefix) return oidtree_each(odb_source_loose_cache(loose, opts->prefix), @@ -1140,13 +1138,12 @@ static int odb_source_loose_fsck(struct odb_source *source, } struct odb_source_loose *odb_source_loose_new(struct object_database *odb, - const char *path, - bool local) + const char *path) { struct odb_source_loose *loose; CALLOC_ARRAY(loose, 1); - odb_source_init(&loose->base, odb, ODB_SOURCE_LOOSE, path, local); + odb_source_init(&loose->base, odb, ODB_SOURCE_LOOSE, path); loose->base.free = odb_source_loose_free; loose->base.close = odb_source_loose_close; diff --git a/odb/source-loose.h b/odb/source-loose.h index 7c7e845fe348f4..a6ae808662b39a 100644 --- a/odb/source-loose.h +++ b/odb/source-loose.h @@ -34,8 +34,7 @@ struct odb_source_loose { }; struct odb_source_loose *odb_source_loose_new(struct object_database *odb, - const char *path, - bool local); + const char *path); /* * Cast the given object database source to the loose backend. This will cause diff --git a/odb/source-packed.c b/odb/source-packed.c index 6cf5be12c01fc4..1d75bac41846e2 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -741,7 +741,7 @@ static void prepare_pack(const char *full_name, size_t full_name_len, midx_contains_pack(data->source->midx, file_name))) { char *trimmed_path = xstrndup(full_name, full_name_len); packfile_store_load_pack(data->source, - trimmed_path, data->source->base.local); + trimmed_path, data->source->local); free(trimmed_path); } @@ -989,8 +989,9 @@ struct odb_source_packed *odb_source_packed_new(struct object_database *odb, struct odb_source_packed *packed; CALLOC_ARRAY(packed, 1); - odb_source_init(&packed->base, odb, ODB_SOURCE_PACKED, path, local); + odb_source_init(&packed->base, odb, ODB_SOURCE_PACKED, path); strmap_init(&packed->packs_by_path); + packed->local = local; packed->base.free = odb_source_packed_free; packed->base.close = odb_source_packed_close; diff --git a/odb/source-packed.h b/odb/source-packed.h index a0f6b5096dcd0f..e8a7b08fbb9b13 100644 --- a/odb/source-packed.h +++ b/odb/source-packed.h @@ -41,6 +41,13 @@ struct odb_source_packed { */ struct strmap packs_by_path; + /* + * Whether this store is part of the local object directory of the + * owning repository. Packfiles loaded via this store inherit this + * flag via `pack_local`. Stores for alternates are not local. + */ + bool local; + /* * Whether packfiles have already been populated with this store's * packs. diff --git a/odb/source.c b/odb/source.c index b25ef14df83ca8..4155fc22dac1be 100644 --- a/odb/source.c +++ b/odb/source.c @@ -32,12 +32,10 @@ struct odb_source *odb_source_new(struct object_database *odb, void odb_source_init(struct odb_source *source, struct object_database *odb, enum odb_source_type type, - const char *path, - bool local) + const char *path) { source->odb = odb; source->type = type; - source->local = local; source->path = xstrdup(path); } diff --git a/odb/source.h b/odb/source.h index c852dd9a25b0a8..d2bdcbc1598d2f 100644 --- a/odb/source.h +++ b/odb/source.h @@ -58,14 +58,6 @@ struct odb_source { /* The type used by this source. */ enum odb_source_type type; - /* - * Figure out whether this is the local source of the owning - * repository, which would typically be its ".git/objects" directory. - * This local object directory is usually where objects would be - * written to. - */ - bool local; - /* * Path to the source. If this is a relative path, it is relative to * the current working directory. @@ -305,8 +297,6 @@ struct odb_source *odb_source_new(struct object_database *odb, /* * Initialize the source for the given object database located at `path`. - * `local` indicates whether or not the source is the local and thus primary - * object source of the object database. * * This function is only supposed to be called by specific object source * implementations. @@ -314,8 +304,7 @@ struct odb_source *odb_source_new(struct object_database *odb, void odb_source_init(struct odb_source *source, struct object_database *odb, enum odb_source_type type, - const char *path, - bool local); + const char *path); /* * Free the object database source, releasing all associated resources and diff --git a/repack.c b/repack.c index 53c34197f3a1c7..4c6d1f507d7135 100644 --- a/repack.c +++ b/repack.c @@ -62,7 +62,7 @@ void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); struct multi_pack_index *m = get_multi_pack_index(files->dirs->packed); strbuf_addf(&buf, "%s.pack", base_name); - if (m && files->base.local && midx_contains_pack(m, buf.buf)) { + if (m && files->dirs->local && midx_contains_pack(m, buf.buf)) { clear_midx_file(repo); if (!wrote_incremental_midx) clear_incremental_midx_files(repo, NULL);