diff --git a/builtin/clone.c b/builtin/clone.c index 690da1a09bdb66..938d7cd51d017a 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -154,24 +154,26 @@ static char *get_repo_path(const char *repo, int *is_bundle) return canon; } -static int add_one_reference(struct string_list_item *item, void *cb_data) +struct collect_alternates_data { + struct strvec *alternates; + bool required; +}; + +static int collect_one_alternate(struct string_list_item *item, void *cb_data) { + struct collect_alternates_data *data = cb_data; struct strbuf err = STRBUF_INIT; - int *required = cb_data; char *ref_git = compute_alternate_path(item->string, &err); if (!ref_git) { - if (*required) + if (data->required) die("%s", err.buf); else fprintf(stderr, _("info: Could not add alternate for '%s': %s\n"), item->string, err.buf); } else { - struct strbuf sb = STRBUF_INIT; - strbuf_addf(&sb, "%s/objects", ref_git); - odb_add_to_alternates_file(the_repository->objects, sb.buf); - strbuf_release(&sb); + strvec_pushf(data->alternates, "%s/objects", ref_git); } strbuf_release(&err); @@ -179,17 +181,7 @@ static int add_one_reference(struct string_list_item *item, void *cb_data) return 0; } -static void setup_reference(void) -{ - int required = 1; - for_each_string_list(&option_required_reference, - add_one_reference, &required); - required = 0; - for_each_string_list(&option_optional_reference, - add_one_reference, &required); -} - -static void copy_alternates(struct strbuf *src, const char *src_repo) +static void read_alternates(struct strvec *alternates, const char *src_repo) { /* * Read from the source objects/info/alternates file @@ -203,29 +195,70 @@ static void copy_alternates(struct strbuf *src, const char *src_repo) * to turn entries with paths relative to the original * absolute, so that they can be used in the new repository. */ - FILE *in = xfopen(src->buf, "r"); + FILE *in; + struct strbuf path = STRBUF_INIT; struct strbuf line = STRBUF_INIT; + strbuf_addf(&path, "%s/objects/info/alternates", src_repo); + + in = fopen(path.buf, "r"); + if (!in) { + if (errno == ENOENT) + goto out; + die_errno("could not read alternates file '%s'", path.buf); + } + while (strbuf_getline(&line, in) != EOF) { char *abs_path; if (!line.len || line.buf[0] == '#') continue; if (is_absolute_path(line.buf)) { - odb_add_to_alternates_file(the_repository->objects, - line.buf); + strvec_push(alternates, line.buf); continue; } abs_path = mkpathdup("%s/objects/%s", src_repo, line.buf); if (!normalize_path_copy(abs_path, abs_path)) - odb_add_to_alternates_file(the_repository->objects, - abs_path); + strvec_push(alternates, abs_path); else warning("skipping invalid relative alternate: %s/%s", src_repo, line.buf); free(abs_path); } + +out: + strbuf_release(&path); strbuf_release(&line); - fclose(in); + if (in) + fclose(in); +} + +static void collect_alternates(struct strvec *alternates, + const char *local_source_repo) +{ + if (option_required_reference.nr || option_optional_reference.nr) { + struct collect_alternates_data data = { + .alternates = alternates, + .required = true, + }; + + for_each_string_list(&option_required_reference, + collect_one_alternate, &data); + data.required = false; + for_each_string_list(&option_optional_reference, + collect_one_alternate, &data); + } + + if (local_source_repo) { + struct strbuf commondir = STRBUF_INIT; + + get_common_dir(&commondir, local_source_repo); + if (option_shared) + strvec_pushf(alternates, "%s/objects", commondir.buf); + else + read_alternates(alternates, commondir.buf); + + strbuf_release(&commondir); + } } static void mkdir_if_missing(const char *pathname, mode_t mode) @@ -301,11 +334,9 @@ static void copy_or_link_directory(struct strbuf *src, struct strbuf *dest, continue; } - /* Files that cannot be copied bit-for-bit... */ - if (!fspathcmp(iter->relative_path, "info/alternates")) { - copy_alternates(src, src_repo); + /* Alternates were already handled earlier. */ + if (!fspathcmp(iter->relative_path, "info/alternates")) continue; - } if (unlink(dest->buf) && errno != ENOENT) die_errno(_("failed to unlink '%s'"), dest->buf); @@ -349,13 +380,7 @@ static void copy_or_link_directory(struct strbuf *src, struct strbuf *dest, static void clone_local(const char *src_repo, const char *dest_repo) { - if (option_shared) { - struct strbuf alt = STRBUF_INIT; - get_common_dir(&alt, src_repo); - strbuf_addstr(&alt, "/objects"); - odb_add_to_alternates_file(the_repository->objects, alt.buf); - strbuf_release(&alt); - } else { + if (!option_shared) { struct strbuf src = STRBUF_INIT; struct strbuf dest = STRBUF_INIT; get_common_dir(&src, src_repo); @@ -999,6 +1024,7 @@ int cmd_clone(int argc, N_("uri"), N_("a URI for downloading bundles before fetching from origin remote")), OPT_END() }; + struct strvec alternates = STRVEC_INIT; const char * const builtin_clone_usage[] = { N_("git clone [] [--] []"), @@ -1185,10 +1211,9 @@ int cmd_clone(int argc, * repository, and reference backends may persist that information into * their on-disk data structures. */ - init_db(the_repository, git_dir, real_git_dir, work_tree, option_template, - GIT_HASH_UNKNOWN, ref_storage_format, NULL, - do_not_override_repo_unix_permissions, - INIT_DB_QUIET | INIT_DB_SKIP_REFDB); + create_repository(the_repository, git_dir, real_git_dir, work_tree, + option_template, GIT_HASH_UNKNOWN, ref_storage_format, + do_not_override_repo_unix_permissions, NULL); if (real_git_dir) { free((char *)git_dir); @@ -1311,9 +1336,6 @@ int cmd_clone(int argc, strbuf_reset(&key); } - if (option_required_reference.nr || option_optional_reference.nr) - setup_reference(); - remote = remote_get_early(remote_name); if (!option_rev) @@ -1342,6 +1364,9 @@ int cmd_clone(int argc, if (option_local > 0 && !is_local) warning(_("--local is ignored")); + collect_alternates(&alternates, is_local ? path : NULL); + create_object_database(the_repository, &alternates); + transport = transport_get(remote, path ? path : remote->url.v[0]); transport_set_verbosity(transport, option_verbosity, option_progress); transport->family = family; @@ -1445,6 +1470,7 @@ int cmd_clone(int argc, initialize_repository_version(the_repository, hash_algo, the_repository->ref_storage_format, 1); repo_set_hash_algo(the_repository, hash_algo); create_reference_database(the_repository, NULL, 1); + startup_info->have_repository = 1; /* * Before fetching from the remote, download and install bundle @@ -1637,6 +1663,7 @@ int cmd_clone(int argc, string_list_clear(&option_not, 0); string_list_clear(&option_config, 0); string_list_clear(&server_options, 0); + strvec_clear(&alternates); free(remote_name); strbuf_release(&reflog_msg); diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index d62005edc0469c..ff244c866fea4f 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -8,6 +8,8 @@ #include "parse-options.h" #include "commit-graph.h" #include "odb.h" +#include "odb/source.h" +#include "odb/source-files.h" #include "progress.h" #include "replace-object.h" #include "strbuf.h" @@ -67,7 +69,7 @@ static int graph_verify(int argc, const char **argv, const char *prefix, struct repository *repo UNUSED) { struct commit_graph *graph = NULL; - struct odb_source *source = NULL; + struct odb_files_dir *dir = NULL; char *graph_name; char *chain_name; enum { OPENED_NONE, OPENED_GRAPH, OPENED_CHAIN } opened = OPENED_NONE; @@ -102,9 +104,12 @@ static int graph_verify(int argc, const char **argv, const char *prefix, if (opts.progress) flags |= COMMIT_GRAPH_WRITE_PROGRESS; - source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); - graph_name = get_commit_graph_filename(source); - chain_name = get_commit_graph_chain_filename(source); + dir = odb_source_files_find_dir(the_repository->objects, opts.obj_dir); + if (!dir) + die(_("could not find object directory matching %s"), opts.obj_dir); + + graph_name = get_commit_graph_filename(dir->abspath); + chain_name = get_commit_graph_chain_filename(dir->abspath); if (open_commit_graph(graph_name, &fd, &st)) opened = OPENED_GRAPH; else if (errno != ENOENT) @@ -122,7 +127,7 @@ static int graph_verify(int argc, const char **argv, const char *prefix, if (opened == OPENED_NONE) return 0; else if (opened == OPENED_GRAPH) - graph = load_commit_graph_one_fd_st(source, fd, &st); + graph = load_commit_graph_one_fd_st(the_repository, dir->abspath, fd, &st); else graph = load_commit_graph_chain_fd_st(the_repository->objects, fd, &st, &incomplete_chain); @@ -225,7 +230,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, struct string_list pack_indexes = STRING_LIST_INIT_DUP; struct strbuf buf = STRBUF_INIT; struct oidset commits = OIDSET_INIT; - struct odb_source *source = NULL; + struct odb_files_dir *dir = NULL; int result = 0; enum commit_graph_write_flags flags = 0; struct progress *progress = NULL; @@ -293,18 +298,30 @@ static int graph_write(int argc, const char **argv, const char *prefix, git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags |= COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); + dir = odb_source_files_find_dir(the_repository->objects, opts.obj_dir); + if (!dir) + die(_("could not find object directory matching %s"), opts.obj_dir); if (opts.reachable) { - if (write_commit_graph_reachable(source, flags, &write_opts)) + if (write_commit_graph_reachable(the_repository, dir->abspath, flags, &write_opts)) result = 1; goto cleanup; } if (opts.stdin_packs) { - while (strbuf_getline(&buf, stdin) != EOF) - string_list_append_nodup(&pack_indexes, - strbuf_detach(&buf, NULL)); + struct strbuf packname = STRBUF_INIT; + size_t dirlen; + + strbuf_addf(&packname, "%s/pack/", dir->abspath); + dirlen = packname.len; + + while (strbuf_getline(&buf, stdin) != EOF) { + strbuf_setlen(&packname, dirlen); + strbuf_addbuf(&packname, &buf); + string_list_append(&pack_indexes, packname.buf); + } + + strbuf_release(&packname); } else if (opts.stdin_commits) { oidset_init(&commits, 0); if (opts.progress) @@ -323,7 +340,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, stop_progress(&progress); } - if (write_commit_graph(source, + if (write_commit_graph(the_repository, dir->abspath, opts.stdin_packs ? &pack_indexes : NULL, opts.stdin_commits ? &commits : NULL, flags, diff --git a/builtin/commit.c b/builtin/commit.c index 577755d54ed4fb..3c13d24cb1bff5 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -2218,7 +2218,7 @@ int cmd_commit(int argc, "new index file. Check that disk is not full and quota is\n" "not exceeded, and then \"git restore --staged :/\" to recover.")); - git_test_write_commit_graph_or_die(the_repository->objects->sources); + git_test_write_commit_graph_or_die(the_repository); repo_rerere(the_repository, 0); run_auto_maintenance(the_repository, quiet); diff --git a/builtin/count-objects.c b/builtin/count-objects.c index 18f6e33b6f913f..3f7b20fde531a3 100644 --- a/builtin/count-objects.c +++ b/builtin/count-objects.c @@ -81,10 +81,10 @@ static int count_cruft(const char *basename UNUSED, const char *path, return 0; } -static int print_alternate(struct odb_source *alternate, void *data UNUSED) +static int print_alternate(struct odb_files_dir *alternate, void *data UNUSED) { printf("alternate: "); - quote_c_style(alternate->path, NULL, stdout, 0); + quote_c_style(alternate->abspath, NULL, stdout, 0); putchar('\n'); return 0; } @@ -118,7 +118,7 @@ int cmd_count_objects(int argc, report_linked_checkout_garbage(the_repository); } - for_each_loose_file_in_source(the_repository->objects->sources, + for_each_loose_file_in_source(the_repository->objects->source, count_loose, count_cruft, NULL, NULL); if (verbose) { diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 921f9a6a6c2865..b93122d2eb3c27 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -895,7 +895,7 @@ static void end_packfile(void) running = 1; clear_delta_base_cache(); if (object_count) { - struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->source); struct packed_git *new_p; struct object_id cur_pack_oid; char *idx_name; @@ -921,7 +921,7 @@ static void end_packfile(void) idx_name = keep_pack(create_index()); /* Register the packfile with core git's machinery. */ - new_p = packfile_store_load_pack(files->packed, idx_name, 1); + new_p = packfile_store_load_pack(files->dirs->packed, idx_name, 1); if (!new_p) die(_("core Git rejected index %s"), idx_name); all_packs[pack_id] = new_p; @@ -975,7 +975,7 @@ static int store_object( struct object_id *oidout, uintmax_t mark) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); void *out, *delta; struct object_entry *e; unsigned char hdr[96]; @@ -1002,10 +1002,8 @@ static int store_object( return 1; } - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!packfile_list_find_oid(packfile_store_get_packs(dir->packed), &oid)) continue; e->type = type; e->pack_id = MAX_PACK_ID; @@ -1122,10 +1120,10 @@ static void truncate_pack(struct hashfile_checkpoint *checkpoint) static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) { + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); size_t in_sz = 64 * 1024, out_sz = 64 * 1024; unsigned char *in_buf = xmalloc(in_sz); unsigned char *out_buf = xmalloc(out_sz); - struct odb_source *source; struct object_entry *e; struct object_id oid; unsigned long hdrlen; @@ -1209,10 +1207,8 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) goto out; } - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!packfile_list_find_oid(packfile_store_get_packs(dir->packed), &oid)) continue; e->type = OBJ_BLOB; e->pack_id = MAX_PACK_ID; diff --git a/builtin/fetch.c b/builtin/fetch.c index 46151a8edb1e25..1d35ba2955913b 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -15,6 +15,7 @@ #include "refspec.h" #include "object-name.h" #include "odb.h" +#include "odb/source.h" #include "oidset.h" #include "oid-array.h" #include "commit.h" @@ -2919,7 +2920,8 @@ int cmd_fetch(int argc, commit_graph_flags |= COMMIT_GRAPH_WRITE_PROGRESS; trace2_region_enter("fetch", "write-commit-graph", the_repository); - write_commit_graph_reachable(the_repository->objects->sources, + write_commit_graph_reachable(the_repository, + the_repository->objects->source->path, commit_graph_flags, NULL); trace2_region_leave("fetch", "write-commit-graph", the_repository); diff --git a/builtin/fsck.c b/builtin/fsck.c index 4a2da5f34a20d8..8cbbaaa486dcf8 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -868,7 +868,6 @@ int cmd_fsck(int argc, OPT_BOOL(0, "references", &check_references, N_("check reference database consistency")), OPT_END(), }; - struct odb_source *source; struct snapshot snap = { .nr = 0, .alloc = 0, @@ -983,13 +982,14 @@ int cmd_fsck(int argc, check_connectivity(repo); if (repo->settings.core_commit_graph) { + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); struct child_process commit_graph_verify = CHILD_PROCESS_INIT; - for (source = repo->objects->sources; source; source = source->next) { + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { child_process_init(&commit_graph_verify); commit_graph_verify.git_cmd = 1; strvec_pushl(&commit_graph_verify.args, "commit-graph", - "verify", "--object-dir", source->path, NULL); + "verify", "--object-dir", dir->abspath, NULL); if (show_progress) strvec_push(&commit_graph_verify.args, "--progress"); else diff --git a/builtin/gc.c b/builtin/gc.c index 99d6424eff54ef..117055fd239772 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -745,7 +745,8 @@ int cmd_gc(int argc, } if (the_repository->settings.gc_write_commit_graph == 1) - write_commit_graph_reachable(the_repository->objects->sources, + write_commit_graph_reachable(the_repository, + the_repository->objects->source->path, !opts.quiet && !daemonized ? COMMIT_GRAPH_WRITE_PROGRESS : 0, NULL); @@ -1009,7 +1010,7 @@ static int loose_object_auto_condition(struct gc_config *cfg UNUSED) if (loose_object_auto_limit < 0) return 1; - return for_each_loose_file_in_source(the_repository->objects->sources, + return for_each_loose_file_in_source(the_repository->objects->source, loose_object_count, NULL, NULL, &count); } @@ -1041,28 +1042,24 @@ static int pack_loose(struct maintenance_run_opts *opts) int result = 0; struct write_loose_object_data data; struct child_process pack_proc = CHILD_PROCESS_INIT; - struct odb_source *prev_source = NULL; - const char *object_dir = r->objects->sources->path; + struct odb_source *source = r->objects->source; + struct odb_files_dir *shared = NULL; /* If set, use the shared object directory. */ if (shared_object_dir) { - odb_set_temporary_primary_source(r->objects, shared_object_dir, - 0, &prev_source); - object_dir = shared_object_dir; + shared = odb_files_dir_new(r->objects, shared_object_dir, + false); + source = &shared->loose->base; } /* * Do not start pack-objects process * if there are no loose objects. */ - if (!for_each_loose_file_in_source(r->objects->sources, + if (!for_each_loose_file_in_source(source, bail_on_loose, - NULL, NULL, NULL)) { - if (shared_object_dir) - odb_restore_primary_source(r->objects, prev_source, - shared_object_dir); - return 0; - } + NULL, NULL, NULL)) + goto out; pack_proc.git_cmd = 1; @@ -1071,7 +1068,7 @@ static int pack_loose(struct maintenance_run_opts *opts) strvec_push(&pack_proc.args, "--quiet"); else strvec_push(&pack_proc.args, "--no-quiet"); - strvec_pushf(&pack_proc.args, "%s/pack/loose", object_dir); + strvec_pushf(&pack_proc.args, "%s/pack/loose", source->path); pack_proc.in = -1; @@ -1083,10 +1080,8 @@ static int pack_loose(struct maintenance_run_opts *opts) if (start_command(&pack_proc)) { error(_("failed to start 'git pack-objects' process")); - if (shared_object_dir) - odb_restore_primary_source(r->objects, prev_source, - shared_object_dir); - return 1; + result = 1; + goto out; } data.in = xfdopen(pack_proc.in, "w"); @@ -1102,7 +1097,7 @@ static int pack_loose(struct maintenance_run_opts *opts) else if (data.batch_size > 0) data.batch_size--; /* Decrease for equality on limit. */ - for_each_loose_file_in_source(r->objects->sources, + for_each_loose_file_in_source(source, write_loose_object_to_stdin, NULL, NULL, &data); @@ -1113,10 +1108,8 @@ static int pack_loose(struct maintenance_run_opts *opts) result = 1; } - if (shared_object_dir) - odb_restore_primary_source(r->objects, prev_source, - shared_object_dir); - +out: + odb_files_dir_free(shared); return result; } @@ -1459,15 +1452,15 @@ static int maintenance_task_cache_local_objs(UNUSED struct maintenance_run_opts return 0; /* If the dest is the same as the local odb path then we do nothing. */ - if (!fspathcmp(r->objects->sources->path, shared_object_dir)) + if (!fspathcmp(r->objects->source->path, shared_object_dir)) goto cleanup; strbuf_addf(&dstdir, "%s/pack", shared_object_dir); - for_each_file_in_pack_dir(r->objects->sources->path, move_pack_to_shared_cache, + for_each_file_in_pack_dir(r->objects->source->path, move_pack_to_shared_cache, dstdir.buf); - ret = for_each_loose_file_in_source(r->objects->sources, + ret = for_each_loose_file_in_source(r->objects->source, move_loose_object_to_shared_cache, NULL, NULL, NULL); @@ -1615,7 +1608,7 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts, int result = 0; struct lock_file lk; struct repository *r = the_repository; - char *lock_path = xstrfmt("%s/maintenance", r->objects->sources->path); + char *lock_path = xstrfmt("%s/maintenance", r->objects->source->path); enum auto_gc_hook_result auto_gc_hook_result = AUTO_GC_HOOK_UNDECIDED; if (repo_hold_lock_file_for_update(r, &lk, lock_path, LOCK_NO_DEREF) < 0) { @@ -3207,7 +3200,7 @@ static int update_background_schedule(const struct maintenance_start_opts *opts, unsigned int i; int result = 0; struct lock_file lk; - char *lock_path = xstrfmt("%s/schedule", the_repository->objects->sources->path); + char *lock_path = xstrfmt("%s/schedule", the_repository->objects->source->path); if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) { if (errno == EEXIST) diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 5028a92f069877..f674c8863d2ef1 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1640,8 +1640,8 @@ static void final(const char *final_pack_name, const char *curr_pack_name, if (do_fsck_object && startup_info->have_repository) { struct odb_source_files *files = - odb_source_files_downcast(the_repository->objects->sources); - packfile_store_load_pack(files->packed, final_index_name, 0); + odb_source_files_downcast(the_repository->objects->source); + packfile_store_load_pack(files->dirs->packed, final_index_name, 0); } if (!from_stdin) { diff --git a/builtin/init-db.c b/builtin/init-db.c index e96b1283b7010d..5c22eae2f3f2d1 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -80,7 +80,7 @@ int cmd_init_db(int argc, char *work_tree = NULL; const char *template_dir = NULL; char *template_dir_to_free = NULL; - unsigned int flags = 0; + int quiet = 0; int bare = startup_info->force_bare_repository ? 1 : -1; const char *object_format = NULL; const char *ref_format = NULL; @@ -102,7 +102,7 @@ int cmd_init_db(int argc, .flags = PARSE_OPT_OPTARG | PARSE_OPT_NONEG, .callback = shared_callback }, - OPT_BIT('q', "quiet", &flags, N_("be quiet"), INIT_DB_QUIET), + OPT_BOOL('q', "quiet", &quiet, N_("be quiet")), OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"), N_("separate git dir from working tree")), OPT_STRING('b', "initial-branch", &initial_branch, N_("name"), @@ -113,7 +113,7 @@ int cmd_init_db(int argc, N_("specify the reference format to use")), OPT_END() }; - int ret; + int reinit; argc = parse_options(argc, argv, prefix, init_db_options, init_db_usage, 0); @@ -247,14 +247,30 @@ int cmd_init_db(int argc, die(_("--separate-git-dir incompatible with bare repository")); } - flags |= INIT_DB_EXIST_OK; - ret = init_db(the_repository, git_dir, real_git_dir, work_tree, - template_dir, hash_algo, ref_storage_format, initial_branch, - init_shared_repository, flags); + create_repository(the_repository, git_dir, real_git_dir, work_tree, + template_dir, hash_algo, ref_storage_format, + init_shared_repository, &reinit); + create_reference_database(the_repository, initial_branch, quiet); + create_object_database(the_repository, NULL); + + if (!quiet) { + int len = strlen(git_dir); + + if (reinit) + printf(repo_settings_get_shared_repository(the_repository) + ? _("Reinitialized existing shared Git repository in %s%s\n") + : _("Reinitialized existing Git repository in %s%s\n"), + git_dir, len && git_dir[len-1] != '/' ? "/" : ""); + else + printf(repo_settings_get_shared_repository(the_repository) + ? _("Initialized empty shared Git repository in %s%s\n") + : _("Initialized empty Git repository in %s%s\n"), + git_dir, len && git_dir[len-1] != '/' ? "/" : ""); + } free(template_dir_to_free); free(real_git_dir_to_free); free(work_tree); free(git_dir); - return ret; + return 0; } diff --git a/builtin/merge.c b/builtin/merge.c index 02a87430f48f97..839f5ef31d223c 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -1864,7 +1864,7 @@ int cmd_merge(int argc, if (squash) { finish(head_commit, remoteheads, NULL, NULL); - git_test_write_commit_graph_or_die(the_repository->objects->sources); + git_test_write_commit_graph_or_die(the_repository); } else write_merge_state(remoteheads); diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 753bd53a70fccc..c48212290cf81b 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -80,19 +80,19 @@ static int parse_object_dir(const struct option *opt, const char *arg, char **value = opt->value; free(*value); if (unset) - *value = xstrdup(the_repository->objects->sources->path); + *value = xstrdup(the_repository->objects->source->path); else *value = real_pathdup(arg, 1); return 0; } -static struct odb_source_files *handle_object_dir_option(struct repository *repo) +static struct odb_source_packed *handle_object_dir_option(struct repository *repo) { - struct odb_source *source = odb_find_source(repo->objects, opts.object_dir); - if (!source) + struct odb_files_dir *dir = odb_source_files_find_dir(repo->objects, opts.object_dir); + if (!dir) die(_("object directory is not an alternate of the current repository: '%s'"), opts.object_dir); - return odb_source_files_downcast(source); + return dir->packed; } static struct option common_opts[] = { @@ -169,7 +169,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, N_("refs snapshot for selecting bitmap commits")), OPT_END(), }; - struct odb_source_files *source; + struct odb_source_packed *packed_source; int ret; opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE; @@ -204,7 +204,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, options); } - source = handle_object_dir_option(repo); + packed_source = handle_object_dir_option(repo); FREE_AND_NULL(options); @@ -213,7 +213,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, read_packs_from_stdin(&packs); - ret = write_midx_file_only(source->packed, &packs, + ret = write_midx_file_only(packed_source, &packs, opts.preferred_pack, opts.refs_snapshot, opts.incremental_base, opts.flags); @@ -225,7 +225,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } - ret = write_midx_file(source->packed, opts.preferred_pack, + ret = write_midx_file(packed_source, opts.preferred_pack, opts.refs_snapshot, opts.flags); free(opts.refs_snapshot); @@ -239,7 +239,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, struct multi_pack_index *m, *cur; struct multi_pack_index *from_midx = NULL; struct multi_pack_index *to_midx = NULL; - struct odb_source_files *source; + struct odb_source_packed *packed_source; int ret; struct option *options; @@ -280,11 +280,11 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, options); } - source = handle_object_dir_option(the_repository); + packed_source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - m = get_multi_pack_index(source->packed); + m = get_multi_pack_index(packed_source); for (cur = m; cur && !(from_midx && to_midx); cur = cur->base_midx) { const char *midx_csum = midx_get_checksum_hex(cur); @@ -307,7 +307,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, die(_("MIDX %s must be an ancestor of %s"), argv[0], argv[1]); } - ret = write_midx_file_compact(source->packed, from_midx, to_midx, + ret = write_midx_file_compact(packed_source, from_midx, to_midx, opts.incremental_base, opts.flags); return ret; @@ -321,7 +321,7 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, static struct option builtin_multi_pack_index_verify_options[] = { OPT_END(), }; - struct odb_source_files *source; + struct odb_source_packed *packed_source; options = add_common_options(builtin_multi_pack_index_verify_options); @@ -335,11 +335,11 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_verify_usage, options); - source = handle_object_dir_option(the_repository); + packed_source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return verify_midx_file(source->packed, opts.flags); + return verify_midx_file(packed_source, opts.flags); } static int cmd_multi_pack_index_expire(int argc, const char **argv, @@ -350,7 +350,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, static struct option builtin_multi_pack_index_expire_options[] = { OPT_END(), }; - struct odb_source_files *source; + struct odb_source_packed *packed_source; options = add_common_options(builtin_multi_pack_index_expire_options); @@ -364,11 +364,11 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_expire_usage, options); - source = handle_object_dir_option(the_repository); + packed_source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return expire_midx_packs(source->packed, opts.flags); + return expire_midx_packs(packed_source, opts.flags); } static int cmd_multi_pack_index_repack(int argc, const char **argv, @@ -381,7 +381,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, N_("during repack, collect pack-files of smaller size into a batch that is larger than this size")), OPT_END(), }; - struct odb_source_files *source; + struct odb_source_packed *packed_source; options = add_common_options(builtin_multi_pack_index_repack_options); @@ -396,11 +396,11 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_repack_usage, options); - source = handle_object_dir_option(the_repository); + packed_source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return midx_repack(source->packed, (size_t)opts.batch_size, opts.flags); + return midx_repack(packed_source, (size_t)opts.batch_size, opts.flags); } int cmd_multi_pack_index(int argc, @@ -426,8 +426,8 @@ int cmd_multi_pack_index(int argc, if (the_repository && the_repository->objects && - the_repository->objects->sources) - opts.object_dir = xstrdup(the_repository->objects->sources->path); + the_repository->objects->source) + opts.object_dir = xstrdup(the_repository->objects->source->path); argc = parse_options(argc, argv, prefix, options, builtin_multi_pack_index_usage, 0); diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 438bdb359bdddf..27236c348c8f2e 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1560,11 +1560,10 @@ static int want_cruft_object_mtime(struct repository *r, const struct object_id *oid, unsigned flags, uint32_t mtime) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct packed_git **cache = packfile_store_get_kept_pack_cache(files->packed, flags); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + struct packed_git **cache = packfile_store_get_kept_pack_cache(dir->packed, flags); for (; *cache; cache++) { struct packed_git *p = *cache; @@ -1747,21 +1746,19 @@ static int want_object_in_pack_mtime(const struct object_id *oid, off_t *found_offset, uint32_t found_mtime) { + struct odb_source_files *files = + odb_source_files_downcast(the_repository->objects->source); int want; struct packfile_list_entry *e; - struct odb_source *source; if (!exclude && local) { /* - * Note that we start iterating at `sources->next` so that we - * skip the local object source. + * Note that we start iterating at `dirs->next` so that we + * skip the local object directory. */ - struct odb_source *source = the_repository->objects->sources->next; - for (; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0, NULL)) + for (struct odb_files_dir *dir = files->dirs->next; dir; dir = dir->next) + if (!odb_source_read_object_info(&dir->loose->base, oid, NULL, 0, NULL)) return 0; - } } /* @@ -1779,9 +1776,8 @@ static int want_object_in_pack_mtime(const struct object_id *oid, *found_offset = 0; } - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct multi_pack_index *m = get_multi_pack_index(files->packed); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + struct multi_pack_index *m = get_multi_pack_index(dir->packed); struct pack_entry e; if (m && midx_fill_entry(m, oid, &e, NULL) == MIDX_FILL_HIT) { @@ -1791,14 +1787,12 @@ static int want_object_in_pack_mtime(const struct object_id *oid, } } - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - for (e = files->packed->packs.head; e; e = e->next) { + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + for (e = dir->packed->packs.head; e; e = e->next) { struct packed_git *p = e->pack; want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime); if (!exclude && want > 0) - packfile_list_prepend(&files->packed->packs, p); + packfile_list_prepend(&dir->packed->packs, p); if (want != -1) return want; } @@ -4155,14 +4149,13 @@ static void add_cruft_object_entry(const struct object_id *oid, enum object_type if (!want_object_in_pack_mtime(oid, 0, &pack, &offset, mtime)) return; if (!pack && type == OBJ_BLOB) { - struct odb_source *source = the_repository->objects->sources; + struct odb_source_files *files = + odb_source_files_downcast(the_repository->objects->source); int found = 0; - for (; !found && source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0, NULL)) + for (struct odb_files_dir *dir = files->dirs; !found && dir; dir = dir->next) + if (!odb_source_read_object_info(&dir->loose->base, oid, NULL, 0, NULL)) found = 1; - } /* * If a traversed tree has a missing blob then we want @@ -4497,7 +4490,8 @@ static int add_object_in_unpacked_pack(const struct object_id *oid, static void add_objects_in_unpacked_packs(void) { - struct odb_source *source; + struct odb_source_files *files = + odb_source_files_downcast(to_pack.repo->objects->source); time_t mtime; struct odb_for_each_object_options opts = { .flags = ODB_FOR_EACH_OBJECT_PACK_ORDER | @@ -4511,13 +4505,11 @@ static void add_objects_in_unpacked_packs(void) .source_infop = &source_info, }; - for (source = to_pack.repo->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - if (!source->local) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!dir->local) continue; - if (odb_source_for_each_object(&files->packed->base, &oi, + if (odb_source_for_each_object(&dir->packed->base, &oi, add_object_in_unpacked_pack, NULL, &opts)) die(_("cannot open pack index")); } @@ -4561,7 +4553,7 @@ static int add_loose_object(const struct object_id *oid, const char *path, */ static void add_unreachable_loose_objects(struct rev_info *revs) { - for_each_loose_file_in_source(the_repository->objects->sources, + for_each_loose_file_in_source(the_repository->objects->source, add_loose_object, NULL, NULL, revs); } @@ -4625,11 +4617,9 @@ static int force_object_loose(struct odb_source *source, size_t len; int ret; - for (struct odb_source *s = source->odb->sources; s; s = s->next) { - struct odb_source_files *files = odb_source_files_downcast(s); - if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0, NULL)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) + if (!odb_source_read_object_info(&dir->loose->base, oid, NULL, 0, NULL)) return 0; - } oi.typep = &type; oi.sizep = &len; @@ -4649,7 +4639,7 @@ static int force_object_loose(struct odb_source *source, compat_oid_p = &compat_oid; } - ret = odb_source_write_object(&files->loose->base, buf, len, type, oid, + ret = odb_source_write_object(&files->dirs->loose->base, buf, len, type, oid, compat_oid_p, mtime, 0); out: @@ -4676,7 +4666,7 @@ static void loosen_unused_packed_objects(void) if (!packlist_find(&to_pack, &oid) && !has_sha1_pack_kept_or_nonlocal(&oid) && !loosened_object_can_be_discarded(&oid, p->mtime)) { - if (force_object_loose(the_repository->objects->sources, + if (force_object_loose(the_repository->objects->source, &oid, &p->mtime)) die(_("unable to force loose object")); loosened_objects_nr++; diff --git a/builtin/prune.c b/builtin/prune.c index a7e4678d117072..97f4c1418bc58e 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -198,7 +198,7 @@ int cmd_prune(int argc, revs.exclude_promisor_objects = 1; } - for_each_loose_file_in_source(repo->objects->sources, + for_each_loose_file_in_source(repo->objects->source, prune_object, prune_cruft, prune_subdir, &revs); prune_packed_objects(show_only ? PRUNE_PACKED_DRY_RUN : 0); diff --git a/builtin/repack.c b/builtin/repack.c index a8652e2d6d447e..4686fe12006125 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -623,7 +623,7 @@ int cmd_repack(int argc, * midx_has_unknown_packs() will make the decision for * us. */ - if (!get_multi_pack_index(files->packed)) + if (!get_multi_pack_index(files->dirs->packed)) midx_must_contain_cruft = 1; } @@ -781,7 +781,7 @@ int cmd_repack(int argc, if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(files->packed, NULL, NULL, flags); + write_midx_file(files->dirs->packed, NULL, NULL, flags); } cleanup: diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 2a29d649b8c486..af6bed2662d0f0 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -30,6 +30,7 @@ #include "object-name.h" #include "odb.h" #include "odb/source.h" +#include "odb/source-files.h" #include "advice.h" #include "branch.h" #include "list-objects-filter-options.h" @@ -1770,7 +1771,7 @@ static const char alternate_error_advice[] = N_( ); static int add_possible_reference_from_superproject( - struct odb_source *alt_odb, void *sas_cb) + struct odb_files_dir *alt_odb, void *sas_cb) { struct submodule_alternate_setup *sas = sas_cb; size_t len; @@ -1779,12 +1780,12 @@ static int add_possible_reference_from_superproject( * If the alternate object store is another repository, try the * standard layout with .git/(modules/)+/objects */ - if (strip_suffix(alt_odb->path, "/objects", &len)) { + if (strip_suffix(alt_odb->abspath, "/objects", &len)) { struct repository alternate; char *sm_alternate; struct strbuf sb = STRBUF_INIT; struct strbuf err = STRBUF_INIT; - strbuf_add(&sb, alt_odb->path, len); + strbuf_add(&sb, alt_odb->abspath, len); if (repo_init(&alternate, sb.buf, NULL) < 0) die(_("could not get a repository handle for gitdir '%s'"), diff --git a/bundle.c b/bundle.c index d371fd7a43101d..ef127891db7c24 100644 --- a/bundle.c +++ b/bundle.c @@ -239,7 +239,7 @@ int verify_bundle(struct repository *r, .quiet = 1, }; - if (!r || !r->objects || !r->objects->sources) + if (!r || !r->objects || !r->objects->source) return error(_("need a repository to verify a bundle")); for (i = 0; i < p->nr; i++) { diff --git a/commit-graph.c b/commit-graph.c index 25b325a1ee4ac9..2e6844e1bdf21e 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -1,6 +1,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" +#include "abspath.h" #include "config.h" #include "csum-file.h" #include "environment.h" @@ -14,6 +15,7 @@ #include "hash-lookup.h" #include "commit-graph.h" #include "odb.h" +#include "odb/source-files.h" #include "oid-array.h" #include "path.h" #include "alloc.h" @@ -28,7 +30,7 @@ #include "tree.h" #include "chunk-format.h" -void git_test_write_commit_graph_or_die(struct odb_source *source) +void git_test_write_commit_graph_or_die(struct repository *repo) { int flags = 0; if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0)) @@ -37,7 +39,7 @@ void git_test_write_commit_graph_or_die(struct odb_source *source) if (git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags = COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - if (write_commit_graph_reachable(source, flags, NULL)) + if (write_commit_graph_reachable(repo, repo->objects->source->path, flags, NULL)) die("failed to write commit-graph under GIT_TEST_COMMIT_GRAPH"); } @@ -196,21 +198,21 @@ static int commit_gen_cmp(const void *va, const void *vb) return 0; } -char *get_commit_graph_filename(struct odb_source *source) +char *get_commit_graph_filename(const char *dir) { - return xstrfmt("%s/info/commit-graph", source->path); + return xstrfmt("%s/info/commit-graph", dir); } -static char *get_split_graph_filename(struct odb_source *source, +static char *get_split_graph_filename(const char *dir, const char *oid_hex) { - return xstrfmt("%s/info/commit-graphs/graph-%s.graph", source->path, + return xstrfmt("%s/info/commit-graphs/graph-%s.graph", dir, oid_hex); } -char *get_commit_graph_chain_filename(struct odb_source *source) +char *get_commit_graph_chain_filename(const char *dir) { - return xstrfmt("%s/info/commit-graphs/commit-graph-chain", source->path); + return xstrfmt("%s/info/commit-graphs/commit-graph-chain", dir); } static struct commit_graph *alloc_commit_graph(void) @@ -253,7 +255,8 @@ int open_commit_graph(const char *graph_file, int *fd, struct stat *st) return 1; } -struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, +struct commit_graph *load_commit_graph_one_fd_st(struct repository *repo, + const char *dir, int fd, struct stat *st) { void *graph_map; @@ -262,7 +265,7 @@ struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, graph_size = xsize_t(st->st_size); - if (graph_size < graph_min_size(source->odb->repo->hash_algo)) { + if (graph_size < graph_min_size(repo->hash_algo)) { close(fd); error(_("commit-graph file is too small")); return NULL; @@ -270,9 +273,9 @@ struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - ret = parse_commit_graph(source->odb->repo, graph_map, graph_size); + ret = parse_commit_graph(repo, graph_map, graph_size); if (ret) - ret->odb_source = source; + ret->dir = absolute_pathdup(dir); else munmap(graph_map, graph_size); @@ -410,6 +413,7 @@ struct commit_graph *parse_commit_graph(struct repository *r, graph = alloc_commit_graph(); + graph->repo = r; graph->hash_algo = r->hash_algo; graph->num_chunks = *(unsigned char*)(data + 6); graph->data = graph_map; @@ -490,7 +494,8 @@ struct commit_graph *parse_commit_graph(struct repository *r, return NULL; } -static struct commit_graph *load_commit_graph_one(struct odb_source *source, +static struct commit_graph *load_commit_graph_one(struct repository *repo, + const char *dir, const char *graph_file) { struct stat st; @@ -501,17 +506,18 @@ static struct commit_graph *load_commit_graph_one(struct odb_source *source, if (!open_ok) return NULL; - g = load_commit_graph_one_fd_st(source, fd, &st); + g = load_commit_graph_one_fd_st(repo, dir, fd, &st); if (g) - g->filename = xstrdup(graph_file); + g->filename = absolute_pathdup(graph_file); return g; } -static struct commit_graph *load_commit_graph_v1(struct odb_source *source) +static struct commit_graph *load_commit_graph_v1(struct repository *repo, + const char *dir) { - char *graph_name = get_commit_graph_filename(source); - struct commit_graph *g = load_commit_graph_one(source, graph_name); + char *graph_name = get_commit_graph_filename(dir); + struct commit_graph *g = load_commit_graph_one(repo, dir, graph_name); free(graph_name); return g; @@ -652,7 +658,7 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, CALLOC_ARRAY(oids, count); for (i = 0; i < count; i++) { - struct odb_source *source; + struct odb_source_files *files; if (strbuf_getline_lf(&line, fp) == EOF) break; @@ -665,9 +671,11 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, } valid = 0; - for (source = odb->sources; source; source = source->next) { - char *graph_name = get_split_graph_filename(source, line.buf); - struct commit_graph *g = load_commit_graph_one(source, graph_name); + + files = odb_source_files_downcast(odb->source); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + char *graph_name = get_split_graph_filename(dir->abspath, line.buf); + struct commit_graph *g = load_commit_graph_one(odb->repo, dir->abspath, graph_name); free(graph_name); @@ -700,29 +708,31 @@ struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, return graph_chain; } -static struct commit_graph *load_commit_graph_chain(struct odb_source *source) +static struct commit_graph *load_commit_graph_chain(struct repository *repo, + const char *dir) { - char *chain_file = get_commit_graph_chain_filename(source); + char *chain_file = get_commit_graph_chain_filename(dir); struct stat st; int fd; struct commit_graph *g = NULL; - if (open_commit_graph_chain(chain_file, &fd, &st, source->odb->repo->hash_algo)) { + if (open_commit_graph_chain(chain_file, &fd, &st, repo->hash_algo)) { int incomplete; /* ownership of fd is taken over by load function */ - g = load_commit_graph_chain_fd_st(source->odb, fd, &st, &incomplete); + g = load_commit_graph_chain_fd_st(repo->objects, fd, &st, &incomplete); } free(chain_file); return g; } -struct commit_graph *read_commit_graph_one(struct odb_source *source) +struct commit_graph *read_commit_graph_one(struct repository *repo, + const char *dir) { - struct commit_graph *g = load_commit_graph_v1(source); + struct commit_graph *g = load_commit_graph_v1(repo, dir); if (!g) - g = load_commit_graph_chain(source); + g = load_commit_graph_chain(repo, dir); return g; } @@ -735,7 +745,7 @@ struct commit_graph *read_commit_graph_one(struct odb_source *source) */ static struct commit_graph *prepare_commit_graph(struct repository *r) { - struct odb_source *source; + struct odb_source_files *files; /* * Early return if there is no object database or if the commit graph is @@ -766,8 +776,9 @@ static struct commit_graph *prepare_commit_graph(struct repository *r) if (!commit_graph_compatible(r)) return NULL; - for (source = r->objects->sources; source; source = source->next) { - r->objects->commit_graph = read_commit_graph_one(source); + files = odb_source_files_downcast(r->objects->source); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + r->objects->commit_graph = read_commit_graph_one(r, dir->abspath); if (r->objects->commit_graph) break; } @@ -866,7 +877,7 @@ static struct commit_list **insert_parent_or_die(struct commit_graph *g, die("invalid parent position %"PRIu32, pos); load_oid_from_graph(g, pos, &oid); - c = lookup_commit(g->odb_source->odb->repo, &oid); + c = lookup_commit(g->repo, &oid); if (!c) die(_("could not find commit %s"), oid_to_hex(&oid)); commit_graph_data_at(c)->graph_pos = pos; @@ -1103,7 +1114,7 @@ static struct tree *load_tree_for_commit(struct commit_graph *g, graph_pos - g->num_commits_in_base); oidread(&oid, commit_data, g->hash_algo); - set_commit_tree(c, lookup_tree(g->odb_source->odb->repo, &oid)); + set_commit_tree(c, lookup_tree(g->repo, &oid)); return c->maybe_tree; } @@ -1126,7 +1137,7 @@ struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit struct write_commit_graph_context { struct repository *r; - struct odb_source *odb_source; + char *dir; char *graph_name; struct oid_array oids; struct commit_stack commits; @@ -1902,7 +1913,8 @@ static int add_ref_to_set(const struct reference *ref, void *cb_data) return 0; } -int write_commit_graph_reachable(struct odb_source *source, +int write_commit_graph_reachable(struct repository *repo, + const char *dir, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts) { @@ -1911,20 +1923,20 @@ int write_commit_graph_reachable(struct odb_source *source, int result; memset(&data, 0, sizeof(data)); - data.repo = source->odb->repo; + data.repo = repo; data.commits = &commits; if (flags & COMMIT_GRAPH_WRITE_PROGRESS) data.progress = start_delayed_progress( - source->odb->repo, + repo, _("Collecting referenced commits"), 0); - refs_for_each_ref(get_main_ref_store(source->odb->repo), add_ref_to_set, + refs_for_each_ref(get_main_ref_store(repo), add_ref_to_set, &data); stop_progress(&data.progress); - result = write_commit_graph(source, NULL, &commits, + result = write_commit_graph(repo, dir, NULL, &commits, flags, opts); oidset_clear(&commits); @@ -1936,12 +1948,8 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, { uint32_t i; struct strbuf progress_title = STRBUF_INIT; - struct strbuf packname = STRBUF_INIT; - int dirlen; int ret = 0; - strbuf_addf(&packname, "%s/pack/", ctx->odb_source->path); - dirlen = packname.len; if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Finding commits for commit graph in %"PRIuMAX" pack", @@ -1954,15 +1962,15 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, } for (i = 0; i < pack_indexes->nr; i++) { struct packed_git *p; - strbuf_setlen(&packname, dirlen); - strbuf_addstr(&packname, pack_indexes->items[i].string); - p = add_packed_git(ctx->r, packname.buf, packname.len, 1); + + p = add_packed_git(ctx->r, pack_indexes->items[i].string, + strlen(pack_indexes->items[i].string), 1); if (!p) { - ret = error(_("error adding pack %s"), packname.buf); + ret = error(_("error adding pack %s"), pack_indexes->items[i].string); goto cleanup; } if (open_pack_index(p)) { - ret = error(_("error opening index for %s"), packname.buf); + ret = error(_("error opening index for %s"), pack_indexes->items[i].string); close_pack(p); free(p); goto cleanup; @@ -1976,7 +1984,6 @@ static int fill_oids_from_packs(struct write_commit_graph_context *ctx, cleanup: stop_progress(&ctx->progress); strbuf_release(&progress_title); - strbuf_release(&packname); return ret; } @@ -2000,7 +2007,7 @@ static int fill_oids_from_commits(struct write_commit_graph_context *ctx, static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) { - struct odb_source *source; + struct odb_source_files *files; enum object_type type; struct odb_for_each_object_options opts = { .flags = ODB_FOR_EACH_OBJECT_PACK_ORDER, @@ -2015,9 +2022,9 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) _("Finding commits for commit graph among packed objects"), ctx->approx_nr_objects); - for (source = ctx->r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_for_each_object(&files->packed->base, &oi, add_packed_commits_oi, + files = odb_source_files_downcast(ctx->r->objects->source); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + odb_source_for_each_object(&dir->packed->base, &oi, add_packed_commits_oi, ctx, &opts); } @@ -2108,10 +2115,10 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) strbuf_addf(&tmp_file, "%s/info/commit-graphs/tmp_graph_XXXXXX", - ctx->odb_source->path); + ctx->dir); ctx->graph_name = strbuf_detach(&tmp_file, NULL); } else { - ctx->graph_name = get_commit_graph_filename(ctx->odb_source); + ctx->graph_name = get_commit_graph_filename(ctx->dir); } if (safe_create_leading_directories(ctx->r, ctx->graph_name)) { @@ -2121,7 +2128,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) } if (ctx->split) { - char *lock_name = get_commit_graph_chain_filename(ctx->odb_source); + char *lock_name = get_commit_graph_chain_filename(ctx->dir); repo_hold_lock_file_for_update_mode(ctx->r, &lk, lock_name, LOCK_DIE_ON_ERROR, 0444); @@ -2210,7 +2217,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) if (ctx->split && ctx->base_graph_name && ctx->num_commit_graphs_after > 1) { char *new_base_hash = xstrdup(oid_to_hex(&ctx->new_base_graph->oid)); - char *new_base_name = get_split_graph_filename(ctx->new_base_graph->odb_source, new_base_hash); + char *new_base_name = get_split_graph_filename(ctx->new_base_graph->dir, new_base_hash); free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2]); free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2]); @@ -2250,7 +2257,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) } } } else { - char *graph_name = get_commit_graph_filename(ctx->odb_source); + char *graph_name = get_commit_graph_filename(ctx->dir); unlink(graph_name); free(graph_name); } @@ -2258,7 +2265,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]); ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1] = xstrdup(hash_to_hex_algop(file_hash, ctx->r->hash_algo)); - final_graph_name = get_split_graph_filename(ctx->odb_source, + final_graph_name = get_split_graph_filename(ctx->dir, ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]); free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1]); ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1] = final_graph_name; @@ -2310,7 +2317,7 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx, flags != COMMIT_GRAPH_SPLIT_REPLACE) { while (g && (g->num_commits <= st_mult(size_mult, num_commits) || (max_commits && num_commits > max_commits))) { - if (g->odb_source != ctx->odb_source) + if (strcmp(g->dir, ctx->dir)) break; if (unsigned_add_overflows(num_commits, g->num_commits)) @@ -2332,10 +2339,10 @@ static void split_graph_merge_strategy(struct write_commit_graph_context *ctx, "should be 1 with --split=replace"); if (ctx->num_commit_graphs_after == 2) { - char *old_graph_name = get_commit_graph_filename(g->odb_source); + char *old_graph_name = get_commit_graph_filename(g->dir); if (!strcmp(g->filename, old_graph_name) && - g->odb_source != ctx->odb_source) { + strcmp(g->dir, ctx->dir)) { ctx->num_commit_graphs_after = 1; ctx->new_base_graph = NULL; } @@ -2505,13 +2512,13 @@ static void expire_commit_graphs(struct write_commit_graph_context *ctx) if (ctx->opts && ctx->opts->expire_time) expire_time = ctx->opts->expire_time; if (!ctx->split) { - char *chain_file_name = get_commit_graph_chain_filename(ctx->odb_source); + char *chain_file_name = get_commit_graph_chain_filename(ctx->dir); unlink(chain_file_name); free(chain_file_name); ctx->num_commit_graphs_after = 0; } - strbuf_addstr(&path, ctx->odb_source->path); + strbuf_addstr(&path, ctx->dir); strbuf_addstr(&path, "/info/commit-graphs"); dir = opendir(path.buf); @@ -2553,16 +2560,15 @@ static void expire_commit_graphs(struct write_commit_graph_context *ctx) strbuf_release(&path); } -int write_commit_graph(struct odb_source *source, +int write_commit_graph(struct repository *r, + const char *dir, const struct string_list *const pack_indexes, struct oidset *commits, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts) { - struct repository *r = source->odb->repo; struct write_commit_graph_context ctx = { .r = r, - .odb_source = source, .append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0, .report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0, .split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0, @@ -2593,6 +2599,8 @@ int write_commit_graph(struct odb_source *source, return 0; } + ctx.dir = absolute_pathdup(dir); + bloom_settings.hash_version = r->settings.commit_graph_changed_paths_version; bloom_settings.bits_per_entry = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY", bloom_settings.bits_per_entry); @@ -2716,6 +2724,7 @@ int write_commit_graph(struct odb_source *source, cleanup: free(ctx.graph_name); free(ctx.base_graph_name); + free(ctx.dir); commit_stack_clear(&ctx.commits); oid_array_clear(&ctx.oids); clear_topo_level_slab(&topo_levels); @@ -2768,7 +2777,7 @@ static int verify_one_commit_graph(struct commit_graph *g, struct progress *progress, uint64_t *seen) { - struct repository *r = g->odb_source->odb->repo; + struct repository *r = g->repo; uint32_t i, cur_fanout_pos = 0; struct object_id prev_oid, cur_oid; struct commit *seen_gen_zero = NULL; @@ -2937,7 +2946,7 @@ int verify_commit_graph(struct commit_graph *g, int flags) if (!(flags & COMMIT_GRAPH_VERIFY_SHALLOW)) total += g->num_commits_in_base; - progress = start_progress(g->odb_source->odb->repo, + progress = start_progress(g->repo, _("Verifying commits in commit graph"), total); } @@ -2960,6 +2969,7 @@ void free_commit_graph(struct commit_graph *g) if (g->data) munmap((void *)g->data, g->data_len); + free(g->dir); free(g->filename); free(g->bloom_filter_settings); free(g); diff --git a/commit-graph.h b/commit-graph.h index 13ca4ff010fa18..bccf9c5c844832 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -21,7 +21,7 @@ * call this method outside of a builtin, and only if you know what * you are doing! */ -void git_test_write_commit_graph_or_die(struct odb_source *source); +void git_test_write_commit_graph_or_die(struct repository *repo); struct commit; struct bloom_filter_settings; @@ -29,8 +29,8 @@ struct repository; struct object_database; struct string_list; -char *get_commit_graph_filename(struct odb_source *source); -char *get_commit_graph_chain_filename(struct odb_source *source); +char *get_commit_graph_filename(const char *dir); +char *get_commit_graph_chain_filename(const char *dir); int open_commit_graph(const char *graph_file, int *fd, struct stat *st); int open_commit_graph_chain(const char *chain_file, int *fd, struct stat *st, const struct git_hash_algo *hash_algo); @@ -85,12 +85,13 @@ struct commit_graph { const unsigned char *data; size_t data_len; + struct repository *repo; const struct git_hash_algo *hash_algo; unsigned char num_chunks; uint32_t num_commits; struct object_id oid; + char *dir; char *filename; - struct odb_source *odb_source; uint32_t num_commits_in_base; unsigned int read_generation_data; @@ -114,12 +115,20 @@ struct commit_graph { struct bloom_filter_settings *bloom_filter_settings; }; -struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source, +/* + * Load commit graphs from the given object directory `dir`. The directory may + * be given as a relative path; it is canonicalized internally so that graphs + * loaded from the same directory compare equal regardless of how the caller + * spelled the path. + */ +struct commit_graph *load_commit_graph_one_fd_st(struct repository *repo, + const char *dir, int fd, struct stat *st); struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb, int fd, struct stat *st, int *incomplete_chain); -struct commit_graph *read_commit_graph_one(struct odb_source *source); +struct commit_graph *read_commit_graph_one(struct repository *repo, + const char *dir); struct repo_settings; @@ -171,11 +180,17 @@ struct commit_graph_opts { * and a negative value on failure. Note that if the repository * is not compatible with the commit-graph feature, then the * methods will return 0 without writing a commit-graph. + * + * The object directory `dir` may be given as a relative path; it is + * canonicalized internally so that it compares equal to the directory of + * graphs that have already been loaded. */ -int write_commit_graph_reachable(struct odb_source *source, +int write_commit_graph_reachable(struct repository *repo, + const char *dir, enum commit_graph_write_flags flags, const struct commit_graph_opts *opts); -int write_commit_graph(struct odb_source *source, +int write_commit_graph(struct repository *r, + const char *dir, const struct string_list *pack_indexes, struct oidset *commits, enum commit_graph_write_flags flags, diff --git a/diagnose.c b/diagnose.c index 9d51ded427a265..2381130c2d5cda 100644 --- a/diagnose.c +++ b/diagnose.c @@ -60,13 +60,13 @@ static void dir_file_stats_objects(const char *full_path, (uintmax_t)st.st_size); } -static int dir_file_stats(struct odb_source *source, void *data) +static int dir_file_stats(struct odb_files_dir *source, void *data) { struct strbuf *buf = data; - strbuf_addf(buf, "Contents of %s:\n", source->path); + strbuf_addf(buf, "Contents of %s:\n", source->abspath); - for_each_file_in_pack_dir(source->path, dir_file_stats_objects, + for_each_file_in_pack_dir(source->abspath, dir_file_stats_objects, data); return 0; @@ -270,7 +270,7 @@ int create_diagnostics_archive(struct repository *r, strbuf_reset(&buf); strbuf_addstr(&buf, "--add-virtual-file=packs-local.txt:"); - dir_file_stats(r->objects->sources, &buf); + dir_file_stats(odb_source_files_downcast(r->objects->source)->dirs, &buf); odb_for_each_alternate(r->objects, dir_file_stats, &buf); strvec_push(&archiver_args, buf.buf); diff --git a/fetch-pack.c b/fetch-pack.c index ea97eed0bcdc48..058de0fb976653 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1077,7 +1077,7 @@ static int get_pack(struct fetch_pack_args *args, die(_("fetch-pack: unable to fork off %s"), cmd_name); if (do_keep && (pack_lockfiles || fsck_objects)) { int is_well_formed; - char *pack_lockfile = index_pack_lockfile(the_repository->objects->sources, + char *pack_lockfile = index_pack_lockfile(the_repository->objects->source, cmd.out, &is_well_formed); diff --git a/gvfs-helper-client.c b/gvfs-helper-client.c index 79731321386944..1458060c4378e5 100644 --- a/gvfs-helper-client.c +++ b/gvfs-helper-client.c @@ -26,7 +26,7 @@ struct gh_server__process { static int gh_server__subprocess_map_initialized; static struct hashmap gh_server__subprocess_map; -static struct odb_source *gh_client__chosen_odb; +static struct odb_files_dir *gh_client__chosen_odb; /* * The "objects" capability has verbs: "get" and "post" and "prefetch". @@ -188,7 +188,8 @@ static void gh_client__update_loose_cache(const char *line) if (get_oid_hex(v1_oid, &oid)) BUG("update_loose_cache: invalid line '%s'", line); - odb_source_loose_cache_add_new_oid(gh_client__chosen_odb, &oid); + odb_source_loose_cache_add_new_oid(&gh_client__chosen_odb->loose->base, + &oid); } /* @@ -271,8 +272,8 @@ static int gh_client__objects__receive_response( } if (ghc & GHC__CREATED__PACKFILE) { - struct odb_source_files *files = odb_source_files_downcast(gh_client__chosen_odb); - odb_source_prepare(&files->packed->base, ODB_PREPARE_FLUSH_CACHES); + odb_source_prepare(&gh_client__chosen_odb->packed->base, + ODB_PREPARE_FLUSH_CACHES); } *p_ghc = ghc; @@ -291,19 +292,20 @@ static int gh_client__objects__receive_response( */ static void gh_client__choose_odb(void) { - struct odb_source *odb; + struct odb_files_dir *odb; if (gh_client__chosen_odb) return; odb_prepare(the_repository->objects, ODB_PREPARE_FLUSH_CACHES); - gh_client__chosen_odb = the_repository->objects->sources; + gh_client__chosen_odb = + odb_source_files_downcast(the_repository->objects->source)->dirs; if (!gvfs_shared_cache_pathname.len) return; - for (odb = the_repository->objects->sources->next; odb; odb = odb->next) { - if (!fspathcmp(odb->path, gvfs_shared_cache_pathname.buf)) { + for (odb = gh_client__chosen_odb->next; odb; odb = odb->next) { + if (!fspathcmp(odb->abspath, gvfs_shared_cache_pathname.buf)) { gh_client__chosen_odb = odb; return; } @@ -356,7 +358,7 @@ static struct gh_server__process *gh_client__find_long_running_process( strvec_push(&argv, "gvfs-helper"); strvec_push(&argv, "--cache-server=trust"); strvec_pushf(&argv, "--shared-cache=%s", - gh_client__chosen_odb->path); + gh_client__chosen_odb->abspath); /* If gvfs.fallback=false, then don't add --fallback. */ if (!repo_config_get_bool(the_repository, "gvfs.fallback", &fallback) && diff --git a/gvfs-helper.c b/gvfs-helper.c index abb68914b7b370..de704826c11b55 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -1416,7 +1416,8 @@ static int option_parse_shared_cache_directory(const struct option *opt, gvfs_shared_cache_pathname.buf); mkdir(gvfs_shared_cache_pathname.buf, 0777); - add_gvfs_shared_cache_to_alternates(the_repository->objects, buf_arg.buf); + odb_add_to_alternates_memory(the_repository->objects, + buf_arg.buf); strbuf_release(&buf_arg); return 0; @@ -1438,7 +1439,8 @@ static int option_parse_shared_cache_directory(const struct option *opt, gvfs_shared_cache_pathname.buf); mkdir(gvfs_shared_cache_pathname.buf, 0777); - add_gvfs_shared_cache_to_alternates(the_repository->objects, buf_arg.buf); + odb_add_to_alternates_memory(the_repository->objects, + buf_arg.buf); /* * alt_odb_usable() releases gvfs_shared_cache_pathname diff --git a/gvfs.h b/gvfs.h index f305c5050b8ac0..02fa0040c948cf 100644 --- a/gvfs.h +++ b/gvfs.h @@ -43,8 +43,4 @@ struct repository; int gvfs_config_is_set(struct repository *r, int mask); int gvfs_virtualize_objects(struct repository *r); -struct object_database; -struct odb_source *add_gvfs_shared_cache_to_alternates(struct object_database *odb, - const char *dir); - #endif /* GVFS_H */ diff --git a/http-walker.c b/http-walker.c index abafca84d65441..4f50dd9b8d02a4 100644 --- a/http-walker.c +++ b/http-walker.c @@ -540,9 +540,9 @@ static int fetch_object(struct walker *walker, const struct object_id *oid) } else if (!oideq(&obj_req->oid, &req->real_oid)) { ret = error("File %s has bad hash", hex); } else if (req->rename < 0) { - struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); struct strbuf buf = STRBUF_INIT; - odb_loose_path(files->loose, &buf, &req->oid); + odb_loose_path(files->dirs->loose, &buf, &req->oid); ret = error("unable to write sha1 filename %s", buf.buf); strbuf_release(&buf); } diff --git a/http.c b/http.c index 67dcf67e8372f4..b2068a1e7eb074 100644 --- a/http.c +++ b/http.c @@ -2850,9 +2850,9 @@ int finish_http_pack_request(struct http_pack_request *preq) void http_install_packfile(struct packed_git *p, struct packfile_list *list_to_remove_from) { - struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); packfile_list_remove(list_to_remove_from, p); - packfile_store_add_pack(files->packed, p); + packfile_store_add_pack(files->dirs->packed, p); } struct http_pack_request *new_http_pack_request( @@ -2978,7 +2978,7 @@ static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb, struct http_object_request *new_http_object_request(const char *base_url, const struct object_id *oid) { - struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); char *hex = oid_to_hex(oid); struct strbuf filename = STRBUF_INIT; struct strbuf prevfile = STRBUF_INIT; @@ -2993,7 +2993,7 @@ struct http_object_request *new_http_object_request(const char *base_url, oidcpy(&freq->oid, oid); freq->localfile = -1; - odb_loose_path(files->loose, &filename, oid); + odb_loose_path(files->dirs->loose, &filename, oid); strbuf_addf(&freq->tmpfile, "%s.temp", filename.buf); strbuf_addf(&prevfile, "%s.prev", filename.buf); @@ -3119,7 +3119,7 @@ void process_http_object_request(struct http_object_request *freq) int finish_http_object_request(struct http_object_request *freq) { - struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->source); struct stat st; struct strbuf filename = STRBUF_INIT; @@ -3146,7 +3146,7 @@ int finish_http_object_request(struct http_object_request *freq) unlink_or_warn(freq->tmpfile.buf); return -1; } - odb_loose_path(files->loose, &filename, &freq->oid); + odb_loose_path(files->dirs->loose, &filename, &freq->oid); freq->rename = finalize_object_file(the_repository, freq->tmpfile.buf, filename.buf); strbuf_release(&filename); diff --git a/log-tree.c b/log-tree.c index 83a3c4bf9b16b9..361dfadbb368f0 100644 --- a/log-tree.c +++ b/log-tree.c @@ -1043,10 +1043,9 @@ static int do_remerge_diff(struct rev_info *opt, * into the alternative object store list as the primary. */ if (opt->remerge_diff && !opt->remerge_objdir) { - opt->remerge_objdir = tmp_objdir_create(the_repository, "remerge-diff"); + opt->remerge_objdir = tmp_objdir_create(the_repository, "remerge-diff", 1); if (!opt->remerge_objdir) return error(_("unable to create temporary object directory")); - tmp_objdir_replace_primary_odb(opt->remerge_objdir, 1); } /* Setup merge options */ diff --git a/loose.c b/loose.c index c159d29d2d2035..d6a2bb5ba3c877 100644 --- a/loose.c +++ b/loose.c @@ -113,11 +113,10 @@ int loose_object_map_load(struct odb_source_loose *loose) int repo_read_loose_object_map(struct repository *repo) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); - for (source = repo->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - if (loose_object_map_load(files->loose) < 0) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (loose_object_map_load(dir->loose) < 0) return -1; } @@ -126,8 +125,8 @@ int repo_read_loose_object_map(struct repository *repo) int repo_write_loose_object_map(struct repository *repo) { - struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources); - kh_oid_map_t *map = files->loose->map->to_compat; + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); + kh_oid_map_t *map = files->dirs->loose->map->to_compat; struct lock_file lock; int fd; khiter_t iter; @@ -231,13 +230,12 @@ int repo_loose_object_map_oid(struct repository *repo, const struct git_hash_algo *to, struct object_id *dest) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); kh_oid_map_t *map; khiter_t pos; - for (source = repo->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct loose_object_map *loose_map = files->loose->map; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + struct loose_object_map *loose_map = dir->loose->map; if (!loose_map) continue; map = (to == repo->compat_hash_algo) ? diff --git a/midx.c b/midx.c index 6d1c548e3dae8e..fbe0a775053648 100644 --- a/midx.c +++ b/midx.c @@ -470,7 +470,7 @@ int prepare_midx_pack(struct multi_pack_index *m, strbuf_addf(&pack_name, "%s/pack/%s", packed->base.path, m->pack_names[pack_int_id]); p = packfile_store_load_pack(packed, - pack_name.buf, packed->base.local); + pack_name.buf, packed->local); strbuf_release(&pack_name); if (!p) { @@ -829,28 +829,22 @@ void clear_incremental_midx_files_ext(struct odb_source_packed *source, const ch void clear_midx_file(struct repository *r) { - struct odb_source_files *files; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); struct strbuf midx = STRBUF_INIT; - if (r->objects) { - struct odb_source *source; - - for (source = r->objects->sources; source; source = source->next) { - files = odb_source_files_downcast(source); - if (files->packed->midx) - close_midx(files->packed->midx); - files->packed->midx = NULL; - } + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (dir->packed->midx) + close_midx(dir->packed->midx); + dir->packed->midx = NULL; } - files = odb_source_files_downcast(r->objects->sources); - get_midx_filename(files->packed, &midx); + get_midx_filename(files->dirs->packed, &midx); if (remove_path(midx.buf)) die(_("failed to clear multi-pack-index at %s"), midx.buf); - clear_midx_files_ext(files->packed, MIDX_EXT_BITMAP, NULL); - clear_midx_files_ext(files->packed, MIDX_EXT_REV, NULL); + clear_midx_files_ext(files->dirs->packed, MIDX_EXT_BITMAP, NULL); + clear_midx_files_ext(files->dirs->packed, MIDX_EXT_REV, NULL); strbuf_release(&midx); } @@ -858,27 +852,24 @@ void clear_midx_file(struct repository *r) void clear_incremental_midx_files(struct repository *r, const struct strvec *keep_hashes) { - struct odb_source_files *files; - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); struct strbuf chain = STRBUF_INIT; - for (source = r->objects->sources; source; source = source->next) { - files = odb_source_files_downcast(source); - if (files->packed->midx) - close_midx(files->packed->midx); - files->packed->midx = NULL; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (dir->packed->midx) + close_midx(dir->packed->midx); + dir->packed->midx = NULL; } - files = odb_source_files_downcast(r->objects->sources); - get_midx_chain_filename(files->packed, &chain); + get_midx_chain_filename(files->dirs->packed, &chain); if (!keep_hashes && remove_path(chain.buf)) die(_("failed to clear multi-pack-index chain at %s"), chain.buf); - clear_incremental_midx_files_ext(files->packed, MIDX_EXT_BITMAP, keep_hashes); - clear_incremental_midx_files_ext(files->packed, MIDX_EXT_REV, keep_hashes); - clear_incremental_midx_files_ext(files->packed, MIDX_EXT_MIDX, keep_hashes); + clear_incremental_midx_files_ext(files->dirs->packed, MIDX_EXT_BITMAP, keep_hashes); + clear_incremental_midx_files_ext(files->dirs->packed, MIDX_EXT_REV, keep_hashes); + clear_incremental_midx_files_ext(files->dirs->packed, MIDX_EXT_MIDX, keep_hashes); strbuf_release(&chain); } diff --git a/object-file.c b/object-file.c index 68751dadc3575d..231e16c3d23033 100644 --- a/object-file.c +++ b/object-file.c @@ -488,7 +488,6 @@ struct odb_transaction_files { enum odb_transaction_flags flags; struct tmp_objdir *objdir; - struct odb_source *quarantine; struct transaction_packfile packfile; const char *prefix; @@ -511,12 +510,10 @@ int odb_transaction_files_prepare(struct odb_transaction *base) if (!transaction || transaction->objdir) return 0; - transaction->objdir = tmp_objdir_create(base->source->odb->repo, transaction->prefix); + transaction->objdir = tmp_objdir_create(base->source->odb->repo, transaction->prefix, 0); if (!transaction->objdir) return error(_("unable to create temporary object directory")); - transaction->quarantine = tmp_objdir_replace_primary_odb(transaction->objdir, 0); - return 0; } @@ -1443,8 +1440,7 @@ static int odb_transaction_files_write_pack(struct odb_transaction *base, return -1; } - odb_source_prepare(transaction->quarantine, - ODB_PREPARE_FLUSH_CACHES); + odb_source_prepare(base->source, ODB_PREPARE_FLUSH_CACHES); } return 0; diff --git a/odb.c b/odb.c index 95adfdb9f07db8..68c1cc1bab609b 100644 --- a/odb.c +++ b/odb.c @@ -16,11 +16,11 @@ #include "object-file.h" #include "object-name.h" #include "odb.h" +#include "odb/source-files.h" #include "odb/source-inmemory.h" #include "path.h" #include "pkt-line.h" #include "promisor-remote.h" -#include "quote.h" #include "replace-object.h" #include "run-command.h" #include "setup.h" @@ -35,48 +35,6 @@ #include "trace2.h" #include "write-or-die.h" -/* - * NEEDSWORK: we're using "core.ignoreCase" to deduplicate alternates that - * _may_ be the same. This requires quite a bit of boilerplate for dubious - * benefit: - * - * - Duplicating alternates should really only lead to regressed performance. - * - * - We don't properly resolve symlinks or mointpoints, so we may still end - * up duplicating alternates. - * - * - The value may be lying, in which case we might deduplicate alternates - * that are in fact not mapping to the same directory. - * - * We should investigate whether we can remove this whole mechanism outright. - */ -static int odb_source_paths_cmp(struct object_database *o, - const char *a, const char *b) -{ - if (o->source_paths_icase < 0) { - int icase = 0; - repo_config_get_bool(o->repo, "core.ignorecase", &icase); - o->source_paths_icase = icase; - } - - return o->source_paths_icase ? strcasecmp(a, b) : strcmp(a, b); -} - -static int odb_source_by_path_cmp(const void *cb_data, - const struct hashmap_entry *entry, - const struct hashmap_entry *entry_or_key, - const void *keydata) -{ - struct object_database *o = (struct object_database *)cb_data; - const struct odb_source *source = container_of(entry, const struct odb_source, by_path_entry); - const char *path = keydata; - - if (!path) - path = container_of(entry_or_key, const struct odb_source, by_path_entry)->path; - - return odb_source_paths_cmp(o, source->path, path); -} - int odb_mkstemp(struct object_database *odb, struct strbuf *temp_filename, const char *pattern) { @@ -98,263 +56,6 @@ int odb_mkstemp(struct object_database *odb, return xmkstemp_mode(temp_filename->buf, mode); } -static int gvfs_matched_shared_cache_to_alternate; - -/* - * Return non-zero iff the path is usable as an alternate object database. - */ -static bool odb_is_source_usable(struct object_database *o, const char *path) -{ - extern struct strbuf gvfs_shared_cache_pathname; - struct strbuf normalized_objdir = STRBUF_INIT; - struct hashmap_entry key; - bool usable = false; - - strbuf_realpath(&normalized_objdir, o->sources->path, 1); - - if (!strcmp(path, gvfs_shared_cache_pathname.buf)) { - /* - * `gvfs.sharedCache` is the preferred alternate that we - * will use with `gvfs-helper.exe` to dynamically fetch - * missing objects. It is set during git_default_config(). - * - * Make sure the directory exists on disk before we let the - * stock code discredit it. - */ - struct strbuf buf_pack_foo = STRBUF_INIT; - enum scld_error scld; - - /* - * Force create the "" and "/pack" directories, if - * not present on disk. Append an extra bogus directory to - * get safe_create_leading_directories() to see "/pack" - * as a leading directory of something deeper (which it - * won't create). - */ - strbuf_addf(&buf_pack_foo, "%s/pack/foo", path); - - scld = safe_create_leading_directories(o->repo, buf_pack_foo.buf); - if (scld != SCLD_OK && scld != SCLD_EXISTS) { - error_errno(_("could not create shared-cache ODB '%s'"), - gvfs_shared_cache_pathname.buf); - - strbuf_release(&buf_pack_foo); - - /* - * Pretend no shared-cache was requested and - * effectively fallback to ".git/objects" for - * fetching missing objects. - */ - strbuf_release(&gvfs_shared_cache_pathname); - return 0; - } - - /* - * We know that there is an alternate (either from - * .git/objects/info/alternates or from a memory-only - * entry) associated with the shared-cache directory. - */ - gvfs_matched_shared_cache_to_alternate++; - strbuf_release(&buf_pack_foo); - } - - /* Detect cases where alternate disappeared */ - if (!is_directory(path)) { - warning(_("object directory %s does not exist; " - "check .git/objects/info/alternates"), - path); - goto out; - } - - /* - * Prevent the common mistake of listing the same - * thing twice, or object directory itself. - */ - if (!hashmap_get_size(&o->source_by_path)) { - assert(!o->sources->next); - hashmap_entry_init(&o->sources->by_path_entry, - strihash(o->sources->path)); - hashmap_add(&o->source_by_path, &o->sources->by_path_entry); - } - - if (!odb_source_paths_cmp(o, path, normalized_objdir.buf)) - goto out; - - hashmap_entry_init(&key, strihash(path)); - if (hashmap_get(&o->source_by_path, &key, path)) - goto out; - - usable = true; - -out: - strbuf_release(&normalized_objdir); - return usable; -} - -void parse_alternates(const char *string, - int sep, - const char *relative_base, - struct strvec *out) -{ - struct strbuf pathbuf = STRBUF_INIT; - struct strbuf buf = STRBUF_INIT; - - if (!string || !*string) - return; - - while (*string) { - const char *end; - - strbuf_reset(&buf); - strbuf_reset(&pathbuf); - - if (*string == '#') { - /* comment; consume up to next separator */ - end = strchrnul(string, sep); - } else if (*string == '"' && !unquote_c_style(&buf, string, &end)) { - /* - * quoted path; unquote_c_style has copied the - * data for us and set "end". Broken quoting (e.g., - * an entry that doesn't end with a quote) falls - * back to the unquoted case below. - */ - } else { - /* normal, unquoted path */ - end = strchrnul(string, sep); - strbuf_add(&buf, string, end - string); - } - - if (*end) - end++; - string = end; - - if (!buf.len) - continue; - - if (!is_absolute_path(buf.buf) && relative_base) { - strbuf_realpath(&pathbuf, relative_base, 1); - strbuf_addch(&pathbuf, '/'); - } - strbuf_addbuf(&pathbuf, &buf); - - strbuf_reset(&buf); - if (!strbuf_realpath(&buf, pathbuf.buf, 0)) { - warning(_("unable to normalize alternate object " - "path: %s"), pathbuf.buf); - continue; - } - - /* - * The trailing slash after the directory name is given by - * this function at the end. Remove duplicates. - */ - while (buf.len && buf.buf[buf.len - 1] == '/') - strbuf_setlen(&buf, buf.len - 1); - - strvec_push(out, buf.buf); - } - - strbuf_release(&pathbuf); - strbuf_release(&buf); -} - -static struct odb_source *odb_add_alternate_recursively(struct object_database *odb, - const char *source, - int depth) -{ - struct odb_source *alternate = NULL; - struct strvec sources = STRVEC_INIT; - - if (!odb_is_source_usable(odb, source)) - goto error; - - alternate = odb_source_new(odb, source, false); - - /* add the alternate entry */ - *odb->sources_tail = alternate; - odb->sources_tail = &(alternate->next); - - hashmap_entry_init(&alternate->by_path_entry, strihash(alternate->path)); - if (hashmap_get(&odb->source_by_path, &alternate->by_path_entry, - alternate->path)) - BUG("source must not yet exist"); - hashmap_add(&odb->source_by_path, &alternate->by_path_entry); - - /* recursively add alternates */ - odb_source_read_alternates(alternate, &sources); - if (sources.nr && depth + 1 > 5) { - error(_("%s: ignoring alternate object stores, nesting too deep"), - source); - } else { - for (size_t i = 0; i < sources.nr; i++) - odb_add_alternate_recursively(odb, sources.v[i], depth + 1); - } - - error: - strvec_clear(&sources); - return alternate; -} - -struct odb_source *add_gvfs_shared_cache_to_alternates(struct object_database *odb, - const char *dir) -{ - return odb_add_alternate_recursively(odb, dir, 0); -} - -void odb_add_to_alternates_file(struct object_database *odb, - const char *dir) -{ - int ret = odb_source_write_alternate(odb->sources, dir); - if (ret < 0) - die(NULL); - odb_add_alternate_recursively(odb, dir, 0); -} - -struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, - const char *dir, int will_destroy, - struct odb_source **prev_source) -{ - struct odb_source *source; - - /* - * Make a new primary odb and link the old primary ODB in as an - * alternate - */ - source = odb_source_new(odb, dir, false); - - /* - * Disable ref updates while a temporary odb is active, since - * the objects in the database may roll back. - */ - odb->repo->disable_ref_updates = true; - source->will_destroy = will_destroy; - source->next = odb->sources; - odb->sources = source; - - if (prev_source) - *prev_source = source->next; - - return source; -} - -void odb_restore_primary_source(struct object_database *odb, - struct odb_source *restore_source, - const char *old_path) -{ - struct odb_source *cur_source = odb->sources; - - if (strcmp(old_path, cur_source->path)) - BUG("expected %s as primary object store; found %s", - old_path, cur_source->path); - - if (cur_source->next != restore_source) - BUG("we expect the old primary object store to be the first alternate"); - - odb->repo->disable_ref_updates = false; - odb->sources = restore_source; - odb_source_free(cur_source); -} - char *compute_alternate_path(const char *path, struct strbuf *err) { char *ref_git = NULL; @@ -419,32 +120,6 @@ char *compute_alternate_path(const char *path, struct strbuf *err) return ref_git; } -struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir) -{ - struct odb_source *source; - char *obj_dir_real = real_pathdup(obj_dir, 1); - struct strbuf odb_path_real = STRBUF_INIT; - - for (source = odb->sources; source; source = source->next) { - strbuf_realpath(&odb_path_real, source->path, 1); - if (!strcmp(obj_dir_real, odb_path_real.buf)) - break; - } - - free(obj_dir_real); - strbuf_release(&odb_path_real); - - return source; -} - -struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir) -{ - struct odb_source *source = odb_find_source(odb, obj_dir); - if (!source) - die(_("could not find object directory matching %s"), obj_dir); - return source; -} - static void fill_alternate_refs_command(struct repository *repo, struct child_process *cmd, const char *repo_path) @@ -507,18 +182,19 @@ static void read_alternate_refs(struct repository *repo, } struct alternate_refs_data { + struct repository *repo; odb_for_each_alternate_ref_fn *fn; void *payload; }; -static int refs_from_alternate_cb(struct odb_source *alternate, +static int refs_from_alternate_cb(struct odb_files_dir *alternate, void *payload) { struct strbuf path = STRBUF_INIT; size_t base_len; struct alternate_refs_data *cb = payload; - if (!strbuf_realpath(&path, alternate->path, 0)) + if (!strbuf_realpath(&path, alternate->abspath, 0)) goto out; if (!strbuf_strip_suffix(&path, "/objects")) goto out; @@ -530,7 +206,7 @@ static int refs_from_alternate_cb(struct odb_source *alternate, goto out; strbuf_setlen(&path, base_len); - read_alternate_refs(alternate->odb->repo, path.buf, cb->fn, cb->payload); + read_alternate_refs(cb->repo, path.buf, cb->fn, cb->payload); out: strbuf_release(&path); @@ -540,73 +216,33 @@ static int refs_from_alternate_cb(struct odb_source *alternate, void odb_for_each_alternate_ref(struct object_database *odb, odb_for_each_alternate_ref_fn cb, void *payload) { - struct alternate_refs_data data; - data.fn = cb; - data.payload = payload; + struct alternate_refs_data data = { + .fn = cb, + .payload = payload, + .repo = odb->repo, + }; odb_for_each_alternate(odb, refs_from_alternate_cb, &data); } int odb_for_each_alternate(struct object_database *odb, odb_for_each_alternate_fn cb, void *payload) { - struct odb_source *alternate; + struct odb_source_files *files = odb_source_files_downcast(odb->source); int r = 0; - for (alternate = odb->sources->next; alternate; alternate = alternate->next) { - r = cb(alternate, payload); + for (struct odb_files_dir *dir = files->dirs->next; dir; dir = dir->next) { + r = cb(dir, payload); if (r) break; } return r; } -static void odb_prepare_alternates(struct object_database *odb, - const char *alternate_db) -{ - extern struct strbuf gvfs_shared_cache_pathname; - struct strvec sources = STRVEC_INIT; - - parse_alternates(alternate_db, PATH_SEP, NULL, &sources); - odb_source_read_alternates(odb->sources, &sources); - - for (size_t i = 0; i < sources.nr; i++) - odb_add_alternate_recursively(odb, sources.v[i], 0); - - if (gvfs_shared_cache_pathname.len && - !gvfs_matched_shared_cache_to_alternate) { - /* - * There is no entry in .git/objects/info/alternates for - * the requested shared-cache directory. Therefore, the - * odb-list does not contain this directory. - * - * Force this directory into the odb-list as an in-memory - * alternate. Implicitly create the directory on disk, if - * necessary. - * - * See GIT_ALTERNATE_OBJECT_DIRECTORIES for another example - * of this kind of usage. - * - * Note: This has the net-effect of allowing Git to treat - * `gvfs.sharedCache` as an unofficial alternate. This - * usage should be discouraged for compatbility reasons - * with other tools in the overall Git ecosystem (that - * won't know about this trick). It would be much better - * for us to update .git/objects/info/alternates instead. - * The code here is considered a backstop. - */ - parse_alternates(gvfs_shared_cache_pathname.buf, '\n', NULL, &sources); - odb_source_read_alternates(odb->sources, &sources); - for (size_t i = 0; i < sources.nr; i++) - odb_add_alternate_recursively(odb, sources.v[i], 0); - - } - - strvec_clear(&sources); -} - int odb_has_alternates(struct object_database *odb) { - return !!odb->sources->next; + if (odb->source->type != ODB_SOURCE_FILES) + return 0; + return !!odb_source_files_downcast(odb->source)->dirs->next; } #define CAP_GET (1u<<0) @@ -752,36 +388,14 @@ void disable_obj_read_lock(void) pthread_mutex_destroy(&obj_read_mutex); } -static enum odb_read_status read_object_info_from_sources( - struct object_database *odb, const struct object_id *oid, - struct object_info *oi, enum object_info_flags flags, - struct strbuf *errmsg) -{ - struct odb_source *source; - enum odb_read_status ret = ODB_READ_NOT_FOUND; - - for (source = odb->sources; source; source = source->next) { - enum odb_read_status source_ret = odb_source_read_object_info( - source, oid, oi, flags, errmsg->len ? NULL : errmsg); - - if (!source_ret) - return ODB_READ_OK; - if (source_ret != ODB_READ_NOT_FOUND) - ret = source_ret; - } - - return ret; -} - static enum odb_read_status do_oid_object_info_extended(struct object_database *odb, const struct object_id *oid, struct object_info *oi, unsigned flags) { + extern int core_use_gvfs_helper; struct strbuf corrupt_err = STRBUF_INIT; const struct object_id *real = oid; enum odb_read_status ret; - int already_retried = 0; - bool corrupt = false; int tried_hook = 0; int tried_gvfs_helper = 0; @@ -795,96 +409,64 @@ static enum odb_read_status do_oid_object_info_extended(struct object_database * if (!odb_source_read_object_info(odb->inmemory_objects, oid, oi, flags, NULL)) return 0; - while (1) { - extern int core_use_gvfs_helper; - enum object_info_flags source_flags = flags; - - if (odb_has_alternates(odb)) { - ret = read_object_info_from_sources(odb, real, oi, - flags | OBJECT_INFO_SKIP_LOOSE, &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; - source_flags |= OBJECT_INFO_SKIP_PACKED; - } - - ret = read_object_info_from_sources(odb, real, oi, source_flags, - &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; - - if (core_use_gvfs_helper && !tried_gvfs_helper && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { - enum gh_client__created ghc; - - gh_client__get_immediate(real, &ghc); - tried_gvfs_helper = 1; + ret = odb_source_read_object_info(odb->source, real, oi, flags, + corrupt_err.len ? NULL : &corrupt_err); + if (!ret) + goto out; - /* - * Retry the lookup IIF `gvfs-helper` created one - * or more new packfiles or loose objects. - */ - if (ghc != GHC__CREATED__NOTHING) - continue; + if (core_use_gvfs_helper && !tried_gvfs_helper && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + enum gh_client__created ghc; - /* - * If `gvfs-helper` fails, we just want to return -1. - * But allow the other providers to have a shot at it. - * (At least until we have a chance to consolidate - * them.) - */ - } + gh_client__get_immediate(real, &ghc); + tried_gvfs_helper = 1; /* - * When the object hasn't been found we try a second read and - * tell the sources so. This may cause them to invalidate - * caches or reload on-disk state. + * Retry the lookup IIF `gvfs-helper` created one + * or more new packfiles or loose objects. */ - if (!(flags & OBJECT_INFO_QUICK)) { - ret = read_object_info_from_sources(odb, real, oi, - flags | OBJECT_INFO_SECOND_READ, - &corrupt_err); - if (!ret) - goto out; - if (ret != ODB_READ_NOT_FOUND) - corrupt = true; - if (gvfs_virtualize_objects(odb->repo) && !tried_hook && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { - // TODO Assert or at least trace2 if gvfs-helper - // TODO was tried and failed and then read-object-hook - // TODO is successful at getting this object. - tried_hook = 1; - // TODO BUG? Should 'oid' be 'real' ? - if (!read_object_process(odb->repo, oid)) - goto retry; - } - } + if (ghc != GHC__CREATED__NOTHING) + goto retry; - /* Check if it is a missing object */ - if (odb->repo->fetch_if_missing && repo_has_promisor_remote(odb->repo) && - !already_retried && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { - promisor_remote_get_direct(odb->repo, real, 1); - already_retried = 1; - continue; - } + /* + * If `gvfs-helper` fails, we just want to return -1. + * But allow the other providers to have a shot at it. + * (At least until we have a chance to consolidate + * them.) + */ + } - if (flags & OBJECT_INFO_DIE_IF_CORRUPT) { - if ((flags & OBJECT_INFO_LOOKUP_REPLACE) && !oideq(real, oid)) - die(_("replacement %s not found for %s"), - oid_to_hex(real), oid_to_hex(oid)); - if (corrupt) { - if (corrupt_err.len) - die("%s", corrupt_err.buf); - die(_("object %s is corrupt"), oid_to_hex(real)); - } + if (!(flags & OBJECT_INFO_QUICK) && + gvfs_virtualize_objects(odb->repo) && !tried_hook && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + // TODO Assert or at least trace2 if gvfs-helper + // TODO was tried and failed and then read-object-hook + // TODO is successful at getting this object. + tried_hook = 1; + // TODO BUG? Should 'oid' be 'real' ? + if (!read_object_process(odb->repo, oid)) + goto retry; + } + + if (ret == ODB_READ_NOT_FOUND && + odb->repo->fetch_if_missing && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT) && + repo_has_promisor_remote(odb->repo)) { + /* Try to fetch the missing object from a promisor remote. */ + promisor_remote_get_direct(odb->repo, real, 1); + ret = odb_source_read_object_info(odb->source, real, oi, flags, + corrupt_err.len ? NULL : &corrupt_err); + } + + if (ret < 0 && (flags & OBJECT_INFO_DIE_IF_CORRUPT)) { + if ((flags & OBJECT_INFO_LOOKUP_REPLACE) && !oideq(real, oid)) + die(_("replacement %s not found for %s"), + oid_to_hex(real), oid_to_hex(oid)); + if (ret != ODB_READ_NOT_FOUND) { + if (corrupt_err.len) + die("%s", corrupt_err.buf); + die(_("object %s is corrupt"), oid_to_hex(real)); } - - ret = corrupt ? ODB_READ_ERROR : ODB_READ_NOT_FOUND; - goto out; } out: @@ -1106,12 +688,8 @@ int odb_freshen_object(struct object_database *odb, const struct object_id *oid, int skip_virtualized_objects) { - struct odb_source *source; - for (source = odb->sources; source; source = source->next) - if (odb_source_freshen_object(source, oid, NULL, - skip_virtualized_objects)) - return 1; - return 0; + return odb_source_freshen_object(odb->source, oid, NULL, + skip_virtualized_objects); } int odb_for_each_object_ext(struct object_database *odb, @@ -1120,18 +698,7 @@ int odb_for_each_object_ext(struct object_database *odb, void *cb_data, const struct odb_for_each_object_options *opts) { - int ret; - - for (struct odb_source *source = odb->sources; source; source = source->next) { - if (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local) - continue; - - ret = odb_source_for_each_object(source, request, cb, cb_data, opts); - if (ret) - return ret; - } - - return 0; + return odb_source_for_each_object(odb->source, request, cb, cb_data, opts); } int odb_for_each_object(struct object_database *odb, @@ -1150,7 +717,6 @@ int odb_count_objects(struct object_database *odb, enum odb_count_objects_flags flags, unsigned long *out) { - struct odb_source *source; unsigned long count = 0; int ret; @@ -1159,15 +725,9 @@ int odb_count_objects(struct object_database *odb, return 0; } - for (source = odb->sources; source; source = source->next) { - unsigned long c; - - ret = odb_source_count_objects(source, flags, &c); - if (ret < 0) - goto out; - - count += c; - } + ret = odb_source_count_objects(odb->source, flags, &count); + if (ret < 0) + goto out; odb->object_count = count; odb->object_count_valid = 1; @@ -1238,13 +798,7 @@ int odb_find_abbrev_len(struct object_database *odb, goto out; } - for (struct odb_source *source = odb->sources; source; source = source->next) { - ret = odb_source_find_abbrev_len(source, oid, len, &len); - if (ret) - goto out; - } - - ret = 0; + ret = odb_source_find_abbrev_len(odb->source, oid, len, &len); *out = len; out: @@ -1300,7 +854,7 @@ int odb_write_object_ext(struct object_database *odb, compat_oid_p = &compat_oid; } - return odb_source_write_object(odb->sources, buf, len, type, + return odb_source_write_object(odb->source, buf, len, type, oid, compat_oid_p, NULL, flags); } @@ -1308,19 +862,19 @@ int odb_write_object_stream(struct object_database *odb, struct odb_stream *stream, struct object_id *oid) { - return odb_source_write_object_stream(odb->sources, stream, oid); + return odb_source_write_object_stream(odb->source, stream, oid); } int odb_optimize(struct object_database *odb, const struct odb_optimize_options *opts) { - return odb_source_optimize(odb->sources, opts); + return odb_source_optimize(odb->source, opts); } bool odb_optimize_required(struct object_database *odb, const struct odb_optimize_options *opts) { - return odb_source_optimize_required(odb->sources, opts); + return odb_source_optimize_required(odb->source, opts); } void odb_generate_pack_options_release(struct odb_generate_pack_options *opts) @@ -1334,9 +888,9 @@ int odb_generate_pack(struct object_database *odb, struct odb_pack_generator **out, const struct odb_generate_pack_options *opts) { - if (!odb->sources->generate_pack) + if (!odb->source->generate_pack) return error(_("primary object source does not support generating packfiles")); - return odb_source_generate_pack(odb->sources, out, opts); + return odb_source_generate_pack(odb->source, out, opts); } int odb_pack_generator_finish(struct odb_pack_generator *generator) @@ -1347,55 +901,29 @@ int odb_pack_generator_finish(struct odb_pack_generator *generator) struct object_database *odb_new(struct repository *repo, enum odb_new_flags flags) { - char *primary_source = NULL, *secondary_sources = NULL; struct object_database *o; CALLOC_ARRAY(o, 1); o->repo = repo; pthread_mutex_init(&o->replace_mutex, NULL); - hashmap_init(&o->source_by_path, odb_source_by_path_cmp, o, 0); - o->source_paths_icase = -1; - if (flags & ODB_NEW_HONOR_ENV) { - primary_source = xstrdup_or_null(getenv(DB_ENVIRONMENT)); - secondary_sources = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); - } - if (!primary_source) - primary_source = xstrfmt("%s/objects", repo->commondir); - - o->sources = odb_source_new(o, primary_source, true); - o->sources_tail = &o->sources->next; + o->source = odb_source_new(o, flags); o->inmemory_objects = &odb_source_inmemory_new(o)->base; - odb_prepare_alternates(o, secondary_sources); - - free(secondary_sources); - free(primary_source); return o; } void odb_close(struct object_database *o) { - struct odb_source *source; - for (source = o->sources; source; source = source->next) - odb_source_close(source); + odb_source_close(o->source); close_commit_graph(o); } static void odb_free_sources(struct object_database *o) { - while (o->sources) { - struct odb_source *next; - - next = o->sources->next; - odb_source_free(o->sources); - o->sources = next; - } - + odb_source_free(o->source); odb_source_free(o->inmemory_objects); o->inmemory_objects = NULL; - - hashmap_clear(&o->source_by_path); } void odb_free(struct object_database *o) @@ -1414,24 +942,12 @@ void odb_free(struct object_database *o) void odb_prepare(struct object_database *o, enum odb_prepare_flags flags) { - struct odb_source *source; - obj_read_lock(); - /* - * Reprepare alt odbs, in case the alternates file was modified - * during the course of this process. This only _adds_ odbs to - * the linked list, so existing odbs will continue to exist for - * the lifetime of the process. Consequently, we don't have to - * reprocess GIT_ALTERNATE_OBJECT_DIRECTORIES here. - */ - if (flags & ODB_PREPARE_FLUSH_CACHES) { - odb_prepare_alternates(o, NULL); + if (flags & ODB_PREPARE_FLUSH_CACHES) o->object_count_valid = 0; - } - for (source = o->sources; source; source = source->next) - odb_source_prepare(source, flags); + odb_source_prepare(o->source, flags); obj_read_unlock(); } @@ -1443,8 +959,5 @@ void odb_reprepare(struct object_database *o) int odb_fsck(struct object_database *odb, struct odb_fsck_options *options) { - int ret = 0; - for (struct odb_source *source = odb->sources; source; source = source->next) - ret |= odb_source_fsck(source, options); - return ret; + return odb_source_fsck(odb->source, options); } diff --git a/odb.h b/odb.h index f6deef02f6b2cf..503a9b598c7dcf 100644 --- a/odb.h +++ b/odb.h @@ -12,6 +12,7 @@ struct cached_object_entry; struct list_objects_filter_options; struct odb_source_inmemory; +struct odb_files_dir; struct packed_git; struct repository; struct strbuf; @@ -27,13 +28,16 @@ char *compute_alternate_path(const char *path, struct strbuf *err); /* * The object database encapsulates access to objects in a repository. It - * manages one or more sources that store the actual objects which are - * configured via alternates. + * manages the object source as well as auxiliary data structures required to + * manage objects. */ struct object_database { /* Repository that owns this database. */ struct repository *repo; + /* The source backing this object database. */ + struct odb_source *source; + /* * State of current object database transaction. Only one * transaction may be pending at a time. Is NULL when no transaction is @@ -41,27 +45,6 @@ struct object_database { */ struct odb_transaction *transaction; - /* - * Set of all object directories; the main directory is first (and - * cannot be NULL after initialization). Subsequent directories are - * alternates. - */ - struct odb_source *sources; - struct odb_source **sources_tail; - - /* - * Map of object database sources, keyed by their respective paths. - * This map is used to detect the case where the same source is - * registered multiple times. - */ - struct hashmap source_by_path; - - /* - * Whether source paths shall be compared case-insensitively, as - * determined by "core.ignoreCase". - */ - int source_paths_icase; - /* * Objects that should be substituted by other objects * (see git-replace(1)). @@ -225,39 +208,13 @@ struct odb_fsck_options { */ int odb_fsck(struct object_database *odb, struct odb_fsck_options *opts); -/* - * Find source by its object directory path. Returns a `NULL` pointer in case - * the source could not be found. - */ -struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir); - -/* Same as `odb_find_source()`, but dies in case the source doesn't exist. */ -struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir); - -/* - * Replace the current writable object directory with the specified temporary - * object directory and return the newly installed primary source. The former - * primary source is reported via `prev_source` when non-NULL. - */ -struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, - const char *dir, int will_destroy, - struct odb_source **prev_source); - -/* - * Restore the primary source that was previously replaced by - * `odb_set_temporary_primary_source()`. - */ -void odb_restore_primary_source(struct object_database *odb, - struct odb_source *restore_source, - const char *old_path); - /* * Iterate through all alternates of the database and execute the provided * callback function for each of them. Stop iterating once the callback * function returns a non-zero value, in which case the value is bubbled up * from the callback. */ -typedef int odb_for_each_alternate_fn(struct odb_source *, void *); +typedef int odb_for_each_alternate_fn(struct odb_files_dir *, void *); int odb_for_each_alternate(struct object_database *odb, odb_for_each_alternate_fn cb, void *payload); @@ -285,11 +242,11 @@ int odb_mkstemp(struct object_database *odb, int odb_has_alternates(struct object_database *odb); /* - * Add the directory to the on-disk alternates file; the new entry will also - * take effect in the current process. + * Add the directory to the in-memory list of alternate sources (along with any + * recursive alternates it points to), but do not modify the on-disk alternates + * file. */ -void odb_add_to_alternates_file(struct object_database *odb, - const char *dir); +void odb_add_to_alternates_memory(struct object_database *odb, const char *dir); /* * Read an object from the database. Returns the object data and assigns object @@ -433,30 +390,6 @@ enum object_info_flags { /* Die if object corruption (not just an object being missing) was detected. */ OBJECT_INFO_DIE_IF_CORRUPT = (1 << 3), - /* - * We have already tried reading the object, but it couldn't be found - * via any of the attached sources, and are now doing a second read. - * This second read asks the individual sources to also evaluate - * whether any on-disk state may have changed that may have caused the - * object to appear. - * - * This flag is for internal use, only. The second read only occurs - * when `OBJECT_INFO_QUICK` was not passed. - */ - OBJECT_INFO_SECOND_READ = (1 << 4), - - /* - * Only consult the packed object store of a source, skipping its loose - * object store (OBJECT_INFO_SKIP_LOOSE), or vice versa - * (OBJECT_INFO_SKIP_PACKED). These are used by - * odb_read_object_info_extended() to scan the packfiles of all sources - * before consulting any source's loose object store, so that an object - * that resides in an alternate's packfile is not preceded by a spurious - * loose-object lookup on an earlier source. - */ - OBJECT_INFO_SKIP_LOOSE = (1 << 5), - OBJECT_INFO_SKIP_PACKED = (1 << 6), - /* * This is meant for bulk prefetching of missing blobs in a partial * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK. @@ -870,11 +803,6 @@ int odb_generate_pack(struct object_database *odb, */ int odb_pack_generator_finish(struct odb_pack_generator *generator); -void parse_alternates(const char *string, - int sep, - const char *relative_base, - struct strvec *out); - int read_object_process(struct repository *r, const struct object_id *oid); #endif /* ODB_H */ diff --git a/odb/source-files.c b/odb/source-files.c index dc212ccb1e0d76..211500da4aa9aa 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -15,14 +15,40 @@ #include "packfile.h" #include "path.h" #include "promisor-remote.h" +#include "quote.h" #include "repack.h" #include "run-command.h" #include "strbuf.h" #include "string-list.h" +#include "strmap.h" #include "strvec.h" #include "tree.h" #include "write-or-die.h" +struct odb_files_dir *odb_files_dir_new(struct object_database *odb, + const char *path, bool local) +{ + struct odb_files_dir *dir; + + CALLOC_ARRAY(dir, 1); + dir->abspath = absolute_pathdup(path); + dir->local = local; + dir->loose = odb_source_loose_new(odb, path); + dir->packed = odb_source_packed_new(odb, path, local); + + return dir; +} + +void odb_files_dir_free(struct odb_files_dir *dir) +{ + if (!dir) + return; + odb_source_free(&dir->loose->base); + odb_source_free(&dir->packed->base); + free(dir->abspath); + free(dir); +} + static void odb_source_files_reparent(const char *old_cwd, const char *new_cwd, void *cb_data) @@ -30,6 +56,7 @@ static void odb_source_files_reparent(const char *old_cwd, struct odb_source_files *files = cb_data; char *path = reparent_relative_path(old_cwd, new_cwd, files->base.path); + free(files->base.path); files->base.path = path; } @@ -37,9 +64,16 @@ static void odb_source_files_reparent(const char *old_cwd, static void odb_source_files_free(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); + chdir_notify_unregister(odb_source_files_reparent, files); - odb_source_free(&files->loose->base); - odb_source_free(&files->packed->base); + + while (files->dirs) { + struct odb_files_dir *next = files->dirs->next; + odb_files_dir_free(files->dirs); + files->dirs = next; + } + hashmap_clear(&files->dirs_by_path); + odb_source_release(&files->base); free(files); } @@ -47,13 +81,21 @@ static void odb_source_files_free(struct odb_source *source) static void odb_source_files_close(struct odb_source *source) { struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_close(&files->loose->base); - odb_source_close(&files->packed->base); + + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + odb_source_close(&dir->loose->base); + odb_source_close(&dir->packed->base); + } } -static int odb_source_files_create_on_disk(struct odb_source *source) +static int odb_source_files_create_on_disk(struct odb_source *source, + const struct odb_create_on_disk_options *opts) { + struct lock_file alternates_lock = LOCK_INIT; struct strbuf path = STRBUF_INIT; + struct strset seen = STRSET_INIT; + struct strbuf line = STRBUF_INIT; + int ret; safe_create_dir(source->odb->repo, source->path, 1); @@ -64,16 +106,368 @@ static int odb_source_files_create_on_disk(struct odb_source *source) strbuf_addf(&path, "%s/info", source->path); safe_create_dir(source->odb->repo, path.buf, 1); + if (opts->alternates && opts->alternates->nr) { + FILE *alternates, *orig; + + strbuf_reset(&path); + strbuf_addf(&path, "%s/info/alternates", source->path); + + repo_hold_lock_file_for_update(source->odb->repo, &alternates_lock, + path.buf, LOCK_DIE_ON_ERROR); + + alternates = fdopen_lock_file(&alternates_lock, "w"); + if (!alternates) { + ret = error_errno(_("unable to fdopen alternates lockfile")); + goto out; + } + + /* + * The alternates file may already exist, e.g. when it has been + * seeded from a template directory. Read any preexisting + * entries so that we don't end up writing duplicates. + */ + orig = fopen(path.buf, "r"); + if (orig) { + while (strbuf_getline(&line, orig) != EOF) { + strset_add(&seen, line.buf); + fprintf(alternates, "%s\n", line.buf); + } + + if (ferror(orig)) { + ret = error_errno(_("unable to read alternates file")); + fclose(orig); + goto out; + } + + fclose(orig); + } else if (errno != ENOENT) { + ret = error_errno(_("unable to read alternates file")); + goto out; + } + + for (size_t i = 0; i < opts->alternates->nr; i++) { + const char *alternate = opts->alternates->v[i]; + if (!strset_add(&seen, alternate)) + continue; + fprintf(alternates, "%s\n", alternate); + } + + if (ferror(alternates)) { + ret = error_errno(_("unable to write alternates file")); + goto out; + } + + if (commit_lock_file(&alternates_lock)) { + ret = error_errno(_("unable to commit alternates file")); + goto out; + } + } + + /* Reprepare the object database to activate alternates. */ + odb_reprepare(source->odb); + + ret = 0; + +out: + rollback_lock_file(&alternates_lock); + strbuf_release(&line); strbuf_release(&path); + strset_clear(&seen); + return ret; +} + +/* + * NEEDSWORK: we're using "core.ignoreCase" to deduplicate alternates that + * _may_ be the same. This requires quite a bit of boilerplate for dubious + * benefit: + * + * - Duplicating alternates should really only lead to regressed performance. + * + * - We don't properly resolve symlinks or mointpoints, so we may still end + * up duplicating alternates. + * + * - The value may be lying, in which case we might deduplicate alternates + * that are in fact not mapping to the same directory. + * + * We should investigate whether we can remove this whole mechanism outright. + */ +static int odb_files_dir_paths_cmp(struct odb_source_files *files, + const char *a, const char *b) +{ + if (files->dirs_paths_icase < 0) { + int icase = 0; + repo_config_get_bool(files->base.odb->repo, "core.ignorecase", &icase); + files->dirs_paths_icase = icase; + } + + return files->dirs_paths_icase ? strcasecmp(a, b) : strcmp(a, b); +} + +static int odb_files_dir_by_path_cmp(const void *cb_data, + const struct hashmap_entry *entry, + const struct hashmap_entry *entry_or_key, + const void *keydata) +{ + struct odb_source_files *files = (struct odb_source_files *)cb_data; + const struct odb_files_dir *dir = container_of(entry, const struct odb_files_dir, by_path_entry); + const char *path = keydata; + + if (!path) + path = container_of(entry_or_key, const struct odb_files_dir, by_path_entry)->abspath; + + return odb_files_dir_paths_cmp(files, dir->abspath, path); +} + +static int gvfs_matched_shared_cache_to_alternate; + +/* + * Return non-zero iff the path is usable as an alternate object directory. + */ +static bool odb_files_dir_is_usable(struct odb_source_files *files, + const char *path) +{ + extern struct strbuf gvfs_shared_cache_pathname; + struct strbuf normalized_objdir = STRBUF_INIT; + struct hashmap_entry key; + bool usable = false; + + strbuf_realpath(&normalized_objdir, files->dirs->abspath, 1); + + if (!strcmp(path, gvfs_shared_cache_pathname.buf)) { + /* + * `gvfs.sharedCache` is the preferred alternate that we + * will use with `gvfs-helper.exe` to dynamically fetch + * missing objects. It is set during git_default_config(). + * + * Make sure the directory exists on disk before we let the + * stock code discredit it. + */ + struct strbuf buf_pack_foo = STRBUF_INIT; + enum scld_error scld; + + /* + * Append a bogus directory so that the leading-directory + * helper creates both the object directory and its pack/. + */ + strbuf_addf(&buf_pack_foo, "%s/pack/foo", path); + scld = safe_create_leading_directories(files->base.odb->repo, + buf_pack_foo.buf); + strbuf_release(&buf_pack_foo); + if (scld != SCLD_OK && scld != SCLD_EXISTS) { + error_errno(_("could not create shared-cache ODB '%s'"), + gvfs_shared_cache_pathname.buf); + strbuf_release(&gvfs_shared_cache_pathname); + goto out; + } + + gvfs_matched_shared_cache_to_alternate++; + } + + /* Detect cases where alternate disappeared */ + if (!is_directory(path)) { + warning(_("object directory %s does not exist; " + "check .git/objects/info/alternates"), + path); + goto out; + } + + /* + * Prevent the common mistake of listing the same + * thing twice, or object directory itself. + */ + if (!hashmap_get_size(&files->dirs_by_path)) { + assert(!files->dirs->next); + hashmap_entry_init(&files->dirs->by_path_entry, + strihash(files->dirs->abspath)); + hashmap_add(&files->dirs_by_path, &files->dirs->by_path_entry); + } + + if (!odb_files_dir_paths_cmp(files, path, normalized_objdir.buf)) + goto out; + + hashmap_entry_init(&key, strihash(path)); + if (hashmap_get(&files->dirs_by_path, &key, path)) + goto out; + + usable = true; + +out: + strbuf_release(&normalized_objdir); + return usable; +} + +static void parse_alternates(const char *string, + int sep, + const char *relative_base, + struct strvec *out) +{ + struct strbuf pathbuf = STRBUF_INIT; + struct strbuf buf = STRBUF_INIT; + + if (!string || !*string) + return; + + while (*string) { + const char *end; + + strbuf_reset(&buf); + strbuf_reset(&pathbuf); + + if (*string == '#') { + /* comment; consume up to next separator */ + end = strchrnul(string, sep); + } else if (*string == '"' && !unquote_c_style(&buf, string, &end)) { + /* + * quoted path; unquote_c_style has copied the + * data for us and set "end". Broken quoting (e.g., + * an entry that doesn't end with a quote) falls + * back to the unquoted case below. + */ + } else { + /* normal, unquoted path */ + end = strchrnul(string, sep); + strbuf_add(&buf, string, end - string); + } + + if (*end) + end++; + string = end; + + if (!buf.len) + continue; + + if (!is_absolute_path(buf.buf) && relative_base) { + strbuf_realpath(&pathbuf, relative_base, 1); + strbuf_addch(&pathbuf, '/'); + } + strbuf_addbuf(&pathbuf, &buf); + + strbuf_reset(&buf); + if (!strbuf_realpath(&buf, pathbuf.buf, 0)) { + warning(_("unable to normalize alternate object " + "path: %s"), pathbuf.buf); + continue; + } + + /* + * The trailing slash after the directory name is given by + * this function at the end. Remove duplicates. + */ + while (buf.len && buf.buf[buf.len - 1] == '/') + strbuf_setlen(&buf, buf.len - 1); + + strvec_push(out, buf.buf); + } + + strbuf_release(&pathbuf); + strbuf_release(&buf); +} + +static int read_alternates(const char *object_dir, struct strvec *out) +{ + struct strbuf buf = STRBUF_INIT; + char *path; + + path = xstrfmt("%s/info/alternates", object_dir); + if (strbuf_read_file(&buf, path, 1024) < 0) { + warn_on_fopen_errors(path); + free(path); + return 0; + } + parse_alternates(buf.buf, '\n', object_dir, out); + + strbuf_release(&buf); + free(path); return 0; } +static void odb_add_alternate_recursively(struct odb_source_files *files, + const char *path, + int depth) +{ + struct odb_files_dir *alternate; + struct strvec alternates = STRVEC_INIT; + + if (!odb_files_dir_is_usable(files, path)) + goto out; + + alternate = odb_files_dir_new(files->base.odb, path, false); + + /* add the alternate entry */ + *files->dirs_tail = alternate; + files->dirs_tail = &(alternate->next); + + hashmap_entry_init(&alternate->by_path_entry, strihash(alternate->abspath)); + if (hashmap_get(&files->dirs_by_path, &alternate->by_path_entry, + alternate->abspath)) + BUG("object directory must not yet exist"); + hashmap_add(&files->dirs_by_path, &alternate->by_path_entry); + + /* recursively add alternates */ + read_alternates(alternate->abspath, &alternates); + if (alternates.nr && depth + 1 > 5) { + error(_("%s: ignoring alternate object stores, nesting too deep"), + path); + } else { + for (size_t i = 0; i < alternates.nr; i++) + odb_add_alternate_recursively(files, alternates.v[i], depth + 1); + } + + out: + strvec_clear(&alternates); +} + +void odb_add_to_alternates_memory(struct object_database *odb, const char *dir) +{ + odb_add_alternate_recursively(odb_source_files_downcast(odb->source), + dir, 0); +} + +static void odb_prepare_alternates(struct odb_source_files *files, + const char *alternate_db) +{ + extern struct strbuf gvfs_shared_cache_pathname; + struct strvec alternates = STRVEC_INIT; + + parse_alternates(alternate_db, PATH_SEP, NULL, &alternates); + read_alternates(files->dirs->abspath, &alternates); + + for (size_t i = 0; i < alternates.nr; i++) + odb_add_alternate_recursively(files, alternates.v[i], 0); + + if (gvfs_shared_cache_pathname.len && + !gvfs_matched_shared_cache_to_alternate) { + /* + * Keep gvfs.sharedCache available as an in-memory alternate + * when the on-disk alternates file does not list it. + */ + parse_alternates(gvfs_shared_cache_pathname.buf, '\n', NULL, + &alternates); + for (size_t i = 0; i < alternates.nr; i++) + odb_add_alternate_recursively(files, alternates.v[i], 0); + } + + strvec_clear(&alternates); +} + static void odb_source_files_prepare(struct odb_source *source, enum odb_prepare_flags flags) { struct odb_source_files *files = odb_source_files_downcast(source); - odb_source_prepare(&files->loose->base, flags); - odb_source_prepare(&files->packed->base, flags); + + /* + * Reprepare alternates, in case the alternates file was modified + * during the course of this process. This only _adds_ directories to + * the linked list, so existing directories will continue to exist + * for the lifetime of the process. + */ + if (flags & ODB_PREPARE_FLUSH_CACHES) + odb_prepare_alternates(files, NULL); + + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + odb_source_prepare(&dir->loose->base, flags); + odb_source_prepare(&dir->packed->base, flags); + } } static enum odb_read_status odb_source_files_read_object_info(struct odb_source *source, @@ -83,34 +477,47 @@ static enum odb_read_status odb_source_files_read_object_info(struct odb_source struct strbuf *errmsg) { struct odb_source_files *files = odb_source_files_downcast(source); - enum odb_read_status ret_packed = ODB_READ_NOT_FOUND; - enum odb_read_status ret_loose = ODB_READ_NOT_FOUND; + enum odb_read_status status = ODB_READ_NOT_FOUND; + enum odb_read_status ret; + bool retry = false; + +retry: + /* + * Reading an object may fail even though the object exists, for + * example because it is corrupt. Report this failure to the caller in + * case none of the directories was able to read the object, and + * prefer the first such error in case multiple reads have failed. + */ + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (retry) + odb_source_prepare(&dir->packed->base, ODB_PREPARE_FLUSH_CACHES); - if (!(flags & OBJECT_INFO_SKIP_PACKED)) { - ret_packed = odb_source_read_object_info(&files->packed->base, - oid, oi, flags, errmsg); - if (!ret_packed) + ret = odb_source_read_object_info(&dir->packed->base, oid, oi, flags, + status == ODB_READ_NOT_FOUND ? errmsg : NULL); + if (!ret) return 0; + if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND) + status = ret; } - if (!(flags & OBJECT_INFO_SKIP_LOOSE)) { - ret_loose = odb_source_read_object_info(&files->loose->base, - oid, oi, flags, - ret_packed == ODB_READ_NOT_FOUND ? errmsg : NULL); - if (!ret_loose) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (retry) + break; + + ret = odb_source_read_object_info(&dir->loose->base, oid, oi, flags, + status == ODB_READ_NOT_FOUND ? errmsg : NULL); + if (!ret) return 0; + if (ret != ODB_READ_NOT_FOUND && status == ODB_READ_NOT_FOUND) + status = ret; } - /* - * Reading the packed object may have failed even though the object - * exists, for example because it is corrupt. Report this failure to - * the caller in case neither of the sources was able to read the - * object, and prefer the error of the packed source in case both - * reads have failed. - */ - if (ret_packed != ODB_READ_NOT_FOUND) - return ret_packed; - return ret_loose; + if (!retry && !(flags & OBJECT_INFO_QUICK)) { + retry = true; + goto retry; + } + + return status; } static int odb_source_files_read_object_stream(struct odb_stream **out, @@ -118,9 +525,14 @@ static int odb_source_files_read_object_stream(struct odb_stream **out, const struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_stream(out, &files->packed->base, oid) || - !odb_source_read_object_stream(out, &files->loose->base, oid)) - return 0; + + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) + if (!odb_source_read_object_stream(out, &dir->packed->base, oid)) + return 0; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) + if (!odb_source_read_object_stream(out, &dir->loose->base, oid)) + return 0; + return -1; } @@ -133,16 +545,21 @@ static int odb_source_files_for_each_object(struct odb_source *source, struct odb_source_files *files = odb_source_files_downcast(source); int ret; - if (!(opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) { - ret = odb_source_for_each_object(&files->loose->base, request, cb, cb_data, opts); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !dir->local) + continue; + + if (!(opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) { + ret = odb_source_for_each_object(&dir->loose->base, request, cb, cb_data, opts); + if (ret) + return ret; + } + + ret = odb_source_for_each_object(&dir->packed->base, request, cb, cb_data, opts); if (ret) return ret; } - ret = odb_source_for_each_object(&files->packed->base, request, cb, cb_data, opts); - if (ret) - return ret; - return 0; } @@ -151,21 +568,23 @@ static int odb_source_files_count_objects(struct odb_source *source, unsigned long *out) { struct odb_source_files *files = odb_source_files_downcast(source); - unsigned long count; + unsigned long count = 0; int ret; - ret = odb_source_count_objects(&files->packed->base, flags, &count); - if (ret < 0) - goto out; - - if (!(flags & ODB_COUNT_OBJECTS_APPROXIMATE)) { - unsigned long loose_count; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + unsigned long dir_count; - ret = odb_source_count_objects(&files->loose->base, flags, &loose_count); + ret = odb_source_count_objects(&dir->packed->base, flags, &dir_count); if (ret < 0) goto out; + count += dir_count; - count += loose_count; + if (!(flags & ODB_COUNT_OBJECTS_APPROXIMATE)) { + ret = odb_source_count_objects(&dir->loose->base, flags, &dir_count); + if (ret < 0) + goto out; + count += dir_count; + } } *out = count; @@ -182,15 +601,17 @@ static int odb_source_files_find_abbrev_len(struct odb_source *source, { struct odb_source_files *files = odb_source_files_downcast(source); unsigned len = min_len; - int ret; + int ret = 0; - ret = odb_source_find_abbrev_len(&files->packed->base, oid, len, &len); - if (ret < 0) - goto out; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + ret = odb_source_find_abbrev_len(&dir->packed->base, oid, len, &len); + if (ret < 0) + goto out; - ret = odb_source_find_abbrev_len(&files->loose->base, oid, len, &len); - if (ret < 0) - goto out; + ret = odb_source_find_abbrev_len(&dir->loose->base, oid, len, &len); + if (ret < 0) + goto out; + } *out = len; ret = 0; @@ -205,11 +626,14 @@ static int odb_source_files_freshen_object(struct odb_source *source, int skip_virtualized_objects) { struct odb_source_files *files = odb_source_files_downcast(source); - if (odb_source_freshen_object(&files->packed->base, oid, mtime, - skip_virtualized_objects) || - odb_source_freshen_object(&files->loose->base, oid, mtime, - skip_virtualized_objects)) - return 1; + + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) + if (odb_source_freshen_object(&dir->packed->base, oid, mtime, + skip_virtualized_objects) || + odb_source_freshen_object(&dir->loose->base, oid, mtime, + skip_virtualized_objects)) + return 1; + return 0; } @@ -222,7 +646,7 @@ static int odb_source_files_write_object(struct odb_source *source, enum odb_write_object_flags flags) { struct odb_source_files *files = odb_source_files_downcast(source); - return odb_source_write_object(&files->loose->base, buf, len, type, + return odb_source_write_object(&files->dirs->loose->base, buf, len, type, oid, compat_oid, mtime, flags); } @@ -231,7 +655,7 @@ static int odb_source_files_write_object_stream(struct odb_source *source, struct object_id *oid) { struct odb_source_files *files = odb_source_files_downcast(source); - return odb_source_write_object_stream(&files->loose->base, stream, oid); + return odb_source_write_object_stream(&files->dirs->loose->base, stream, oid); } static int odb_source_files_begin_transaction(struct odb_source *source, @@ -241,78 +665,6 @@ static int odb_source_files_begin_transaction(struct odb_source *source, return odb_transaction_files_begin(source, out, flags); } -static int odb_source_files_read_alternates(struct odb_source *source, - struct strvec *out) -{ - struct strbuf buf = STRBUF_INIT; - char *path; - - path = xstrfmt("%s/info/alternates", source->path); - if (strbuf_read_file(&buf, path, 1024) < 0) { - warn_on_fopen_errors(path); - free(path); - return 0; - } - parse_alternates(buf.buf, '\n', source->path, out); - - strbuf_release(&buf); - free(path); - return 0; -} - -static int odb_source_files_write_alternate(struct odb_source *source, - const char *alternate) -{ - struct lock_file lock = LOCK_INIT; - char *path = xstrfmt("%s/%s", source->path, "info/alternates"); - FILE *in, *out; - int found = 0; - int ret; - - repo_hold_lock_file_for_update(source->odb->repo, &lock, path, - LOCK_DIE_ON_ERROR); - out = fdopen_lock_file(&lock, "w"); - if (!out) { - ret = error_errno(_("unable to fdopen alternates lockfile")); - goto out; - } - - in = fopen(path, "r"); - if (in) { - struct strbuf line = STRBUF_INIT; - - while (strbuf_getline(&line, in) != EOF) { - if (!strcmp(alternate, line.buf)) { - found = 1; - break; - } - fprintf_or_die(out, "%s\n", line.buf); - } - - strbuf_release(&line); - fclose(in); - } else if (errno != ENOENT) { - ret = error_errno(_("unable to read alternates file")); - goto out; - } - - if (found) { - rollback_lock_file(&lock); - } else { - fprintf_or_die(out, "%s\n", alternate); - if (commit_lock_file(&lock)) { - ret = error_errno(_("unable to move new alternates file into place")); - goto out; - } - } - - ret = 0; - -out: - free(path); - return ret; -} - static int too_many_loose_objects(struct odb_source_files *files, int limit) { unsigned long loose_count; @@ -320,7 +672,7 @@ static int too_many_loose_objects(struct odb_source_files *files, int limit) if (limit <= 0) return 0; - if (odb_source_count_objects(&files->loose->base, ODB_COUNT_OBJECTS_APPROXIMATE, + if (odb_source_count_objects(&files->dirs->loose->base, ODB_COUNT_OBJECTS_APPROXIMATE, &loose_count) < 0) return 0; @@ -339,7 +691,7 @@ static struct packed_git *find_base_packs(struct odb_source_files *files, struct packfile_list_entry *e; struct packed_git *base = NULL; - for (e = packfile_store_get_packs(files->packed); e; e = e->next) { + for (e = packfile_store_get_packs(files->dirs->packed); e; e = e->next) { if (e->pack->is_cruft) continue; if (limit) { @@ -364,7 +716,7 @@ static int too_many_packs(struct odb_source_files *files, int gc_auto_pack_limit if (gc_auto_pack_limit <= 0) return 0; - for (e = packfile_store_get_packs(files->packed); e; e = e->next) { + for (e = packfile_store_get_packs(files->dirs->packed); e; e = e->next) { if (e->pack->pack_keep) continue; /* @@ -926,25 +1278,57 @@ static int odb_source_files_fsck(struct odb_source *source, struct odb_source_files *files = odb_source_files_downcast(source); int ret = 0; - if (!(opts->flags & ODB_FSCK_FULL) && !source->local) - return 0; + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!(opts->flags & ODB_FSCK_FULL) && !dir->local) + continue; - ret |= odb_source_fsck(&files->loose->base, opts); - ret |= odb_source_fsck(&files->packed->base, opts); + ret |= odb_source_fsck(&dir->loose->base, opts); + ret |= odb_source_fsck(&dir->packed->base, opts); + } return ret; } +struct odb_files_dir *odb_source_files_find_dir(struct object_database *odb, const char *obj_dir) +{ + struct odb_source_files *files = odb_source_files_downcast(odb->source); + char *obj_dir_real = real_pathdup(obj_dir, 1); + struct strbuf odb_path_real = STRBUF_INIT; + struct odb_files_dir *dir; + + for (dir = files->dirs; dir; dir = dir->next) { + strbuf_realpath(&odb_path_real, dir->abspath, 1); + if (!strcmp(obj_dir_real, odb_path_real.buf)) + break; + } + + free(obj_dir_real); + strbuf_release(&odb_path_real); + return dir; +} + struct odb_source_files *odb_source_files_new(struct object_database *odb, - const char *path, - bool local) + enum odb_new_flags flags) { struct odb_source_files *files; + char *object_dir = NULL; + char *alternates = NULL; + + if (flags & ODB_NEW_HONOR_ENV) { + object_dir = xstrdup_or_null(getenv(DB_ENVIRONMENT)); + alternates = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); + } + if (!object_dir) + object_dir = xstrfmt("%s/objects", odb->repo->commondir); CALLOC_ARRAY(files, 1); - odb_source_init(&files->base, odb, ODB_SOURCE_FILES, path, local); - files->loose = odb_source_loose_new(odb, path, local); - files->packed = odb_source_packed_new(odb, path, local); + odb_source_init(&files->base, odb, ODB_SOURCE_FILES, object_dir); + + hashmap_init(&files->dirs_by_path, odb_files_dir_by_path_cmp, files, 0); + files->dirs_paths_icase = -1; + + files->dirs = odb_files_dir_new(odb, object_dir, true); + files->dirs_tail = &files->dirs->next; files->base.free = odb_source_files_free; files->base.close = odb_source_files_close; @@ -960,8 +1344,6 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, files->base.write_object = odb_source_files_write_object; files->base.write_object_stream = odb_source_files_write_object_stream; files->base.begin_transaction = odb_source_files_begin_transaction; - files->base.read_alternates = odb_source_files_read_alternates; - files->base.write_alternate = odb_source_files_write_alternate; files->base.optimize = odb_source_files_optimize; files->base.optimize_required = odb_source_files_optimize_required; files->base.generate_pack = odb_source_files_generate_pack; @@ -971,8 +1353,12 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, * is not (yet) possible though because we access and assume relative * paths in the primary ODB source in some user-facing functionality. */ - if (!is_absolute_path(path)) + if (!is_absolute_path(object_dir)) chdir_notify_register(odb_source_files_reparent, files); + odb_prepare_alternates(files, alternates); + + free(object_dir); + free(alternates); return files; } diff --git a/odb/source-files.h b/odb/source-files.h index 9630b5f962e15f..837e8903044c2a 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -6,20 +6,71 @@ struct odb_source_loose; struct odb_source_packed; +/* + * A single object directory that encapsulates access to both the loose and + * packed backend. This can either be the primary or an alternate object + * directory. + */ +struct odb_files_dir { + /* Absolute path to the object directory. */ + char *abspath; + + /* List of alternate object directories. */ + struct odb_files_dir *next; + + /* + * Entry in the files source's map of directories, keyed by this + * directory's path. + */ + struct hashmap_entry by_path_entry; + + /* The two sources derived from this object directory. */ + struct odb_source_loose *loose; + struct odb_source_packed *packed; + + /* + * Whether this is the local object directory of the owning + * repository. Directories added via alternates are not local. + */ + bool local; +}; + +struct odb_files_dir *odb_files_dir_new(struct object_database *odb, + const char *path, bool local); +void odb_files_dir_free(struct odb_files_dir *dir); + /* * The files object database source uses a combination of loose objects and * packfiles. It is the default backend used by Git to store objects. */ struct odb_source_files { struct odb_source base; - struct odb_source_loose *loose; - struct odb_source_packed *packed; + + /* + * List of all object directories; the main directory is first (and + * cannot be NULL after initialization). Subsequent directories are + * alternates. + */ + struct odb_files_dir *dirs; + struct odb_files_dir **dirs_tail; + + /* + * Map of object directories, keyed by their respective paths. This + * map is used to detect the case where the same directory is + * registered multiple times. + */ + struct hashmap dirs_by_path; + + /* + * Whether directory paths shall be compared case-insensitively, as + * determined by "core.ignoreCase". + */ + int dirs_paths_icase; }; /* Allocate and initialize a new object source. */ struct odb_source_files *odb_source_files_new(struct object_database *odb, - const char *path, - bool local); + enum odb_new_flags flags); /* * Optimize the files object database source by repacking loose objects and @@ -49,4 +100,10 @@ static inline struct odb_source_files *odb_source_files_downcast(struct odb_sour return container_of(source, struct odb_source_files, base); } +/* + * Find "files" directory by its object directory path. Returns a `NULL` + * pointer in case the object directory could not be found. + */ +struct odb_files_dir *odb_source_files_find_dir(struct object_database *odb, const char *obj_dir); + #endif diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index ed5db345c957ae..b23da3f6788f12 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -161,8 +161,12 @@ static int odb_source_inmemory_for_each_object(struct odb_source *source, }; struct object_id null_oid = { 0 }; - if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY) || - (opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local)) + /* + * In-memory objects are neither promisor objects nor do they live in + * the local object directory, so there is nothing to iterate over. + */ + if (opts->flags & (ODB_FOR_EACH_OBJECT_PROMISOR_ONLY | + ODB_FOR_EACH_OBJECT_LOCAL_ONLY)) return 0; if (!inmemory->objects) return 0; @@ -323,18 +327,6 @@ static int odb_source_inmemory_begin_transaction(struct odb_source *source UNUSE return error("in-memory source does not support transactions"); } -static int odb_source_inmemory_read_alternates(struct odb_source *source UNUSED, - struct strvec *out UNUSED) -{ - return 0; -} - -static int odb_source_inmemory_write_alternate(struct odb_source *source UNUSED, - const char *alternate UNUSED) -{ - return error("in-memory source does not support alternates"); -} - static void odb_source_inmemory_close(struct odb_source *source UNUSED) { } @@ -382,7 +374,7 @@ struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb) struct odb_source_inmemory *source; CALLOC_ARRAY(source, 1); - odb_source_init(&source->base, odb, ODB_SOURCE_INMEMORY, "source", false); + odb_source_init(&source->base, odb, ODB_SOURCE_INMEMORY, "source"); source->base.free = odb_source_inmemory_free; source->base.close = odb_source_inmemory_close; @@ -397,8 +389,6 @@ struct odb_source_inmemory *odb_source_inmemory_new(struct object_database *odb) source->base.write_object_stream = odb_source_inmemory_write_object_stream; source->base.freshen_object = odb_source_inmemory_freshen_object; source->base.begin_transaction = odb_source_inmemory_begin_transaction; - source->base.read_alternates = odb_source_inmemory_read_alternates; - source->base.write_alternate = odb_source_inmemory_write_alternate; return source; } diff --git a/odb/source-loose.c b/odb/source-loose.c index 94d2372669aea0..a7ddd3c2d531f2 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -223,16 +223,6 @@ static enum odb_read_status odb_source_loose_read_object_info(struct odb_source { struct odb_source_loose *loose = odb_source_loose_downcast(source); static struct strbuf buf = STRBUF_INIT; - - /* - * The second read shouldn't cause new loose objects to show up, unless - * there was a race condition with a secondary process. We don't care - * about this case though, so we simply skip reading loose objects a - * second time. - */ - if (flags & OBJECT_INFO_SECOND_READ) - return ODB_READ_NOT_FOUND; - odb_loose_path(loose, &buf, oid); return read_object_info_from_path(loose, buf.buf, oid, oi, flags, errmsg); } @@ -475,8 +465,6 @@ static int odb_source_loose_for_each_object(struct odb_source *source, /* There are no loose promisor objects, so we can return immediately. */ if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) return 0; - if ((opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY) && !source->local) - return 0; if (opts->prefix) return oidtree_each(odb_source_loose_cache(loose, opts->prefix), @@ -610,7 +598,7 @@ static int odb_source_loose_freshen_object(struct odb_source *source, static void close_loose_object(struct odb_source_loose *loose, int fd, const char *filename) { - if (loose->base.will_destroy) + if (loose->will_destroy) goto out; if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT)) @@ -996,18 +984,6 @@ static int odb_source_loose_begin_transaction(struct odb_source *source UNUSED, return error("loose source does not support transactions"); } -static int odb_source_loose_read_alternates(struct odb_source *source UNUSED, - struct strvec *out UNUSED) -{ - return 0; -} - -static int odb_source_loose_write_alternate(struct odb_source *source UNUSED, - const char *alternate UNUSED) -{ - return error("loose source does not support alternates"); -} - void odb_source_loose_cache_add_new_oid(struct odb_source *source, const struct object_id *oid) { @@ -1016,7 +992,7 @@ void odb_source_loose_cache_add_new_oid(struct odb_source *source, switch (source->type) { case ODB_SOURCE_FILES: - loose = odb_source_files_downcast(source)->loose; + loose = odb_source_files_downcast(source)->dirs->loose; break; case ODB_SOURCE_LOOSE: loose = odb_source_loose_downcast(source); @@ -1162,13 +1138,12 @@ static int odb_source_loose_fsck(struct odb_source *source, } struct odb_source_loose *odb_source_loose_new(struct object_database *odb, - const char *path, - bool local) + const char *path) { struct odb_source_loose *loose; CALLOC_ARRAY(loose, 1); - odb_source_init(&loose->base, odb, ODB_SOURCE_LOOSE, path, local); + odb_source_init(&loose->base, odb, ODB_SOURCE_LOOSE, path); loose->base.free = odb_source_loose_free; loose->base.close = odb_source_loose_close; @@ -1183,8 +1158,6 @@ struct odb_source_loose *odb_source_loose_new(struct object_database *odb, loose->base.write_object = odb_source_loose_write_object; loose->base.write_object_stream = odb_source_loose_write_object_stream; loose->base.begin_transaction = odb_source_loose_begin_transaction; - loose->base.read_alternates = odb_source_loose_read_alternates; - loose->base.write_alternate = odb_source_loose_write_alternate; if (!is_absolute_path(loose->base.path)) chdir_notify_register(odb_source_loose_reparent, loose); diff --git a/odb/source-loose.h b/odb/source-loose.h index 3cf2e1f8f1d5e2..a6ae808662b39a 100644 --- a/odb/source-loose.h +++ b/odb/source-loose.h @@ -28,11 +28,13 @@ struct odb_source_loose { /* Map between object IDs for loose objects. */ struct loose_object_map *map; + + /* Whether this is a source that will never be committed to disk. */ + int will_destroy; }; struct odb_source_loose *odb_source_loose_new(struct object_database *odb, - const char *path, - bool local); + const char *path); /* * Cast the given object database source to the loose backend. This will cause diff --git a/odb/source-packed.c b/odb/source-packed.c index ef09fde1b91c9b..1d75bac41846e2 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -52,12 +52,11 @@ static int find_pack_entry(struct odb_source_packed *store, * packs, and repreparing the on-disk pack set does not reload the * borrowed, cached MIDX, so scan its packs directly for the survivor. * - * Do this only on the second read, by which point repreparing packs has - * already had a chance to find an object merely relocated into a new, - * uncovered pack; only a genuine hidden duplicate reaches here. + * Skip the scan for quick lookups, which are expected to accept false + * negatives in favor of speed. */ if (midx_result == MIDX_FILL_OWNER_UNAVAILABLE && - (flags & OBJECT_INFO_SECOND_READ)) { + !(flags & OBJECT_INFO_QUICK)) { struct multi_pack_index *m = store->midx; uint32_t i; @@ -86,14 +85,6 @@ static enum odb_read_status odb_source_packed_read_object_info(struct odb_source struct pack_entry e; int ret; - /* - * In case the first read didn't surface the object, we have to reload - * packfiles. This may cause us to discover new packfiles that have - * been added since the last time we have prepared the packfile store. - */ - if (flags & OBJECT_INFO_SECOND_READ) - odb_source_prepare(source, ODB_PREPARE_FLUSH_CACHES); - if (!find_pack_entry(packed, oid, &e, flags, &bad_pack)) { /* * The lookup may have failed because the object is known to be @@ -688,18 +679,6 @@ static int odb_source_packed_begin_transaction(struct odb_source *source UNUSED, return error("packed backend cannot begin transactions"); } -static int odb_source_packed_read_alternates(struct odb_source *source UNUSED, - struct strvec *out UNUSED) -{ - return 0; -} - -static int odb_source_packed_write_alternate(struct odb_source *source UNUSED, - const char *alternate UNUSED) -{ - return error("packed backend cannot write alternates"); -} - void (*report_garbage)(unsigned seen_bits, const char *path); static void report_helper(const struct string_list *list, @@ -762,7 +741,7 @@ static void prepare_pack(const char *full_name, size_t full_name_len, midx_contains_pack(data->source->midx, file_name))) { char *trimmed_path = xstrndup(full_name, full_name_len); packfile_store_load_pack(data->source, - trimmed_path, data->source->base.local); + trimmed_path, data->source->local); free(trimmed_path); } @@ -1010,8 +989,9 @@ struct odb_source_packed *odb_source_packed_new(struct object_database *odb, struct odb_source_packed *packed; CALLOC_ARRAY(packed, 1); - odb_source_init(&packed->base, odb, ODB_SOURCE_PACKED, path, local); + odb_source_init(&packed->base, odb, ODB_SOURCE_PACKED, path); strmap_init(&packed->packs_by_path); + packed->local = local; packed->base.free = odb_source_packed_free; packed->base.close = odb_source_packed_close; @@ -1026,8 +1006,6 @@ struct odb_source_packed *odb_source_packed_new(struct object_database *odb, packed->base.write_object = odb_source_packed_write_object; packed->base.write_object_stream = odb_source_packed_write_object_stream; packed->base.begin_transaction = odb_source_packed_begin_transaction; - packed->base.read_alternates = odb_source_packed_read_alternates; - packed->base.write_alternate = odb_source_packed_write_alternate; if (!is_absolute_path(path)) chdir_notify_register(odb_source_packed_reparent, packed); diff --git a/odb/source-packed.h b/odb/source-packed.h index a0f6b5096dcd0f..e8a7b08fbb9b13 100644 --- a/odb/source-packed.h +++ b/odb/source-packed.h @@ -41,6 +41,13 @@ struct odb_source_packed { */ struct strmap packs_by_path; + /* + * Whether this store is part of the local object directory of the + * owning repository. Packfiles loaded via this store inherit this + * flag via `pack_local`. Stores for alternates are not local. + */ + bool local; + /* * Whether packfiles have already been populated with this store's * packs. diff --git a/odb/source.c b/odb/source.c index 30188b806d524d..4155fc22dac1be 100644 --- a/odb/source.c +++ b/odb/source.c @@ -24,21 +24,18 @@ const char *odb_source_type_to_name(enum odb_source_type type) } struct odb_source *odb_source_new(struct object_database *odb, - const char *path, - bool local) + enum odb_new_flags flags) { - return &odb_source_files_new(odb, path, local)->base; + return &odb_source_files_new(odb, flags)->base; } void odb_source_init(struct odb_source *source, struct object_database *odb, enum odb_source_type type, - const char *path, - bool local) + const char *path) { source->odb = odb; source->type = type; - source->local = local; source->path = xstrdup(path); } diff --git a/odb/source.h b/odb/source.h index cafeef1f8f8367..d2bdcbc1598d2f 100644 --- a/odb/source.h +++ b/odb/source.h @@ -37,46 +37,27 @@ struct odb_stream; struct strbuf; struct strvec; +struct odb_create_on_disk_options { + /* + * Alternates that shall be written into the newly created object + * database. Whether or not this option can be handled is specific to + * the backend. + */ + const struct strvec *alternates; +}; + /* * The source is the part of the object database that stores the actual * objects. It thus encapsulates the logic to read and write the specific - * on-disk format. An object database can have multiple sources: - * - * - The primary source, which is typically located in "$GIT_DIR/objects". - * This is where new objects are usually written to. - * - * - Alternate sources, which are configured via "objects/info/alternates" or - * via the GIT_ALTERNATE_OBJECT_DIRECTORIES environment variable. These - * alternate sources are only used to read objects. + * on-disk format. */ struct odb_source { - struct odb_source *next; - - /* - * Entry in the object database's map of sources, keyed by this - * source's path. - */ - struct hashmap_entry by_path_entry; - /* Object database that owns this object source. */ struct object_database *odb; /* The type used by this source. */ enum odb_source_type type; - /* - * Figure out whether this is the local source of the owning - * repository, which would typically be its ".git/objects" directory. - * This local object directory is usually where objects would be - * written to. - */ - bool local; - - /* - * This object store is ephemeral, so there is no need to fsync. - */ - int will_destroy; - /* * Path to the source. If this is a relative path, it is relative to * the current working directory. @@ -107,7 +88,8 @@ struct odb_source { * This callback may be NULL in case the source does not need any * on-disk setup. */ - int (*create_on_disk)(struct odb_source *source); + int (*create_on_disk)(struct odb_source *source, + const struct odb_create_on_disk_options *opts); /* * This callback is expected to prepare the source so that it becomes @@ -129,17 +111,6 @@ struct odb_source { * - `OBJECT_INFO_QUICK` indicates it is fine to use caches without * re-verifying the data. * - * - `OBJECT_INFO_SECOND_READ` indicates that the initial object - * lookup has failed and that the object sources should check - * whether any of its on-disk state has changed that may have - * caused the object to appear. Sources are free to ignore the - * second read in case they know that the first read would have - * already surfaced the object without reloading any on-disk state. - * - * - `OBJECT_INFO_SKIP_LOOSE` and `OBJECT_INFO_SKIP_PACKED` tell the - * files backend not to consult its loose or packed source, - * respectively. - * * The callback is expected to return an `enum odb_read_status`. Please * refer to the individual values that can be returned. In case reading * the object has failed with a generic error and `errmsg` is non-NULL, @@ -273,32 +244,6 @@ struct odb_source { struct odb_transaction **out, enum odb_transaction_flags flags); - /* - * This callback is expected to read the list of alternate object - * database sources connected to it and write them into the `strvec`. - * - * The result is expected to be paths to the alternates. All paths must - * be resolved to absolute paths. - * - * The callback is expected to return 0 on success, a negative error - * code otherwise. - */ - int (*read_alternates)(struct odb_source *source, - struct strvec *out); - - /* - * This callback is expected to persist the singular alternate passed - * to it into its list of alternates. Any pre-existing alternates are - * expected to remain active. Subsequent calls to `read_alternates` are - * thus expected to yield the pre-existing list of alternates plus the - * newly added alternate appended to its end. - * - * The callback is expected to return 0 on success, a negative error - * code otherwise. - */ - int (*write_alternate)(struct odb_source *source, - const char *alternate); - /* * This callback is expected to optimize the object database source. * Returns 0 on success, a negative error code otherwise. @@ -344,18 +289,14 @@ struct odb_source { }; /* - * Allocate and initialize a new source for the given object database located - * at `path`. `local` indicates whether or not the source is the local and thus - * primary object source of the object database. + * Allocate and initialize a new source for the given object database. The path + * of the source is derived from repository paths. */ struct odb_source *odb_source_new(struct object_database *odb, - const char *path, - bool local); + enum odb_new_flags flags); /* * Initialize the source for the given object database located at `path`. - * `local` indicates whether or not the source is the local and thus primary - * object source of the object database. * * This function is only supposed to be called by specific object source * implementations. @@ -363,8 +304,7 @@ struct odb_source *odb_source_new(struct object_database *odb, void odb_source_init(struct odb_source *source, struct object_database *odb, enum odb_source_type type, - const char *path, - bool local); + const char *path); /* * Free the object database source, releasing all associated resources and @@ -394,11 +334,12 @@ static inline void odb_source_close(struct odb_source *source) * Create on-disk data structures that are required for this source to operate * correctly. Returns 0 on success, a negative error code otherwise. */ -static inline int odb_source_create_on_disk(struct odb_source *source) +static inline int odb_source_create_on_disk(struct odb_source *source, + const struct odb_create_on_disk_options *opts) { if (!source->create_on_disk) return 0; - return source->create_on_disk(source); + return source->create_on_disk(source, opts); } /* @@ -538,33 +479,6 @@ static inline int odb_source_write_object_stream(struct odb_source *source, return source->write_object_stream(source, stream, oid); } -/* - * Read the list of alternative object database sources from the given backend - * and populate the `strvec` with them. The listing is not recursive -- that - * is, if any of the yielded alternate sources has alternates itself, those - * will not be yielded as part of this function call. - * - * Return 0 on success, a negative error code otherwise. - */ -static inline int odb_source_read_alternates(struct odb_source *source, - struct strvec *out) -{ - return source->read_alternates(source, out); -} - -/* - * Write and persist a new alternate object database source for the given - * source. Any preexisting alternates are expected to stay valid, and the new - * alternate shall be appended to the end of the list. - * - * Returns 0 on success, a negative error code otherwise. - */ -static inline int odb_source_write_alternate(struct odb_source *source, - const char *alternate) -{ - return source->write_alternate(source, alternate); -} - /* * Create a new transaction that can be used to write objects into a temporary * staging area. The objects will only be persisted when the transaction is diff --git a/odb/streaming.c b/odb/streaming.c index 282c2412f1e0a2..d4af0f3466885a 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -184,12 +184,8 @@ static int istream_source(struct odb_stream **out, struct object_database *odb, const struct object_id *oid) { - struct odb_source *source; - - for (source = odb->sources; source; source = source->next) - if (!odb_source_read_object_stream(out, source, oid)) - return 0; - + if (!odb_source_read_object_stream(out, odb->source, oid)) + return 0; return open_istream_incore(out, odb, oid); } diff --git a/odb/transaction.c b/odb/transaction.c index f6f20088ec1fde..69824a3921d18f 100644 --- a/odb/transaction.c +++ b/odb/transaction.c @@ -12,7 +12,7 @@ int odb_transaction_begin(struct object_database *odb, if (odb->transaction) return error(_("object database transaction already pending")); - ret = odb_source_begin_transaction(odb->sources, out, flags); + ret = odb_source_begin_transaction(odb->source, out, flags); if (!ret) odb->transaction = *out; diff --git a/pack-bitmap.c b/pack-bitmap.c index 54cba883228c3f..e77cad81f85073 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -712,15 +712,13 @@ static int open_bitmap_for_source(struct odb_source_packed *source, static int open_bitmap(struct repository *r, struct bitmap_index *bitmap_git) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); bool found = false; assert(!bitmap_git->map); - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - - if (!open_bitmap_for_source(files->packed, bitmap_git)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!open_bitmap_for_source(dir->packed, bitmap_git)) found = true; /* diff --git a/packfile.c b/packfile.c index 702cf33622054a..35e03601f4d25f 100644 --- a/packfile.c +++ b/packfile.c @@ -273,14 +273,13 @@ static void scan_windows(struct packed_git *p, static int unuse_one_window(struct object_database *odb) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(odb->source); struct packfile_list_entry *e; struct packed_git *lru_p = NULL; struct pack_window *lru_w = NULL, *lru_l = NULL; - for (source = odb->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - for (e = files->packed->packs.head; e; e = e->next) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + for (e = dir->packed->packs.head; e; e = e->next) scan_windows(e->pack, &lru_p, &lru_w, &lru_l); } @@ -450,15 +449,14 @@ static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struc static int close_one_pack(struct repository *r) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); struct packfile_list_entry *e; struct packed_git *lru_p = NULL; struct pack_window *mru_w = NULL; int accept_windows_inuse = 1; - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - for (e = files->packed->packs.head; e; e = e->next) { + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + for (e = dir->packed->packs.head; e; e = e->next) { if (e->pack->pack_fd == -1) continue; find_lru_pack(e->pack, &lru_p, &mru_w, &accept_windows_inuse); @@ -1930,11 +1928,10 @@ struct packed_git **packfile_store_get_kept_pack_cache(struct odb_source_packed int has_object_pack(struct repository *r, const struct object_id *oid) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - if (!odb_source_read_object_info(&files->packed->base, oid, NULL, 0, NULL)) + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + if (!odb_source_read_object_info(&dir->packed->base, oid, NULL, 0, NULL)) return 1; } @@ -1944,14 +1941,13 @@ int has_object_pack(struct repository *r, const struct object_id *oid) int has_object_kept_pack(struct repository *r, const struct object_id *oid, unsigned flags) { - struct odb_source *source; + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); struct pack_entry e; - for (source = r->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { struct packed_git **cache; - cache = packfile_store_get_kept_pack_cache(files->packed, flags); + cache = packfile_store_get_kept_pack_cache(dir->packed, flags); for (; *cache; cache++) { struct packed_git *p = *cache; diff --git a/packfile.h b/packfile.h index b3c66273f6272c..b5e9740ef76c7e 100644 --- a/packfile.h +++ b/packfile.h @@ -69,20 +69,20 @@ void packfile_store_add_pack(struct odb_source_packed *store, struct packfile_list_entry *packfile_store_get_packs(struct odb_source_packed *store); struct repo_for_each_pack_data { - struct odb_source *source; + struct odb_files_dir *dir; struct packfile_list_entry *entry; }; static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct repository *repo) { struct repo_for_each_pack_data data = { 0 }; + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); - for (struct odb_source *source = repo->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); + for (struct odb_files_dir *dir = files->dirs; dir; dir = dir->next) { + struct packfile_list_entry *entry = packfile_store_get_packs(dir->packed); if (!entry) continue; - data.source = source; + data.dir = dir; data.entry = entry; break; } @@ -92,23 +92,22 @@ static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct static inline void repo_for_each_pack_data_next(struct repo_for_each_pack_data *data) { - struct odb_source *source; + struct odb_files_dir *dir; data->entry = data->entry->next; if (data->entry) return; - for (source = data->source->next; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); + for (dir = data->dir->next; dir; dir = dir->next) { + struct packfile_list_entry *entry = packfile_store_get_packs(dir->packed); if (!entry) continue; - data->source = source; + data->dir = dir; data->entry = entry; return; } - data->source = NULL; + data->dir = NULL; data->entry = NULL; } diff --git a/path.c b/path.c index 08da7fb4278758..80f7a8f39e2b40 100644 --- a/path.c +++ b/path.c @@ -396,7 +396,7 @@ static void adjust_git_path(struct repository *repo, strbuf_splice(buf, 0, buf->len, repo->index_file, strlen(repo->index_file)); else if (dir_prefix(base, "objects")) - replace_dir(buf, git_dir_len + 7, repo->objects->sources->path); + replace_dir(buf, git_dir_len + 7, repo->objects->source->path); else if (repo_settings_get_hooks_path(repo) && dir_prefix(base, "hooks")) replace_dir(buf, git_dir_len + 5, repo_settings_get_hooks_path(repo)); else if (repo->different_commondir) diff --git a/prune-packed.c b/prune-packed.c index d49dc11957c7a6..b6749b8d13233d 100644 --- a/prune-packed.c +++ b/prune-packed.c @@ -40,7 +40,7 @@ void prune_packed_objects(int opts) progress = start_delayed_progress(the_repository, _("Removing duplicate objects"), 256); - for_each_loose_file_in_source(the_repository->objects->sources, + for_each_loose_file_in_source(the_repository->objects->source, prune_object, NULL, prune_subdir, &opts); /* Ensure we show 100% before finishing progress */ diff --git a/repack-geometry.c b/repack-geometry.c index 15b34129506d2a..b541c34af5f094 100644 --- a/repack-geometry.c +++ b/repack-geometry.c @@ -33,7 +33,7 @@ void pack_geometry_init(struct pack_geometry *geometry, struct packed_git *p; struct strbuf buf = STRBUF_INIT; struct odb_source_files *files = odb_source_files_downcast(existing->source); - struct multi_pack_index *m = get_multi_pack_index(files->packed); + struct multi_pack_index *m = get_multi_pack_index(files->dirs->packed); repo_for_each_pack(existing->repo, p) { if (geometry->midx_layer_threshold_set && m && diff --git a/repack-midx.c b/repack-midx.c index 64c7f8d0f42372..25140569bd9d2b 100644 --- a/repack-midx.c +++ b/repack-midx.c @@ -564,7 +564,7 @@ static void repack_make_midx_append_plan(struct repack_write_midx_opts *opts, size_t steps_nr = 0, steps_alloc = 0; odb_reprepare(opts->existing->repo->objects); - m = get_multi_pack_index(files->packed); + m = get_multi_pack_index(files->dirs->packed); if (opts->names->nr) { struct strbuf buf = STRBUF_INIT; @@ -620,7 +620,7 @@ static int repack_make_midx_compaction_plan(struct repack_write_midx_opts *opts, opts->existing->repo); odb_reprepare(opts->existing->repo->objects); - m = get_multi_pack_index(files->packed); + m = get_multi_pack_index(files->dirs->packed); for (i = 0; m && i < m->num_packs + m->num_packs_in_base; i++) { if (prepare_midx_pack(m, i)) { @@ -949,7 +949,7 @@ static int write_midx_incremental(struct repack_write_midx_opts *opts) size_t i; int ret = 0; - get_midx_chain_filename(files->packed, &lock_name); + get_midx_chain_filename(files->dirs->packed, &lock_name); if (safe_create_leading_directories(opts->existing->repo, lock_name.buf)) die_errno(_("unable to create leading directories of %s"), diff --git a/repack.c b/repack.c index d2aa58e13484fd..4c6d1f507d7135 100644 --- a/repack.c +++ b/repack.c @@ -59,10 +59,10 @@ void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, bool wrote_incremental_midx) { struct strbuf buf = STRBUF_INIT; - struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources); - struct multi_pack_index *m = get_multi_pack_index(files->packed); + struct odb_source_files *files = odb_source_files_downcast(repo->objects->source); + struct multi_pack_index *m = get_multi_pack_index(files->dirs->packed); strbuf_addf(&buf, "%s.pack", base_name); - if (m && files->base.local && midx_contains_pack(m, buf.buf)) { + if (m && files->dirs->local && midx_contains_pack(m, buf.buf)) { clear_midx_file(repo); if (!wrote_incremental_midx) clear_incremental_midx_files(repo, NULL); @@ -158,7 +158,7 @@ void existing_packs_collect(struct existing_packs *existing, string_list_append(&existing->non_kept_packs, buf.buf); } - existing->source = existing->repo->objects->sources; + existing->source = existing->repo->objects->source; string_list_sort(&existing->kept_packs); string_list_sort(&existing->non_kept_packs); diff --git a/repository.c b/repository.c index bb88f6b5fc6daf..fdb60fd96629bc 100644 --- a/repository.c +++ b/repository.c @@ -126,9 +126,9 @@ const char *repo_get_common_dir(struct repository *repo) const char *repo_get_object_directory(struct repository *repo) { - if (!repo->objects->sources) + if (!repo->objects->source) BUG("repository hasn't been set up"); - return repo->objects->sources->path; + return repo->objects->source->path; } const char *repo_get_index_file(struct repository *repo) diff --git a/setup.c b/setup.c index a8ac9495b32cff..f5bba2e084d0a8 100644 --- a/setup.c +++ b/setup.c @@ -2711,8 +2711,13 @@ static int create_default_files(struct repository *repo, return reinit; } -static void create_object_database(struct repository *repo) +void create_object_database(struct repository *repo, + const struct strvec *alternates) { + struct odb_create_on_disk_options opts = { + .alternates = alternates, + }; + /* * Create the "objects" directory in the common directory. This is done * so that the repository can be discovered regardless of the backend @@ -2732,7 +2737,7 @@ static void create_object_database(struct repository *repo) repo->objects = odb_new(repo, ODB_NEW_HONOR_ENV); - if (odb_source_create_on_disk(repo->objects->sources) < 0) + if (odb_source_create_on_disk(repo->objects->source, &opts) < 0) die(_("failed creating object database")); } @@ -2886,17 +2891,17 @@ static void repository_format_configure(struct repository_format *repo_fmt, } } -int init_db(struct repository *repo, - const char *git_dir, - const char *real_git_dir, - const char *worktree, - const char *template_dir, int hash, - enum ref_storage_format ref_storage_format, - const char *initial_branch, - int init_shared_repository, unsigned int flags) +void create_repository(struct repository *repo, + const char *git_dir, + const char *real_git_dir, + const char *worktree, + const char *template_dir, + int hash, + enum ref_storage_format ref_storage_format, + int init_shared_repository, + int *reinit_ok) { - int reinit; - int exist_ok = flags & INIT_DB_EXIST_OK; + int reinit_ignored; char *original_git_dir = real_pathdup(git_dir, 1); struct repository_format repo_fmt = REPOSITORY_FORMAT_INIT; struct strbuf err = STRBUF_INIT; @@ -2904,10 +2909,10 @@ int init_db(struct repository *repo, if (real_git_dir) { struct stat st; - if (!exist_ok && !stat(git_dir, &st)) + if (!reinit_ok && !stat(git_dir, &st)) die(_("%s already exists"), git_dir); - if (!exist_ok && !stat(real_git_dir, &st)) + if (!reinit_ok && !stat(real_git_dir, &st)) die(_("%s already exists"), real_git_dir); apply_and_export_relative_gitdir(repo, real_git_dir, 1); @@ -2941,8 +2946,10 @@ int init_db(struct repository *repo, safe_create_dir(repo, git_dir, 0); - reinit = create_default_files(repo, template_dir, original_git_dir, - &repo_fmt, init_shared_repository); + if (!reinit_ok) + reinit_ok = &reinit_ignored; + *reinit_ok = create_default_files(repo, template_dir, original_git_dir, + &repo_fmt, init_shared_repository); if (repo_settings_get_shared_repository(repo)) { char buf[10]; @@ -2965,29 +2972,7 @@ int init_db(struct repository *repo, repo_config_set(repo, "receive.denyNonFastforwards", "true"); } - if (!(flags & INIT_DB_SKIP_REFDB)) - create_reference_database(repo, initial_branch, flags & INIT_DB_QUIET); - create_object_database(repo); - - startup_info->have_repository = 1; - - if (!(flags & INIT_DB_QUIET)) { - int len = strlen(git_dir); - - if (reinit) - printf(repo_settings_get_shared_repository(repo) - ? _("Reinitialized existing shared Git repository in %s%s\n") - : _("Reinitialized existing Git repository in %s%s\n"), - git_dir, len && git_dir[len-1] != '/' ? "/" : ""); - else - printf(repo_settings_get_shared_repository(repo) - ? _("Initialized empty shared Git repository in %s%s\n") - : _("Initialized empty Git repository in %s%s\n"), - git_dir, len && git_dir[len-1] != '/' ? "/" : ""); - } - clear_repository_format(&repo_fmt); strbuf_release(&err); free(original_git_dir); - return 0; } diff --git a/setup.h b/setup.h index 7394473e954df1..9d0c5182119248 100644 --- a/setup.h +++ b/setup.h @@ -257,24 +257,47 @@ int apply_repository_format(struct repository *repo, const char *get_template_dir(const char *option_template); -#define INIT_DB_QUIET (1 << 0) -#define INIT_DB_EXIST_OK (1 << 1) -#define INIT_DB_SKIP_REFDB (1 << 2) - -int init_db(struct repository *repo, - const char *git_dir, - const char *real_git_dir, - const char *worktree, - const char *template_dir, int hash_algo, - enum ref_storage_format ref_storage_format, - const char *initial_branch, int init_shared_repository, - unsigned int flags); +/* + * Create the repository by creating the necessary directory structures, + * setting up the configuration and configuring the repository's format. If + * `template_dir` is set, copy over templates from that directory. Furthermore, + * if and only if `reinit_ok` is a non-NULL pointer, then the function may + * reinitialize a preexisting repository. In that case, the pointer will be set + * to `1` in case the repo was reinitialized and `0` if it didn't exist yet. + * + * Note that this function does not create the reference and object databases. + */ +void create_repository(struct repository *repo, + const char *git_dir, + const char *real_git_dir, + const char *worktree, + const char *template_dir, + int hash_algo, + enum ref_storage_format ref_storage_format, + int init_shared_repository, + int *reinit_ok); + void initialize_repository_version(struct repository *repo, int hash_algo, enum ref_storage_format ref_storage_format, int reinit); + +/* + * Create the reference database for the repository. The repository and its ref + * storage format must have already been configured properly before calling + * this function. When set, `initial_branch` overrides the default branch that + * HEAD will point to. + */ void create_reference_database(struct repository *repo, const char *initial_branch, int quiet); +/* + * Create the object database for the repository. The repository must have + * already been configured properly before calling this function. When set, + * `alternates` is the list of alternates that should be written into the + * object database. + */ +void create_object_database(struct repository *repo, const struct strvec *alternates); + /* * NOTE NOTE NOTE!! * diff --git a/t/helper/test-read-graph.c b/t/helper/test-read-graph.c index 9f07b9c25a66e3..62e23840522270 100644 --- a/t/helper/test-read-graph.c +++ b/t/helper/test-read-graph.c @@ -4,6 +4,7 @@ #include "commit-graph.h" #include "repository.h" #include "odb.h" +#include "odb/source.h" #include "bloom.h" #include "setup.h" @@ -77,11 +78,11 @@ int cmd__read_graph(int argc, const char **argv) int ret = 0; setup_git_directory(the_repository); - source = the_repository->objects->sources; + source = the_repository->objects->source; prepare_repo_settings(the_repository); - graph = read_commit_graph_one(source); + graph = read_commit_graph_one(the_repository, source->path); if (!graph) { ret = 1; goto done; diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index 83b07c6236b167..88a79fdfc8d0d5 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -16,13 +16,13 @@ static struct multi_pack_index *setup_midx(const char *object_dir, struct odb_source_packed **out) { struct odb_source_packed *packed; - struct odb_source *source; + struct odb_files_dir *dir; setup_git_directory(the_repository); - source = odb_find_source(the_repository->objects, object_dir); - if (source) { - packed = odb_source_files_downcast(source)->packed; + dir = odb_source_files_find_dir(the_repository->objects, object_dir); + if (dir) { + packed = dir->packed; } else { packed = odb_source_packed_new(the_repository->objects, object_dir, false); diff --git a/t/t4216-log-bloom.sh b/t/t4216-log-bloom.sh index ad2686669d4b30..f57a3d66215c3f 100755 --- a/t/t4216-log-bloom.sh +++ b/t/t4216-log-bloom.sh @@ -755,7 +755,7 @@ test_expect_success PERL_TEST_HELPERS 'Bloom reader notices too-small data chunk test_expect_success PERL_TEST_HELPERS 'Bloom reader notices out-of-bounds filter offsets' ' check_corrupt_graph BIDX 12 FFFFFFFF && # use grep to avoid depending on exact chunk size - test_grep "warning: ignoring out-of-range offset (4294967295) for changed-path filter at pos 3 of .git/objects/info/commit-graph" err + test_grep "warning: ignoring out-of-range offset (4294967295) for changed-path filter at pos 3 of $(pwd)/.git/objects/info/commit-graph" err ' test_expect_success PERL_TEST_HELPERS 'Bloom reader notices too-small index chunk' ' @@ -773,7 +773,7 @@ test_expect_success PERL_TEST_HELPERS 'Bloom reader notices out-of-order index o # actually reading from the bogus offsets anyway. corrupt_graph BIDX 4 0000000c00000005 && echo "warning: ignoring decreasing changed-path index offsets" \ - "(12 > 5) for positions 1 and 2 of .git/objects/info/commit-graph" >expect.err && + "(12 > 5) for positions 1 and 2 of $(pwd)/.git/objects/info/commit-graph" >expect.err && git -c core.commitGraph=false log -- A/B/file2 >expect.out && git -c core.commitGraph=true log -- A/B/file2 >out 2>err && test_cmp expect.out out && diff --git a/t/t5604-clone-reference.sh b/t/t5604-clone-reference.sh index 39a0c318dfb2d4..9e4b98fdb82597 100755 --- a/t/t5604-clone-reference.sh +++ b/t/t5604-clone-reference.sh @@ -383,4 +383,29 @@ test_expect_success 'dissociate from repo with commit graph' ' git clone --no-local --reference graph.git --dissociate orig clone ' +test_expect_success 'local clone from linked worktree carries over alternates' ' + rm -fr base derived derived-wt dst expect && + git init base && + test_commit -C base one && + git clone --shared base derived && + git -C derived worktree add ../derived-wt && + git clone derived-wt dst && + echo "$(pwd)/base/.git/objects" >expect && + test_cmp expect dst/.git/objects/info/alternates && + git -C dst fsck +' + +test_expect_success 'local clone from linked worktree resolves relative alternates' ' + rm -fr base derived derived-wt dst expect && + git init base && + test_commit -C base one && + git clone --shared base derived && + echo "../../../base/.git/objects" >derived/.git/objects/info/alternates && + git -C derived worktree add ../derived-wt && + git clone derived-wt dst && + echo "$(pwd)/base/.git/objects" >expect && + test_cmp expect dst/.git/objects/info/alternates && + git -C dst fsck +' + test_done diff --git a/t/t5615-alternate-env.sh b/t/t5615-alternate-env.sh index d82dba65d94b14..4b801b93433239 100755 --- a/t/t5615-alternate-env.sh +++ b/t/t5615-alternate-env.sh @@ -122,7 +122,7 @@ test_expect_success 'packs across sources are checked before loose objects' ' # Main repo: write B as a loose object, before any alternate is active. git init main && git -C main hash-object -w --stdin /dev/null && - test -e "main/.git/objects/$(test_oid_to_path "$B")" && + test_path_is_file "main/.git/objects/$(test_oid_to_path "$B")" && # With the alternate active, B must resolve to the alternate packfile # (deltabase O), not to the main store loose copy (deltabase zero oid). diff --git a/tmp-objdir.c b/tmp-objdir.c index deaaf6ba2ea315..3debb60270f7b0 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -12,13 +12,16 @@ #include "quote.h" #include "odb.h" #include "odb/source.h" +#include "odb/source-files.h" +#include "odb/source-loose.h" #include "repository.h" struct tmp_objdir { struct repository *repo; struct strbuf path; struct strvec env; - struct odb_source *prev_source; + struct odb_files_dir *temp_dir; + struct odb_files_dir *orig_dir; int will_destroy; }; @@ -51,6 +54,27 @@ static void tmp_objdir_reparent(const char *old_cwd, free(path); } +/* + * Restore the primary source that was previously replaced by + * `tmp_objdir_create()`. + */ +static void tmp_objdir_restore_source(struct tmp_objdir *t) +{ + struct odb_source_files *files = odb_source_files_downcast(t->repo->objects->source); + struct odb_files_dir *cur_dir = files->dirs; + + if (t->temp_dir != files->dirs) + BUG("expected %s as primary object store; found %s", + t->temp_dir->abspath, cur_dir->abspath); + + if (t->temp_dir->next != t->orig_dir) + BUG("we expect the old primary object store to be the first alternate"); + + t->repo->disable_ref_updates = false; + files->dirs = t->orig_dir; + odb_files_dir_free(cur_dir); +} + int tmp_objdir_destroy(struct tmp_objdir *t) { int err; @@ -61,8 +85,8 @@ int tmp_objdir_destroy(struct tmp_objdir *t) if (t == the_tmp_objdir) the_tmp_objdir = NULL; - if (t->prev_source) - odb_restore_primary_source(t->repo->objects, t->prev_source, t->path.buf); + if (t->orig_dir) + tmp_objdir_restore_source(t); err = remove_dir_recursively(&t->path, 0); @@ -132,8 +156,10 @@ static int setup_tmp_objdir(const char *root) } struct tmp_objdir *tmp_objdir_create(struct repository *r, - const char *prefix) + const char *prefix, + int will_destroy) { + struct odb_source_files *files = odb_source_files_downcast(r->objects->source); static int installed_handlers; struct tmp_objdir *t; @@ -142,6 +168,7 @@ struct tmp_objdir *tmp_objdir_create(struct repository *r, t = xcalloc(1, sizeof(*t)); t->repo = r; + t->will_destroy = will_destroy; strbuf_init(&t->path, 0); strvec_init(&t->env); @@ -179,6 +206,19 @@ struct tmp_objdir *tmp_objdir_create(struct repository *r, env_replace(&t->env, GIT_QUARANTINE_ENVIRONMENT, absolute_path(t->path.buf)); + /* + * Make a new primary source and link the old primary source in as an + * alternate. Disable ref updates while a temporary source is active, + * since the objects in the database may roll back. + */ + t->temp_dir = odb_files_dir_new(t->repo->objects, t->path.buf, false); + t->temp_dir->loose->will_destroy = will_destroy; + t->temp_dir->next = files->dirs; + + t->orig_dir = files->dirs; + files->dirs = t->temp_dir; + t->repo->disable_ref_updates = true; + return t; } @@ -295,11 +335,11 @@ int tmp_objdir_migrate(struct tmp_objdir *t) if (!t) return 0; - if (t->prev_source) { - if (t->repo->objects->sources->will_destroy) + if (t->orig_dir) { + if (t->will_destroy) BUG("migrating an ODB that was marked for destruction"); - odb_restore_primary_source(t->repo->objects, t->prev_source, t->path.buf); - t->prev_source = NULL; + tmp_objdir_restore_source(t); + t->orig_dir = NULL; } strbuf_addbuf(&src, &t->path); @@ -320,14 +360,3 @@ const char **tmp_objdir_env(const struct tmp_objdir *t) return NULL; return t->env.v; } - -struct odb_source *tmp_objdir_replace_primary_odb(struct tmp_objdir *t, - int will_destroy) -{ - if (t->prev_source) - BUG("the primary object database is already replaced"); - t->will_destroy = will_destroy; - - return odb_set_temporary_primary_source(t->repo->objects, t->path.buf, - will_destroy, &t->prev_source); -} diff --git a/tmp-objdir.h b/tmp-objdir.h index 05f0d08d1065c4..a301b01d84cb84 100644 --- a/tmp-objdir.h +++ b/tmp-objdir.h @@ -11,7 +11,7 @@ * Example: * * struct child_process child = CHILD_PROCESS_INIT; - * struct tmp_objdir *t = tmp_objdir_create(repo, "incoming"); + * struct tmp_objdir *t = tmp_objdir_create(repo, "incoming", 0); * strvec_push(&child.args, cmd); * strvec_pushv(&child.env, tmp_objdir_env(t)); * if (!run_command(&child)) && !tmp_objdir_migrate(t)) @@ -25,10 +25,11 @@ struct repository; struct tmp_objdir; /* - * Create a new temporary object directory with the specified prefix; - * returns NULL on failure. + * Create a new temporary object directory with the specified prefix and + * install the directory as the primary write target; returns NULL on failure. */ -struct tmp_objdir *tmp_objdir_create(struct repository *r, const char *prefix); +struct tmp_objdir *tmp_objdir_create(struct repository *r, const char *prefix, + int will_destroy); /* * Return a list of environment strings, suitable for use with @@ -55,13 +56,4 @@ int tmp_objdir_destroy(struct tmp_objdir *); */ void tmp_objdir_discard_objects(struct tmp_objdir *); -/* - * Replaces the writable object store in the current process with the temporary - * object directory and makes the former main object store an alternate. - * If will_destroy is nonzero, the object directory may not be migrated. Returns - * the newly installed primary source. - */ -struct odb_source *tmp_objdir_replace_primary_odb(struct tmp_objdir *, - int will_destroy); - #endif /* TMP_OBJDIR_H */