diff --git a/builtin/backfill.c b/builtin/backfill.c index 80056abe4730ae6c81da5c6ad9e432be37f6dccc..e80fc1b694df616115ac4b6a538ed22a4e7a772b 100644 --- a/builtin/backfill.c +++ b/builtin/backfill.c @@ -53,7 +53,7 @@ static void download_batch(struct backfill_context *ctx) * We likely have a new packfile. Add it to the packed list to * avoid possible duplicate downloads of the same objects. */ - reprepare_packed_git(ctx->repo); + odb_reprepare(ctx->repo->objects); } static int fill_missing_blobs(const char *path UNUSED, diff --git a/builtin/cat-file.c b/builtin/cat-file.c index fce0b06451c5cdff3036797ab506dc30498bd68a..ee6715fa523ce6094dd65a83b2f8b413f3854a87 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -852,9 +852,10 @@ static void batch_each_object(struct batch_options *opt, if (bitmap && !for_each_bitmapped_object(bitmap, &opt->objects_filter, batch_one_object_bitmapped, &payload)) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *pack; - for (pack = get_all_packs(the_repository); pack; pack = pack->next) { + for (pack = packfile_store_get_all_packs(packs); pack; pack = pack->next) { if (bitmap_index_contains_pack(bitmap, pack) || open_pack_index(pack)) continue; diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 6656187f90d1e4c3edcf1359c5fb75258c4818be..fe3ebaadadadb6966d8da799568b2ce8a6c9c7af 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -102,7 +102,7 @@ static int graph_verify(int argc, const char **argv, const char *prefix, if (opts.progress) flags |= COMMIT_GRAPH_WRITE_PROGRESS; - source = odb_find_source(the_repository->objects, opts.obj_dir); + source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); graph_name = get_commit_graph_filename(source); chain_name = get_commit_graph_chain_filename(source); if (open_commit_graph(graph_name, &fd, &st)) @@ -291,7 +291,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags |= COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - source = odb_find_source(the_repository->objects, opts.obj_dir); + source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); if (opts.reachable) { if (write_commit_graph_reachable(source, flags, &write_opts)) diff --git a/builtin/count-objects.c b/builtin/count-objects.c index a61d3b46aac6273c3a7ad2c9ac5e5855c3c8bd22..f2f407c2a7818334d9fce189228a76aac0c245f7 100644 --- a/builtin/count-objects.c +++ b/builtin/count-objects.c @@ -122,6 +122,7 @@ int cmd_count_objects(int argc, count_loose, count_cruft, NULL, NULL); if (verbose) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; unsigned long num_pack = 0; off_t size_pack = 0; @@ -129,7 +130,7 @@ int cmd_count_objects(int argc, struct strbuf pack_buf = STRBUF_INIT; struct strbuf garbage_buf = STRBUF_INIT; - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (!p->pack_local) continue; if (open_pack_index(p)) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 2c35f9345d02d7c711ee522de5973b205c85ef51..b1d5549815ac66b773ad4c1592b34f9a8d590ee2 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -897,11 +897,11 @@ static void end_packfile(void) idx_name = keep_pack(create_index()); /* Register the packfile with core git's machinery. */ - new_p = add_packed_git(pack_data->repo, idx_name, strlen(idx_name), 1); + new_p = packfile_store_load_pack(pack_data->repo->objects->packfiles, + idx_name, 1); if (!new_p) die("core git rejected index %s", idx_name); all_packs[pack_id] = new_p; - install_packed_git(the_repository, new_p); free(idx_name); /* Print the boundary */ @@ -952,6 +952,7 @@ static int store_object( struct object_id *oidout, uintmax_t mark) { + struct packfile_store *packs = the_repository->objects->packfiles; void *out, *delta; struct object_entry *e; unsigned char hdr[96]; @@ -975,7 +976,7 @@ static int store_object( if (e->idx.offset) { duplicate_count_by_type[type]++; return 1; - } else if (find_oid_pack(&oid, get_all_packs(the_repository))) { + } else if (find_oid_pack(&oid, packfile_store_get_all_packs(packs))) { e->type = type; e->pack_id = MAX_PACK_ID; e->idx.offset = 1; /* just not zero! */ @@ -1092,6 +1093,7 @@ static void truncate_pack(struct hashfile_checkpoint *checkpoint) static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) { + struct packfile_store *packs = the_repository->objects->packfiles; size_t in_sz = 64 * 1024, out_sz = 64 * 1024; unsigned char *in_buf = xmalloc(in_sz); unsigned char *out_buf = xmalloc(out_sz); @@ -1175,7 +1177,7 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) duplicate_count_by_type[OBJ_BLOB]++; truncate_pack(&checkpoint); - } else if (find_oid_pack(&oid, get_all_packs(the_repository))) { + } else if (find_oid_pack(&oid, packfile_store_get_all_packs(packs))) { e->type = OBJ_BLOB; e->pack_id = MAX_PACK_ID; e->idx.offset = 1; /* just not zero! */ diff --git a/builtin/fsck.c b/builtin/fsck.c index d2eb9d4fbe922bc6ada11573ef61c1731b97e162..8ee95e0d67cf374b9814c92558ce969796300a7b 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -867,19 +867,20 @@ static int mark_packed_for_connectivity(const struct object_id *oid, static int check_pack_rev_indexes(struct repository *r, int show_progress) { + struct packfile_store *packs = r->objects->packfiles; struct progress *progress = NULL; uint32_t pack_count = 0; int res = 0; if (show_progress) { - for (struct packed_git *p = get_all_packs(r); p; p = p->next) + for (struct packed_git *p = packfile_store_get_all_packs(packs); p; p = p->next) pack_count++; progress = start_delayed_progress(the_repository, "Verifying reverse pack-indexes", pack_count); pack_count = 0; } - for (struct packed_git *p = get_all_packs(r); p; p = p->next) { + for (struct packed_git *p = packfile_store_get_all_packs(packs); p; p = p->next) { int load_error = load_pack_revindex_from_disk(p); if (load_error < 0) { @@ -999,6 +1000,8 @@ int cmd_fsck(int argc, for_each_packed_object(the_repository, mark_packed_for_connectivity, NULL, 0); } else { + struct packfile_store *packs = the_repository->objects->packfiles; + odb_prepare_alternates(the_repository->objects); for (source = the_repository->objects->sources; source; source = source->next) fsck_source(source); @@ -1009,7 +1012,7 @@ int cmd_fsck(int argc, struct progress *progress = NULL; if (show_progress) { - for (p = get_all_packs(the_repository); p; + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (open_pack_index(p)) continue; @@ -1019,7 +1022,7 @@ int cmd_fsck(int argc, progress = start_progress(the_repository, _("Checking objects"), total); } - for (p = get_all_packs(the_repository); p; + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { /* verify gives error messages itself */ if (verify_pack(the_repository, diff --git a/builtin/gc.c b/builtin/gc.c index 03ae4926b209820a9cd83520d18b067bc714500c..e19e13d9788076cf05029561a4fdcfea9f1fc095 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -487,9 +487,10 @@ static int too_many_loose_objects(struct gc_config *cfg) static struct packed_git *find_base_packs(struct string_list *packs, unsigned long limit) { + struct packfile_store *packfiles = the_repository->objects->packfiles; struct packed_git *p, *base = NULL; - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packfiles); p; p = p->next) { if (!p->pack_local || p->is_cruft) continue; if (limit) { @@ -508,13 +509,14 @@ static struct packed_git *find_base_packs(struct string_list *packs, static int too_many_packs(struct gc_config *cfg) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; int cnt; if (cfg->gc_auto_pack_limit <= 0) return 0; - for (cnt = 0, p = get_all_packs(the_repository); p; p = p->next) { + for (cnt = 0, p = packfile_store_get_all_packs(packs); p; p = p->next) { if (!p->pack_local) continue; if (p->pack_keep) @@ -1042,7 +1044,7 @@ int cmd_gc(int argc, die(FAILED_RUN, "rerere"); report_garbage = report_pack_garbage; - reprepare_packed_git(the_repository); + odb_reprepare(the_repository->objects); if (pack_garbage.nr > 0) { close_object_store(the_repository->objects); clean_pack_garbage(); @@ -1423,7 +1425,7 @@ static int incremental_repack_auto_condition(struct gc_config *cfg UNUSED) if (incremental_repack_auto_limit < 0) return 1; - for (p = get_packed_git(the_repository); + for (p = packfile_store_get_packs(the_repository->objects->packfiles); count < incremental_repack_auto_limit && p; p = p->next) { if (!p->multi_pack_index) @@ -1491,8 +1493,8 @@ static off_t get_auto_pack_size(void) struct packed_git *p; struct repository *r = the_repository; - reprepare_packed_git(r); - for (p = get_all_packs(r); p; p = p->next) { + odb_reprepare(r->objects); + for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { if (p->pack_size > max_size) { second_largest_size = max_size; max_size = p->pack_size; diff --git a/builtin/grep.c b/builtin/grep.c index 5df653733371d809d63c4be283c231de468b8ffd..63a4959568fa1cc375da211282d203c1898dafa2 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -1214,7 +1214,7 @@ int cmd_grep(int argc, if (recurse_submodules) repo_read_gitmodules(the_repository, 1); if (startup_info->have_repository) - (void)get_packed_git(the_repository); + (void)packfile_store_get_packs(the_repository->objects->packfiles); start_threads(&opt); } else { diff --git a/builtin/index-pack.c b/builtin/index-pack.c index f91c301bba9fbd9171aba65d708d27f9e58c8012..2b78ba7fe4d14a8bcb12165395ff02deeb8c047f 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1640,13 +1640,9 @@ static void final(const char *final_pack_name, const char *curr_pack_name, rename_tmp_packfile(&final_index_name, curr_index_name, &index_name, hash, "idx", 1); - if (do_fsck_object) { - struct packed_git *p; - p = add_packed_git(the_repository, final_index_name, - strlen(final_index_name), 0); - if (p) - install_packed_git(the_repository, p); - } + if (do_fsck_object) + packfile_store_load_pack(the_repository->objects->packfiles, + final_index_name, 0); if (!from_stdin) { printf("%s\n", hash_to_hex(hash)); diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index d3b9e98be37424b02d7bfc6c1b456fe765a227ce..5f364aa816ba25d48c8f59ffd3a33c3344ce84a8 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -65,12 +65,20 @@ static int parse_object_dir(const struct option *opt, const char *arg, char **value = opt->value; free(*value); if (unset) - *value = xstrdup(repo_get_object_directory(the_repository)); + *value = xstrdup(the_repository->objects->sources->path); else *value = real_pathdup(arg, 1); return 0; } +static struct odb_source *handle_object_dir_option(struct repository *repo) +{ + struct odb_source *source = odb_find_source(repo->objects, opts.object_dir); + if (!source) + source = odb_add_to_alternates_memory(repo->objects, opts.object_dir); + return source; +} + static struct option common_opts[] = { OPT_CALLBACK(0, "object-dir", &opts.object_dir, N_("directory"), @@ -140,6 +148,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, N_("refs snapshot for selecting bitmap commits")), OPT_END(), }; + struct odb_source *source; int ret; opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE; @@ -158,6 +167,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_write_usage, options); + source = handle_object_dir_option(repo); FREE_AND_NULL(options); @@ -166,7 +176,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, read_packs_from_stdin(&packs); - ret = write_midx_file_only(repo, opts.object_dir, &packs, + ret = write_midx_file_only(source, &packs, opts.preferred_pack, opts.refs_snapshot, opts.flags); @@ -177,7 +187,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } - ret = write_midx_file(repo, opts.object_dir, opts.preferred_pack, + ret = write_midx_file(source, opts.preferred_pack, opts.refs_snapshot, opts.flags); free(opts.refs_snapshot); @@ -194,6 +204,8 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; + options = add_common_options(builtin_multi_pack_index_verify_options); trace2_cmd_mode(argv[0]); @@ -206,10 +218,11 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_verify_usage, options); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return verify_midx_file(the_repository, opts.object_dir, opts.flags); + return verify_midx_file(source, opts.flags); } static int cmd_multi_pack_index_expire(int argc, const char **argv, @@ -222,6 +235,8 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; + options = add_common_options(builtin_multi_pack_index_expire_options); trace2_cmd_mode(argv[0]); @@ -234,10 +249,11 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_expire_usage, options); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return expire_midx_packs(the_repository, opts.object_dir, opts.flags); + return expire_midx_packs(source, opts.flags); } static int cmd_multi_pack_index_repack(int argc, const char **argv, @@ -252,6 +268,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; options = add_common_options(builtin_multi_pack_index_repack_options); @@ -266,11 +283,11 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_repack_usage, options); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return midx_repack(the_repository, opts.object_dir, - (size_t)opts.batch_size, opts.flags); + return midx_repack(source, (size_t)opts.batch_size, opts.flags); } int cmd_multi_pack_index(int argc, diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 53a225625039eae43a0d5aef6538f13b2a40d6de..61bbbdfb83feeb1f1df2bbd2792d8e6f14ef4fa2 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1741,19 +1741,19 @@ static int want_object_in_pack_mtime(const struct object_id *oid, struct multi_pack_index *m = get_multi_pack_index(source); struct pack_entry e; - if (m && fill_midx_entry(the_repository, oid, &e, m)) { + if (m && fill_midx_entry(m, oid, &e)) { want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset, found_mtime); if (want != -1) return want; } } - list_for_each(pos, get_packed_git_mru(the_repository)) { + list_for_each(pos, packfile_store_get_packs_mru(the_repository->objects->packfiles)) { struct packed_git *p = list_entry(pos, struct packed_git, mru); want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime); if (!exclude && want > 0) list_move(&p->mru, - get_packed_git_mru(the_repository)); + packfile_store_get_packs_mru(the_repository->objects->packfiles)); if (want != -1) return want; } @@ -3831,6 +3831,7 @@ static int pack_mtime_cmp(const void *_a, const void *_b) static void read_packs_list_from_stdin(struct rev_info *revs) { + struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct string_list include_packs = STRING_LIST_INIT_DUP; struct string_list exclude_packs = STRING_LIST_INIT_DUP; @@ -3855,7 +3856,7 @@ static void read_packs_list_from_stdin(struct rev_info *revs) string_list_sort(&exclude_packs); string_list_remove_duplicates(&exclude_packs, 0); - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { const char *pack_name = pack_basename(p); if ((item = string_list_lookup(&include_packs, pack_name))) @@ -4076,6 +4077,7 @@ static void enumerate_cruft_objects(void) static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; struct rev_info revs; int ret; @@ -4105,7 +4107,7 @@ static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs * Re-mark only the fresh packs as kept so that objects in * unknown packs do not halt the reachability traversal early. */ - for (p = get_all_packs(the_repository); p; p = p->next) + for (p = packfile_store_get_all_packs(packs); p; p = p->next) p->pack_keep_in_core = 0; mark_pack_kept_in_core(fresh_packs, 1); @@ -4122,6 +4124,7 @@ static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs static void read_cruft_objects(void) { + struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct string_list discard_packs = STRING_LIST_INIT_DUP; struct string_list fresh_packs = STRING_LIST_INIT_DUP; @@ -4142,7 +4145,7 @@ static void read_cruft_objects(void) string_list_sort(&discard_packs); string_list_sort(&fresh_packs); - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { const char *pack_name = pack_basename(p); struct string_list_item *item; @@ -4390,11 +4393,12 @@ static void add_unreachable_loose_objects(struct rev_info *revs) static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid) { + struct packfile_store *packs = the_repository->objects->packfiles; static struct packed_git *last_found = (void *)1; struct packed_git *p; p = (last_found != (void *)1) ? last_found : - get_all_packs(the_repository); + packfile_store_get_all_packs(packs); while (p) { if ((!p->pack_local || p->pack_keep || @@ -4404,7 +4408,7 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid) return 1; } if (p == last_found) - p = get_all_packs(the_repository); + p = packfile_store_get_all_packs(packs); else p = p->next; if (p == last_found) @@ -4436,12 +4440,13 @@ static int loosened_object_can_be_discarded(const struct object_id *oid, static void loosen_unused_packed_objects(void) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; uint32_t i; uint32_t loosened_objects_nr = 0; struct object_id oid; - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (!p->pack_local || p->pack_keep || p->pack_keep_in_core) continue; @@ -4742,12 +4747,13 @@ static void get_object_list(struct rev_info *revs, int ac, const char **av) static void add_extra_kept_packs(const struct string_list *names) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; if (!names->nr) return; - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { const char *name = basename(p->pack_name); int i; @@ -5185,8 +5191,10 @@ int cmd_pack_objects(int argc, add_extra_kept_packs(&keep_pack_list); if (ignore_packed_keep_on_disk) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = get_all_packs(the_repository); p; p = p->next) + + for (p = packfile_store_get_all_packs(packs); p; p = p->next) if (p->pack_local && p->pack_keep) break; if (!p) /* no keep-able packs found */ @@ -5198,8 +5206,10 @@ int cmd_pack_objects(int argc, * want to unset "local" based on looking at packs, as * it also covers non-local objects */ + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = get_all_packs(the_repository); p; p = p->next) { + + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (!p->pack_local) { have_non_local_packs = 1; break; diff --git a/builtin/pack-redundant.c b/builtin/pack-redundant.c index fe81c293e3af6f6b1db03c8e27b5283af8c7d3fe..dd28171f0a179a156c4e48c82dd444bde5a7995a 100644 --- a/builtin/pack-redundant.c +++ b/builtin/pack-redundant.c @@ -566,7 +566,8 @@ static struct pack_list * add_pack(struct packed_git *p) static struct pack_list * add_pack_file(const char *filename) { - struct packed_git *p = get_all_packs(the_repository); + struct packfile_store *packs = the_repository->objects->packfiles; + struct packed_git *p = packfile_store_get_all_packs(packs); if (strlen(filename) < 40) die("Bad pack filename: %s", filename); @@ -581,7 +582,8 @@ static struct pack_list * add_pack_file(const char *filename) static void load_all(void) { - struct packed_git *p = get_all_packs(the_repository); + struct packfile_store *packs = the_repository->objects->packfiles; + struct packed_git *p = packfile_store_get_all_packs(packs); while (p) { add_pack(p); diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 1113137a6f0b3f961fd02822154a41bcdedb6657..c9288a9c7e382bd19193d8b94268e0d0bec2c7f8 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -2389,7 +2389,7 @@ static const char *unpack(int err_fd, struct shallow_info *si) status = finish_command(&child); if (status) return "index-pack abnormal exit"; - reprepare_packed_git(the_repository); + odb_reprepare(the_repository->objects); } return NULL; } diff --git a/builtin/repack.c b/builtin/repack.c index a4def3919787ed6f0c60a06197dd3d2ee7c18ab7..e8730808c535a96732610d8250a31e5b0ced68a7 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -223,9 +223,10 @@ static void mark_packs_for_deletion(struct existing_packs *existing, static void remove_redundant_pack(const char *dir_name, const char *base_name) { struct strbuf buf = STRBUF_INIT; - struct multi_pack_index *m = get_multi_pack_index(the_repository->objects->sources); + struct odb_source *source = the_repository->objects->sources; + struct multi_pack_index *m = get_multi_pack_index(source); strbuf_addf(&buf, "%s.pack", base_name); - if (m && m->local && midx_contains_pack(m, buf.buf)) + if (m && source->local && midx_contains_pack(m, buf.buf)) clear_midx_file(the_repository); strbuf_insertf(&buf, 0, "%s/", dir_name); unlink_pack_path(buf.buf, 1); @@ -264,10 +265,11 @@ static void existing_packs_release(struct existing_packs *existing) static void collect_pack_filenames(struct existing_packs *existing, const struct string_list *extra_keep) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; struct strbuf buf = STRBUF_INIT; - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { int i; const char *base; @@ -496,10 +498,11 @@ static void init_pack_geometry(struct pack_geometry *geometry, struct existing_packs *existing, const struct pack_objects_args *args) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; struct strbuf buf = STRBUF_INIT; - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (args->local && !p->pack_local) /* * When asked to only repack local packfiles we skip @@ -1136,11 +1139,12 @@ static int write_filtered_pack(const struct pack_objects_args *args, static void combine_small_cruft_packs(FILE *in, size_t combine_cruft_below_size, struct existing_packs *existing) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; struct strbuf buf = STRBUF_INIT; size_t i; - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (!(p->is_cruft && p->pack_local)) continue; @@ -1684,7 +1688,7 @@ int cmd_repack(int argc, goto cleanup; } - reprepare_packed_git(the_repository); + odb_reprepare(the_repository->objects); if (delete_redundant) { int opts = 0; @@ -1711,7 +1715,7 @@ int cmd_repack(int argc, unsigned flags = 0; if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(the_repository, repo_get_object_directory(the_repository), + write_midx_file(the_repository->objects->sources, NULL, NULL, flags); } diff --git a/bulk-checkin.c b/bulk-checkin.c index b2809ab0398136ea0bd18b5d1c90f6a5729415ab..f65439a748a4ece5371b244008d135e7ae809f5d 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -90,7 +90,7 @@ static void flush_bulk_checkin_packfile(struct bulk_checkin_packfile *state) strbuf_release(&packname); /* Make objects we just wrote available to ourselves */ - reprepare_packed_git(the_repository); + odb_reprepare(the_repository->objects); } /* diff --git a/connected.c b/connected.c index 18c13245d8e40c66a22fa5d7c2d139b85e5113c2..b288a18b17c33fcead8b14947dfce38276c3d093 100644 --- a/connected.c +++ b/connected.c @@ -72,11 +72,12 @@ int check_connected(oid_iterate_fn fn, void *cb_data, * Before checking for promisor packs, be sure we have the * latest pack-files loaded into memory. */ - reprepare_packed_git(the_repository); + odb_reprepare(the_repository->objects); do { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (!p->pack_promisor) continue; if (find_pack_entry_one(oid, p)) diff --git a/fetch-pack.c b/fetch-pack.c index 6ed566295189dcfa57291e55fc452b1a28d39e4e..fe7a84bf2f97fa89882bbfda079d346c7de21ddd 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1983,7 +1983,7 @@ static void update_shallow(struct fetch_pack_args *args, * remote is shallow, but this is a clone, there are * no objects in repo to worry about. Accept any * shallow points that exist in the pack (iow in repo - * after get_pack() and reprepare_packed_git()) + * after get_pack() and odb_reprepare()) */ struct oid_array extra = OID_ARRAY_INIT; struct object_id *oid = si->shallow->oid; @@ -2108,7 +2108,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args, ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, &si, pack_lockfiles); } - reprepare_packed_git(the_repository); + odb_reprepare(the_repository->objects); if (!args->cloning && args->deepen) { struct check_connected_options opt = CHECK_CONNECTED_INIT; diff --git a/http-backend.c b/http-backend.c index d5dfe762bb51783cf6f81de8cfa79075d26075c3..9084058f1e9f138bda8525ce6db52df24b2fa95e 100644 --- a/http-backend.c +++ b/http-backend.c @@ -603,18 +603,19 @@ static void get_head(struct strbuf *hdr, char *arg UNUSED) static void get_info_packs(struct strbuf *hdr, char *arg UNUSED) { size_t objdirlen = strlen(repo_get_object_directory(the_repository)); + struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct packed_git *p; size_t cnt = 0; select_getanyfile(hdr); - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (p->pack_local) cnt++; } strbuf_grow(&buf, cnt * 53 + 2); - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (p->pack_local) strbuf_addf(&buf, "P %s\n", p->pack_name + objdirlen + 6); } diff --git a/http.c b/http.c index 98853d64834f99f881635944443cc368dc415eb0..077e879de9ebabc9fec996e2b9ab2842c7d18c54 100644 --- a/http.c +++ b/http.c @@ -2408,6 +2408,7 @@ static char *fetch_pack_index(unsigned char *hash, const char *base_url) static int fetch_and_setup_pack_index(struct packed_git **packs_head, unsigned char *sha1, const char *base_url) { + struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *new_pack, *p; char *tmp_idx = NULL; int ret; @@ -2416,7 +2417,7 @@ static int fetch_and_setup_pack_index(struct packed_git **packs_head, * If we already have the pack locally, no need to fetch its index or * even add it to list; we already have all of its objects. */ - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { if (hasheq(p->hash, sha1, the_repository->hash_algo)) return 0; } @@ -2541,7 +2542,7 @@ void http_install_packfile(struct packed_git *p, lst = &((*lst)->next); *lst = (*lst)->next; - install_packed_git(the_repository, p); + packfile_store_add_pack(the_repository->objects->packfiles, p); } struct http_pack_request *new_http_pack_request( diff --git a/http.h b/http.h index 36202139f451ffa1e5b2cbaeae49d0875a4f76a7..e5a5380c6c384e56966d12d0c4d5a9d2e12654c0 100644 --- a/http.h +++ b/http.h @@ -210,7 +210,7 @@ int finish_http_pack_request(struct http_pack_request *preq); void release_http_pack_request(struct http_pack_request *preq); /* - * Remove p from the given list, and invoke install_packed_git() on it. + * Remove p from the given list, and invoke packfile_store_add_pack() on it. * * This is a convenience function for users that have obtained a list of packs * from http_get_info_packs() and have chosen a specific pack to fetch. diff --git a/midx-write.c b/midx-write.c index a0aceab5e0d1acd9de27a719676d62fc97fef4d0..1e99f6a26e9e9103ceac2b9ff5aa04ab8428a236 100644 --- a/midx-write.c +++ b/midx-write.c @@ -26,9 +26,9 @@ #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t)) extern int midx_checksum_valid(struct multi_pack_index *m); -extern void clear_midx_files_ext(const char *object_dir, const char *ext, +extern void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash); -extern void clear_incremental_midx_files_ext(const char *object_dir, +extern void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, const char **keep_hashes, uint32_t hashes_nr); @@ -112,6 +112,7 @@ struct write_midx_context { struct string_list *to_include; struct repository *repo; + struct odb_source *source; }; static int should_include_pack(const struct write_midx_context *ctx, @@ -648,7 +649,6 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx) } static void write_midx_reverse_index(struct write_midx_context *ctx, - const char *object_dir, unsigned char *midx_hash) { struct strbuf buf = STRBUF_INIT; @@ -657,11 +657,10 @@ static void write_midx_reverse_index(struct write_midx_context *ctx, trace2_region_enter("midx", "write_midx_reverse_index", ctx->repo); if (ctx->incremental) - get_split_midx_filename_ext(ctx->repo->hash_algo, &buf, - object_dir, midx_hash, - MIDX_EXT_REV); + get_split_midx_filename_ext(ctx->source, &buf, + midx_hash, MIDX_EXT_REV); else - get_midx_filename_ext(ctx->repo->hash_algo, &buf, object_dir, + get_midx_filename_ext(ctx->source, &buf, midx_hash, MIDX_EXT_REV); tmp_file = write_rev_file_order(ctx->repo, NULL, ctx->pack_order, @@ -836,7 +835,6 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr } static int write_midx_bitmap(struct write_midx_context *ctx, - const char *object_dir, const unsigned char *midx_hash, struct packing_data *pdata, struct commit **commits, @@ -852,12 +850,11 @@ static int write_midx_bitmap(struct write_midx_context *ctx, trace2_region_enter("midx", "write_midx_bitmap", ctx->repo); if (ctx->incremental) - get_split_midx_filename_ext(ctx->repo->hash_algo, &bitmap_name, - object_dir, midx_hash, - MIDX_EXT_BITMAP); + get_split_midx_filename_ext(ctx->source, &bitmap_name, + midx_hash, MIDX_EXT_BITMAP); else - get_midx_filename_ext(ctx->repo->hash_algo, &bitmap_name, - object_dir, midx_hash, MIDX_EXT_BITMAP); + get_midx_filename_ext(ctx->source, &bitmap_name, + midx_hash, MIDX_EXT_BITMAP); if (flags & MIDX_WRITE_BITMAP_HASH_CACHE) options |= BITMAP_OPT_HASH_CACHE; @@ -913,13 +910,6 @@ static int write_midx_bitmap(struct write_midx_context *ctx, return ret; } -static struct multi_pack_index *lookup_multi_pack_index(struct repository *r, - const char *object_dir) -{ - struct odb_source *source = odb_find_source(r->objects, object_dir); - return get_multi_pack_index(source); -} - static int fill_packs_from_midx(struct write_midx_context *ctx, const char *preferred_pack_name, uint32_t flags) { @@ -942,8 +932,7 @@ static int fill_packs_from_midx(struct write_midx_context *ctx, */ if (flags & MIDX_WRITE_REV_INDEX || preferred_pack_name) { - if (prepare_midx_pack(ctx->repo, m, - m->num_packs_in_base + i)) { + if (prepare_midx_pack(m, m->num_packs_in_base + i)) { error(_("could not load pack")); return 1; } @@ -989,10 +978,9 @@ static int link_midx_to_chain(struct multi_pack_index *m) for (i = 0; i < ARRAY_SIZE(midx_exts); i++) { const unsigned char *hash = get_midx_checksum(m); - get_midx_filename_ext(m->repo->hash_algo, &from, m->object_dir, + get_midx_filename_ext(m->source, &from, hash, midx_exts[i].non_split); - get_split_midx_filename_ext(m->repo->hash_algo, &to, - m->object_dir, hash, + get_split_midx_filename_ext(m->source, &to, hash, midx_exts[i].split); if (link(from.buf, to.buf) < 0 && errno != ENOENT) { @@ -1011,7 +999,7 @@ static int link_midx_to_chain(struct multi_pack_index *m) return ret; } -static void clear_midx_files(struct repository *r, const char *object_dir, +static void clear_midx_files(struct odb_source *source, const char **hashes, uint32_t hashes_nr, unsigned incremental) { @@ -1030,16 +1018,16 @@ static void clear_midx_files(struct repository *r, const char *object_dir, uint32_t i, j; for (i = 0; i < ARRAY_SIZE(exts); i++) { - clear_incremental_midx_files_ext(object_dir, exts[i], + clear_incremental_midx_files_ext(source, exts[i], hashes, hashes_nr); for (j = 0; j < hashes_nr; j++) - clear_midx_files_ext(object_dir, exts[i], hashes[j]); + clear_midx_files_ext(source, exts[i], hashes[j]); } if (incremental) - get_midx_filename(r->hash_algo, &buf, object_dir); + get_midx_filename(source, &buf); else - get_midx_chain_filename(&buf, object_dir); + get_midx_chain_filename(source, &buf); if (unlink(buf.buf) && errno != ENOENT) die_errno(_("failed to clear multi-pack-index at %s"), buf.buf); @@ -1047,13 +1035,14 @@ static void clear_midx_files(struct repository *r, const char *object_dir, strbuf_release(&buf); } -static int write_midx_internal(struct repository *r, const char *object_dir, +static int write_midx_internal(struct odb_source *source, struct string_list *packs_to_include, struct string_list *packs_to_drop, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { + struct repository *r = source->odb->repo; struct strbuf midx_name = STRBUF_INIT; unsigned char midx_hash[GIT_MAX_RAWSZ]; uint32_t i, start_pack; @@ -1071,21 +1060,22 @@ static int write_midx_internal(struct repository *r, const char *object_dir, trace2_region_enter("midx", "write_midx_internal", r); ctx.repo = r; + ctx.source = source; ctx.incremental = !!(flags & MIDX_WRITE_INCREMENTAL); if (ctx.incremental) strbuf_addf(&midx_name, "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX", - object_dir); + source->path); else - get_midx_filename(r->hash_algo, &midx_name, object_dir); + get_midx_filename(source, &midx_name); if (safe_create_leading_directories(r, midx_name.buf)) die_errno(_("unable to create leading directories of %s"), midx_name.buf); if (!packs_to_include || ctx.incremental) { - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); if (m && !midx_checksum_valid(m)) { warning(_("ignoring existing multi-pack-index; checksum mismatch")); m = NULL; @@ -1116,7 +1106,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (flags & MIDX_WRITE_BITMAP && load_midx_revindex(m)) { error(_("could not load reverse index for MIDX %s"), hash_to_hex_algop(get_midx_checksum(m), - m->repo->hash_algo)); + m->source->odb->repo->hash_algo)); result = 1; goto cleanup; } @@ -1139,7 +1129,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, ctx.to_include = packs_to_include; - for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx); + for_each_file_in_pack_dir(source->path, add_pack_to_midx, &ctx); stop_progress(&ctx.progress); if ((ctx.m && ctx.nr == ctx.m->num_packs + ctx.m->num_packs_in_base) && @@ -1159,7 +1149,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, * corresponding bitmap (or one wasn't requested). */ if (!want_bitmap) - clear_midx_files_ext(object_dir, "bitmap", NULL); + clear_midx_files_ext(source, "bitmap", NULL); goto cleanup; } } @@ -1327,7 +1317,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (ctx.incremental) { struct strbuf lock_name = STRBUF_INIT; - get_midx_chain_filename(&lock_name, object_dir); + get_midx_chain_filename(source, &lock_name); hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR); strbuf_release(&lock_name); @@ -1390,7 +1380,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (flags & MIDX_WRITE_REV_INDEX && git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0)) - write_midx_reverse_index(&ctx, object_dir, midx_hash); + write_midx_reverse_index(&ctx, midx_hash); if (flags & MIDX_WRITE_BITMAP) { struct packing_data pdata; @@ -1413,7 +1403,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, FREE_AND_NULL(ctx.entries); ctx.entries_nr = 0; - if (write_midx_bitmap(&ctx, object_dir, + if (write_midx_bitmap(&ctx, midx_hash, &pdata, commits, commits_nr, flags) < 0) { error(_("could not write multi-pack bitmap")); @@ -1446,8 +1436,8 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (link_midx_to_chain(ctx.base_midx) < 0) return -1; - get_split_midx_filename_ext(r->hash_algo, &final_midx_name, - object_dir, midx_hash, MIDX_EXT_MIDX); + get_split_midx_filename_ext(source, &final_midx_name, + midx_hash, MIDX_EXT_MIDX); if (rename_tempfile(&incr, final_midx_name.buf) < 0) { error_errno(_("unable to rename new multi-pack-index layer")); @@ -1480,7 +1470,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (commit_lock_file(&lk) < 0) die_errno(_("could not write multi-pack-index")); - clear_midx_files(r, object_dir, keep_hashes, + clear_midx_files(source, keep_hashes, ctx.num_multi_pack_indexes_before + 1, ctx.incremental); @@ -1509,29 +1499,29 @@ static int write_midx_internal(struct repository *r, const char *object_dir, return result; } -int write_midx_file(struct repository *r, const char *object_dir, +int write_midx_file(struct odb_source *source, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { - return write_midx_internal(r, object_dir, NULL, NULL, + return write_midx_internal(source, NULL, NULL, preferred_pack_name, refs_snapshot, flags); } -int write_midx_file_only(struct repository *r, const char *object_dir, +int write_midx_file_only(struct odb_source *source, struct string_list *packs_to_include, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { - return write_midx_internal(r, object_dir, packs_to_include, NULL, + return write_midx_internal(source, packs_to_include, NULL, preferred_pack_name, refs_snapshot, flags); } -int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags) +int expire_midx_packs(struct odb_source *source, unsigned flags) { uint32_t i, *count, result = 0; struct string_list packs_to_drop = STRING_LIST_INIT_DUP; - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); struct progress *progress = NULL; if (!m) @@ -1544,7 +1534,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (flags & MIDX_PROGRESS) progress = start_delayed_progress( - r, + source->odb->repo, _("Counting referenced objects"), m->num_objects); for (i = 0; i < m->num_objects; i++) { @@ -1556,7 +1546,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (flags & MIDX_PROGRESS) progress = start_delayed_progress( - r, + source->odb->repo, _("Finding and deleting unreferenced packfiles"), m->num_packs); for (i = 0; i < m->num_packs; i++) { @@ -1566,7 +1556,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (count[i]) continue; - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) continue; if (m->packs[i]->pack_keep || m->packs[i]->is_cruft) @@ -1584,7 +1574,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla free(count); if (packs_to_drop.nr) - result = write_midx_internal(r, object_dir, NULL, + result = write_midx_internal(source, NULL, &packs_to_drop, NULL, NULL, flags); string_list_clear(&packs_to_drop, 0); @@ -1612,13 +1602,12 @@ static int compare_by_mtime(const void *a_, const void *b_) return 0; } -static int want_included_pack(struct repository *r, - struct multi_pack_index *m, +static int want_included_pack(struct multi_pack_index *m, int pack_kept_objects, uint32_t pack_int_id) { struct packed_git *p; - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return 0; p = m->packs[pack_int_id]; if (!pack_kept_objects && p->pack_keep) @@ -1640,7 +1629,7 @@ static void fill_included_packs_all(struct repository *r, repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects); for (i = 0; i < m->num_packs; i++) { - if (!want_included_pack(r, m, pack_kept_objects, i)) + if (!want_included_pack(m, pack_kept_objects, i)) continue; include_pack[i] = 1; @@ -1664,7 +1653,7 @@ static void fill_included_packs_batch(struct repository *r, for (i = 0; i < m->num_packs; i++) { pack_info[i].pack_int_id = i; - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) continue; pack_info[i].mtime = m->packs[i]->mtime; @@ -1683,7 +1672,7 @@ static void fill_included_packs_batch(struct repository *r, struct packed_git *p = m->packs[pack_int_id]; uint64_t expected_size; - if (!want_included_pack(r, m, pack_kept_objects, pack_int_id)) + if (!want_included_pack(m, pack_kept_objects, pack_int_id)) continue; /* @@ -1710,14 +1699,15 @@ static void fill_included_packs_batch(struct repository *r, free(pack_info); } -int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags) +int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags) { + struct repository *r = source->odb->repo; int result = 0; uint32_t i, packs_to_repack = 0; unsigned char *include_pack; struct child_process cmd = CHILD_PROCESS_INIT; FILE *cmd_in; - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); /* * When updating the default for these configuration @@ -1751,7 +1741,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, strvec_push(&cmd.args, "pack-objects"); - strvec_pushf(&cmd.args, "%s/pack/pack", object_dir); + strvec_pushf(&cmd.args, "%s/pack/pack", source->path); if (delta_base_offset) strvec_push(&cmd.args, "--delta-base-offset"); @@ -1792,7 +1782,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, goto cleanup; } - result = write_midx_internal(r, object_dir, NULL, NULL, NULL, NULL, + result = write_midx_internal(source, NULL, NULL, NULL, NULL, flags); cleanup: diff --git a/midx.c b/midx.c index 7d407682e60a6f4adbd9f760f37c73a59a2712a4..1d6269f957e7819fb62963cde191360422b29229 100644 --- a/midx.c +++ b/midx.c @@ -16,9 +16,9 @@ #define MIDX_PACK_ERROR ((void *)(intptr_t)-1) int midx_checksum_valid(struct multi_pack_index *m); -void clear_midx_files_ext(const char *object_dir, const char *ext, +void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash); -void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, +void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, char **keep_hashes, uint32_t hashes_nr); int cmp_idx_or_pack_name(const char *idx_or_pack_name, @@ -26,22 +26,20 @@ int cmp_idx_or_pack_name(const char *idx_or_pack_name, const unsigned char *get_midx_checksum(struct multi_pack_index *m) { - return m->data + m->data_len - m->repo->hash_algo->rawsz; + return m->data + m->data_len - m->source->odb->repo->hash_algo->rawsz; } -void get_midx_filename(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir) +void get_midx_filename(struct odb_source *source, struct strbuf *out) { - get_midx_filename_ext(hash_algo, out, object_dir, NULL, NULL); + get_midx_filename_ext(source, out, NULL, NULL); } -void get_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir, +void get_midx_filename_ext(struct odb_source *source, struct strbuf *out, const unsigned char *hash, const char *ext) { - strbuf_addf(out, "%s/pack/multi-pack-index", object_dir); + strbuf_addf(out, "%s/pack/multi-pack-index", source->path); if (ext) - strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, hash_algo), ext); + strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext); } static int midx_read_oid_fanout(const unsigned char *chunk_start, @@ -95,11 +93,16 @@ static int midx_read_object_offsets(const unsigned char *chunk_start, return 0; } -static struct multi_pack_index *load_multi_pack_index_one(struct repository *r, - const char *object_dir, - const char *midx_name, - int local) +struct multi_pack_index *get_multi_pack_index(struct odb_source *source) { + packfile_store_prepare(source->odb->packfiles); + return source->midx; +} + +static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source, + const char *midx_name) +{ + struct repository *r = source->odb->repo; struct multi_pack_index *m = NULL; int fd; struct stat st; @@ -129,11 +132,10 @@ static struct multi_pack_index *load_multi_pack_index_one(struct repository *r, midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - FLEX_ALLOC_STR(m, object_dir, object_dir); + CALLOC_ARRAY(m, 1); m->data = midx_map; m->data_len = midx_size; - m->local = local; - m->repo = r; + m->source = source; m->signature = get_be32(m->data); if (m->signature != MIDX_SIGNATURE) @@ -224,24 +226,23 @@ static struct multi_pack_index *load_multi_pack_index_one(struct repository *r, return NULL; } -void get_midx_chain_dirname(struct strbuf *buf, const char *object_dir) +void get_midx_chain_dirname(struct odb_source *source, struct strbuf *buf) { - strbuf_addf(buf, "%s/pack/multi-pack-index.d", object_dir); + strbuf_addf(buf, "%s/pack/multi-pack-index.d", source->path); } -void get_midx_chain_filename(struct strbuf *buf, const char *object_dir) +void get_midx_chain_filename(struct odb_source *source, struct strbuf *buf) { - get_midx_chain_dirname(buf, object_dir); + get_midx_chain_dirname(source, buf); strbuf_addstr(buf, "/multi-pack-index-chain"); } -void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *buf, const char *object_dir, +void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf, const unsigned char *hash, const char *ext) { - get_midx_chain_dirname(buf, object_dir); + get_midx_chain_dirname(source, buf); strbuf_addf(buf, "/multi-pack-index-%s.%s", - hash_to_hex_algop(hash, hash_algo), ext); + hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext); } static int open_multi_pack_index_chain(const struct git_hash_algo *hash_algo, @@ -297,19 +298,18 @@ static int add_midx_to_chain(struct multi_pack_index *midx, return 1; } -static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, - const char *object_dir, - int local, +static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source, int fd, struct stat *st, int *incomplete_chain) { + const struct git_hash_algo *hash_algo = source->odb->repo->hash_algo; struct multi_pack_index *midx_chain = NULL; struct strbuf buf = STRBUF_INIT; int valid = 1; uint32_t i, count; FILE *fp = xfdopen(fd, "r"); - count = st->st_size / (r->hash_algo->hexsz + 1); + count = st->st_size / (hash_algo->hexsz + 1); for (i = 0; i < count; i++) { struct multi_pack_index *m; @@ -318,7 +318,7 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, if (strbuf_getline_lf(&buf, fp) == EOF) break; - if (get_oid_hex_algop(buf.buf, &layer, r->hash_algo)) { + if (get_oid_hex_algop(buf.buf, &layer, hash_algo)) { warning(_("invalid multi-pack-index chain: line '%s' " "not a hash"), buf.buf); @@ -329,9 +329,9 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, valid = 0; strbuf_reset(&buf); - get_split_midx_filename_ext(r->hash_algo, &buf, object_dir, + get_split_midx_filename_ext(source, &buf, layer.hash, MIDX_EXT_MIDX); - m = load_multi_pack_index_one(r, object_dir, buf.buf, local); + m = load_multi_pack_index_one(source, buf.buf); if (m) { if (add_midx_to_chain(m, midx_chain)) { @@ -354,40 +354,34 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, return midx_chain; } -static struct multi_pack_index *load_multi_pack_index_chain(struct repository *r, - const char *object_dir, - int local) +static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *source) { struct strbuf chain_file = STRBUF_INIT; struct stat st; int fd; struct multi_pack_index *m = NULL; - get_midx_chain_filename(&chain_file, object_dir); - if (open_multi_pack_index_chain(r->hash_algo, chain_file.buf, &fd, &st)) { + get_midx_chain_filename(source, &chain_file); + if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) { int incomplete; /* ownership of fd is taken over by load function */ - m = load_midx_chain_fd_st(r, object_dir, local, fd, &st, - &incomplete); + m = load_midx_chain_fd_st(source, fd, &st, &incomplete); } strbuf_release(&chain_file); return m; } -struct multi_pack_index *load_multi_pack_index(struct repository *r, - const char *object_dir, - int local) +struct multi_pack_index *load_multi_pack_index(struct odb_source *source) { struct strbuf midx_name = STRBUF_INIT; struct multi_pack_index *m; - get_midx_filename(r->hash_algo, &midx_name, object_dir); + get_midx_filename(source, &midx_name); - m = load_multi_pack_index_one(r, object_dir, - midx_name.buf, local); + m = load_multi_pack_index_one(source, midx_name.buf); if (!m) - m = load_multi_pack_index_chain(r, object_dir, local); + m = load_multi_pack_index_chain(source); strbuf_release(&midx_name); @@ -450,11 +444,11 @@ static uint32_t midx_for_pack(struct multi_pack_index **_m, return pack_int_id - m->num_packs_in_base; } -int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, +int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id) { + struct repository *r = m->source->odb->repo; struct strbuf pack_name = STRBUF_INIT; - struct strbuf key = STRBUF_INIT; struct packed_git *p; pack_int_id = midx_for_pack(&m, pack_int_id); @@ -464,26 +458,13 @@ int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, if (m->packs[pack_int_id]) return 0; - strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir, + strbuf_addf(&pack_name, "%s/pack/%s", m->source->path, m->pack_names[pack_int_id]); - - /* pack_map holds the ".pack" name, but we have the .idx */ - strbuf_addbuf(&key, &pack_name); - strbuf_strip_suffix(&key, ".idx"); - strbuf_addstr(&key, ".pack"); - p = hashmap_get_entry_from_hash(&r->objects->pack_map, - strhash(key.buf), key.buf, - struct packed_git, packmap_ent); - if (!p) { - p = add_packed_git(r, pack_name.buf, pack_name.len, m->local); - if (p) { - install_packed_git(r, p); - list_add_tail(&p->mru, &r->objects->packed_git_mru); - } - } - + p = packfile_store_load_pack(r->objects->packfiles, + pack_name.buf, m->source->local); + if (p) + list_add_tail(&p->mru, &r->objects->packfiles->mru); strbuf_release(&pack_name); - strbuf_release(&key); if (!p) { m->packs[pack_int_id] = MIDX_PACK_ERROR; @@ -507,7 +488,7 @@ struct packed_git *nth_midxed_pack(struct multi_pack_index *m, #define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t)) -int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, +int nth_bitmapped_pack(struct multi_pack_index *m, struct bitmapped_pack *bp, uint32_t pack_int_id) { uint32_t local_pack_int_id = midx_for_pack(&m, pack_int_id); @@ -515,7 +496,7 @@ int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, if (!m->chunk_bitmapped_packs) return error(_("MIDX does not contain the BTMP chunk")); - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return error(_("could not load bitmapped pack %"PRIu32), pack_int_id); bp->p = m->packs[local_pack_int_id]; @@ -534,7 +515,8 @@ int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result) { int ret = bsearch_hash(oid->hash, m->chunk_oid_fanout, - m->chunk_oid_lookup, m->repo->hash_algo->rawsz, + m->chunk_oid_lookup, + m->source->odb->repo->hash_algo->rawsz, result); if (result) *result += m->num_objects_in_base; @@ -565,7 +547,7 @@ struct object_id *nth_midxed_object_oid(struct object_id *oid, n = midx_for_object(&m, n); oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n), - m->repo->hash_algo); + m->source->odb->repo->hash_algo); return oid; } @@ -600,10 +582,9 @@ uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos) (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH); } -int fill_midx_entry(struct repository *r, +int fill_midx_entry(struct multi_pack_index *m, const struct object_id *oid, - struct pack_entry *e, - struct multi_pack_index *m) + struct pack_entry *e) { uint32_t pos; uint32_t pack_int_id; @@ -615,7 +596,7 @@ int fill_midx_entry(struct repository *r, midx_for_object(&m, pos); pack_int_id = nth_midxed_pack_int_id(m, pos); - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return 0; p = m->packs[pack_int_id - m->num_packs_in_base]; @@ -723,7 +704,7 @@ int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) return 0; } -int prepare_multi_pack_index_one(struct odb_source *source, int local) +int prepare_multi_pack_index_one(struct odb_source *source) { struct repository *r = source->odb->repo; @@ -734,14 +715,14 @@ int prepare_multi_pack_index_one(struct odb_source *source, int local) if (source->midx) return 1; - source->midx = load_multi_pack_index(r, source->path, local); + source->midx = load_multi_pack_index(source); return !!source->midx; } int midx_checksum_valid(struct multi_pack_index *m) { - return hashfile_checksum_valid(m->repo->hash_algo, + return hashfile_checksum_valid(m->source->odb->repo->hash_algo, m->data, m->data_len); } @@ -768,7 +749,7 @@ static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUS die_errno(_("failed to remove %s"), full_path); } -void clear_midx_files_ext(const char *object_dir, const char *ext, +void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash) { struct clear_midx_data data; @@ -782,7 +763,7 @@ void clear_midx_files_ext(const char *object_dir, const char *ext, } data.ext = ext; - for_each_file_in_pack_dir(object_dir, + for_each_file_in_pack_dir(source->path, clear_midx_file_ext, &data); @@ -791,7 +772,7 @@ void clear_midx_files_ext(const char *object_dir, const char *ext, free(data.keep); } -void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, +void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, char **keep_hashes, uint32_t hashes_nr) { @@ -807,7 +788,7 @@ void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, data.keep_nr = hashes_nr; data.ext = ext; - for_each_file_in_pack_subdir(object_dir, "multi-pack-index.d", + for_each_file_in_pack_subdir(source->path, "multi-pack-index.d", clear_midx_file_ext, &data); for (i = 0; i < hashes_nr; i++) @@ -819,7 +800,7 @@ void clear_midx_file(struct repository *r) { struct strbuf midx = STRBUF_INIT; - get_midx_filename(r->hash_algo, &midx, r->objects->sources->path); + get_midx_filename(r->objects->sources, &midx); if (r->objects) { struct odb_source *source; @@ -834,8 +815,8 @@ void clear_midx_file(struct repository *r) if (remove_path(midx.buf)) die(_("failed to clear multi-pack-index at %s"), midx.buf); - clear_midx_files_ext(r->objects->sources->path, MIDX_EXT_BITMAP, NULL); - clear_midx_files_ext(r->objects->sources->path, MIDX_EXT_REV, NULL); + clear_midx_files_ext(r->objects->sources, MIDX_EXT_BITMAP, NULL); + clear_midx_files_ext(r->objects->sources, MIDX_EXT_REV, NULL); strbuf_release(&midx); } @@ -879,12 +860,13 @@ static int compare_pair_pos_vs_id(const void *_a, const void *_b) display_progress(progress, _n); \ } while (0) -int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags) +int verify_midx_file(struct odb_source *source, unsigned flags) { + struct repository *r = source->odb->repo; struct pair_pos_vs_id *pairs = NULL; uint32_t i; struct progress *progress = NULL; - struct multi_pack_index *m = load_multi_pack_index(r, object_dir, 1); + struct multi_pack_index *m = load_multi_pack_index(source); struct multi_pack_index *curr; verify_midx_error = 0; @@ -893,7 +875,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag struct stat sb; struct strbuf filename = STRBUF_INIT; - get_midx_filename(r->hash_algo, &filename, object_dir); + get_midx_filename(source, &filename); if (!stat(filename.buf, &sb)) { error(_("multi-pack-index file exists, but failed to parse")); @@ -911,7 +893,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag _("Looking for referenced packfiles"), m->num_packs + m->num_packs_in_base); for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) { - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) midx_report("failed to load pack in position %d", i); display_progress(progress, i + 1); @@ -988,7 +970,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag nth_midxed_object_oid(&oid, m, pairs[i].pos); - if (!fill_midx_entry(r, &oid, &e, m)) { + if (!fill_midx_entry(m, &oid, &e)) { midx_report(_("failed to load pack entry for oid[%d] = %s"), pairs[i].pos, oid_to_hex(&oid)); continue; diff --git a/midx.h b/midx.h index 076382de8acd26c25c179c03b01994daa15bd711..6e54d73503d56088923ceff1c69e1f8d6013ac12 100644 --- a/midx.h +++ b/midx.h @@ -35,6 +35,8 @@ struct odb_source; "GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL" struct multi_pack_index { + struct odb_source *source; + const unsigned char *data; size_t data_len; @@ -50,7 +52,6 @@ struct multi_pack_index { uint32_t num_objects; int preferred_pack_idx; - int local; int has_chain; const unsigned char *chunk_pack_names; @@ -71,10 +72,6 @@ struct multi_pack_index { const char **pack_names; struct packed_git **packs; - - struct repository *repo; - - char object_dir[FLEX_ARRAY]; }; #define MIDX_PROGRESS (1 << 0) @@ -89,24 +86,20 @@ struct multi_pack_index { #define MIDX_EXT_MIDX "midx" const unsigned char *get_midx_checksum(struct multi_pack_index *m); -void get_midx_filename(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir); -void get_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir, +void get_midx_filename(struct odb_source *source, struct strbuf *out); +void get_midx_filename_ext(struct odb_source *source, struct strbuf *out, const unsigned char *hash, const char *ext); -void get_midx_chain_dirname(struct strbuf *buf, const char *object_dir); -void get_midx_chain_filename(struct strbuf *buf, const char *object_dir); -void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *buf, const char *object_dir, +void get_midx_chain_dirname(struct odb_source *source, struct strbuf *out); +void get_midx_chain_filename(struct odb_source *source, struct strbuf *out); +void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf, const unsigned char *hash, const char *ext); -struct multi_pack_index *load_multi_pack_index(struct repository *r, - const char *object_dir, - int local); -int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id); +struct multi_pack_index *get_multi_pack_index(struct odb_source *source); +struct multi_pack_index *load_multi_pack_index(struct odb_source *source); +int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id); struct packed_git *nth_midxed_pack(struct multi_pack_index *m, uint32_t pack_int_id); -int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, +int nth_bitmapped_pack(struct multi_pack_index *m, struct bitmapped_pack *bp, uint32_t pack_int_id); int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result); @@ -118,27 +111,27 @@ uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos); struct object_id *nth_midxed_object_oid(struct object_id *oid, struct multi_pack_index *m, uint32_t n); -int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m); +int fill_midx_entry(struct multi_pack_index *m, const struct object_id *oid, struct pack_entry *e); int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id); -int prepare_multi_pack_index_one(struct odb_source *source, int local); +int prepare_multi_pack_index_one(struct odb_source *source); /* * Variant of write_midx_file which writes a MIDX containing only the packs * specified in packs_to_include. */ -int write_midx_file(struct repository *r, const char *object_dir, +int write_midx_file(struct odb_source *source, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); -int write_midx_file_only(struct repository *r, const char *object_dir, +int write_midx_file_only(struct odb_source *source, struct string_list *packs_to_include, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); void clear_midx_file(struct repository *r); -int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags); -int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags); -int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags); +int verify_midx_file(struct odb_source *source, unsigned flags); +int expire_midx_packs(struct odb_source *source, unsigned flags); +int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags); void close_midx(struct multi_pack_index *m); diff --git a/object-name.c b/object-name.c index 732056ff5e305b7d79d3f272e321824a9107f0a2..53356819a3def23eef662b90b9e23c4a08843706 100644 --- a/object-name.c +++ b/object-name.c @@ -213,7 +213,7 @@ static void find_short_packed_object(struct disambiguate_state *ds) unique_in_midx(m, ds); } - for (p = get_packed_git(ds->repo); p && !ds->ambiguous; + for (p = packfile_store_get_packs(ds->repo->objects->packfiles); p && !ds->ambiguous; p = p->next) unique_in_pack(p, ds); } @@ -596,7 +596,7 @@ static enum get_oid_result get_short_oid(struct repository *r, * or migrated from loose to packed. */ if (status == MISSING_OBJECT) { - reprepare_packed_git(r); + odb_reprepare(r->objects); find_short_object_filename(&ds); find_short_packed_object(&ds); status = finish_object_disambiguation(&ds, oid); @@ -806,7 +806,7 @@ static void find_abbrev_len_packed(struct min_abbrev_data *mad) find_abbrev_len_for_midx(m, mad); } - for (p = get_packed_git(mad->repo); p; p = p->next) + for (p = packfile_store_get_packs(mad->repo->objects->packfiles); p; p = p->next) find_abbrev_len_for_pack(p, mad); } diff --git a/odb.c b/odb.c index 2a92a018c42940f131edd4e7cacd3454e66ee370..65a6cc67b61ccf99e7b7293906e6772f2927da9b 100644 --- a/odb.c +++ b/odb.c @@ -139,23 +139,21 @@ static void read_info_alternates(struct object_database *odb, const char *relative_base, int depth); -static int link_alt_odb_entry(struct object_database *odb, - const struct strbuf *entry, - const char *relative_base, - int depth, - const char *normalized_objdir) +static struct odb_source *link_alt_odb_entry(struct object_database *odb, + const char *dir, + const char *relative_base, + int depth) { - struct odb_source *alternate; + struct odb_source *alternate = NULL; struct strbuf pathbuf = STRBUF_INIT; struct strbuf tmp = STRBUF_INIT; khiter_t pos; - int ret = -1; - if (!is_absolute_path(entry->buf) && relative_base) { + if (!is_absolute_path(dir) && relative_base) { strbuf_realpath(&pathbuf, relative_base, 1); strbuf_addch(&pathbuf, '/'); } - strbuf_addbuf(&pathbuf, entry); + strbuf_addstr(&pathbuf, dir); if (!strbuf_realpath(&tmp, pathbuf.buf, 0)) { error(_("unable to normalize alternate object path: %s"), @@ -171,11 +169,15 @@ static int link_alt_odb_entry(struct object_database *odb, while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/') strbuf_setlen(&pathbuf, pathbuf.len - 1); - if (!alt_odb_usable(odb, &pathbuf, normalized_objdir, &pos)) + strbuf_reset(&tmp); + strbuf_realpath(&tmp, odb->sources->path, 1); + + if (!alt_odb_usable(odb, &pathbuf, tmp.buf, &pos)) goto error; CALLOC_ARRAY(alternate, 1); alternate->odb = odb; + alternate->local = false; /* pathbuf.buf is already in r->objects->source_by_path */ alternate->path = strbuf_detach(&pathbuf, NULL); @@ -188,11 +190,11 @@ static int link_alt_odb_entry(struct object_database *odb, /* recursively add alternates */ read_info_alternates(odb, alternate->path, depth + 1); - ret = 0; + error: strbuf_release(&tmp); strbuf_release(&pathbuf); - return ret; + return alternate; } static const char *parse_alt_odb_entry(const char *string, @@ -227,8 +229,7 @@ static const char *parse_alt_odb_entry(const char *string, static void link_alt_odb_entries(struct object_database *odb, const char *alt, int sep, const char *relative_base, int depth) { - struct strbuf objdirbuf = STRBUF_INIT; - struct strbuf entry = STRBUF_INIT; + struct strbuf dir = STRBUF_INIT; if (!alt || !*alt) return; @@ -239,17 +240,13 @@ static void link_alt_odb_entries(struct object_database *odb, const char *alt, return; } - strbuf_realpath(&objdirbuf, odb->sources->path, 1); - while (*alt) { - alt = parse_alt_odb_entry(alt, sep, &entry); - if (!entry.len) + alt = parse_alt_odb_entry(alt, sep, &dir); + if (!dir.len) continue; - link_alt_odb_entry(odb, &entry, - relative_base, depth, objdirbuf.buf); + link_alt_odb_entry(odb, dir.buf, relative_base, depth); } - strbuf_release(&entry); - strbuf_release(&objdirbuf); + strbuf_release(&dir); } static void read_info_alternates(struct object_database *odb, @@ -272,7 +269,7 @@ static void read_info_alternates(struct object_database *odb, } void odb_add_to_alternates_file(struct object_database *odb, - const char *reference) + const char *dir) { struct lock_file lock = LOCK_INIT; char *alts = repo_git_path(odb->repo, "objects/info/alternates"); @@ -289,7 +286,7 @@ void odb_add_to_alternates_file(struct object_database *odb, struct strbuf line = STRBUF_INIT; while (strbuf_getline(&line, in) != EOF) { - if (!strcmp(reference, line.buf)) { + if (!strcmp(dir, line.buf)) { found = 1; break; } @@ -305,27 +302,24 @@ void odb_add_to_alternates_file(struct object_database *odb, if (found) { rollback_lock_file(&lock); } else { - fprintf_or_die(out, "%s\n", reference); + fprintf_or_die(out, "%s\n", dir); if (commit_lock_file(&lock)) die_errno(_("unable to move new alternates file into place")); if (odb->loaded_alternates) - link_alt_odb_entries(odb, reference, - '\n', NULL, 0); + link_alt_odb_entries(odb, dir, '\n', NULL, 0); } free(alts); } -void odb_add_to_alternates_memory(struct object_database *odb, - const char *reference) +struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, + const char *dir) { /* * Make sure alternates are initialized, or else our entry may be * overwritten when they are. */ odb_prepare_alternates(odb); - - link_alt_odb_entries(odb, reference, - '\n', NULL, 0); + return link_alt_odb_entry(odb, dir, NULL, 0); } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, @@ -463,6 +457,12 @@ struct odb_source *odb_find_source(struct object_database *odb, const char *obj_ free(obj_dir_real); strbuf_release(&odb_path_real); + return source; +} + +struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir) +{ + struct odb_source *source = odb_find_source(odb, obj_dir); if (!source) die(_("could not find object directory matching %s"), obj_dir); return source; @@ -694,7 +694,7 @@ static int do_oid_object_info_extended(struct object_database *odb, /* Not a loose object; someone else may have just packed it. */ if (!(flags & OBJECT_INFO_QUICK)) { - reprepare_packed_git(odb->repo); + odb_reprepare(odb->repo->objects); if (find_pack_entry(odb->repo, real, &e)) break; } @@ -996,8 +996,7 @@ struct object_database *odb_new(struct repository *repo) memset(o, 0, sizeof(*o)); o->repo = repo; - INIT_LIST_HEAD(&o->packed_git_mru); - hashmap_init(&o->pack_map, pack_map_entry_cmp, NULL, 0); + o->packfiles = packfile_store_new(o); pthread_mutex_init(&o->replace_mutex, NULL); string_list_init_dup(&o->submodule_source_paths); return o; @@ -1035,19 +1034,34 @@ void odb_clear(struct object_database *o) free((char *) o->cached_objects[i].value.buf); FREE_AND_NULL(o->cached_objects); - INIT_LIST_HEAD(&o->packed_git_mru); close_object_store(o); + packfile_store_free(o->packfiles); + o->packfiles = NULL; + + string_list_clear(&o->submodule_source_paths, 0); +} + +void odb_reprepare(struct object_database *o) +{ + struct odb_source *source; + + obj_read_lock(); /* - * `close_object_store()` only closes the packfiles, but doesn't free - * them. We thus have to do this manually. + * Reprepare alt odbs, in case the alternates file was modified + * during the course of this process. This only _adds_ odbs to + * the linked list, so existing odbs will continue to exist for + * the lifetime of the process. */ - for (struct packed_git *p = o->packed_git, *next; p; p = next) { - next = p->next; - free(p); - } - o->packed_git = NULL; + o->loaded_alternates = 0; + odb_prepare_alternates(o); - hashmap_clear(&o->pack_map); - string_list_clear(&o->submodule_source_paths, 0); + for (source = o->sources; source; source = source->next) + odb_clear_loose_cache(source); + + o->approximate_object_count_valid = 0; + + packfile_store_reprepare(o->packfiles); + + obj_read_unlock(); } diff --git a/odb.h b/odb.h index 3dfc66d75a3d20dd64d787d04e3ab4e5f9010e8a..ab39e3605d541997d8b12fbf14bc9e78def515a5 100644 --- a/odb.h +++ b/odb.h @@ -3,7 +3,6 @@ #include "hashmap.h" #include "object.h" -#include "list.h" #include "oidset.h" #include "oidmap.h" #include "string-list.h" @@ -63,6 +62,14 @@ struct odb_source { */ struct multi_pack_index *midx; + /* + * Figure out whether this is the local source of the owning + * repository, which would typically be its ".git/objects" directory. + * This local object directory is usually where objects would be + * written to. + */ + bool local; + /* * This is a temporary object store created by the tmp_objdir * facility. Disable ref updates since the objects in the store @@ -83,6 +90,7 @@ struct odb_source { }; struct packed_git; +struct packfile_store; struct cached_object_entry; /* @@ -123,20 +131,8 @@ struct object_database { struct commit_graph *commit_graph; unsigned commit_graph_attempted : 1; /* if loading has been attempted */ - /* - * private data - * - * should only be accessed directly by packfile.c - */ - - struct packed_git *packed_git; - /* A most-recently-used ordered version of the packed_git list. */ - struct list_head packed_git_mru; - - struct { - struct packed_git **packs; - unsigned flags; - } kept_pack_cache; + /* Should only be accessed directly by packfile.c and midx.c. */ + struct packfile_store *packfiles; /* * This is meant to hold a *small* number of objects that you would @@ -147,12 +143,6 @@ struct object_database { struct cached_object_entry *cached_objects; size_t cached_object_nr, cached_object_alloc; - /* - * A map of packfiles to packed_git structs for tracking which - * packs have been loaded already. - */ - struct hashmap pack_map; - /* * A fast, rough count of the number of objects in the repository. * These two fields are not meant for direct access. Use @@ -161,12 +151,6 @@ struct object_database { unsigned long approximate_object_count; unsigned approximate_object_count_valid : 1; - /* - * Whether packed_git has already been populated with this repository's - * packs. - */ - unsigned packed_git_initialized : 1; - /* * Submodule source paths that will be added as additional sources to * allow lookup of submodule objects via the main object database. @@ -178,11 +162,20 @@ struct object_database *odb_new(struct repository *repo); void odb_clear(struct object_database *o); /* - * Find source by its object directory path. Dies in case the source couldn't - * be found. + * Clear caches, reload alternates and then reload object sources so that new + * objects may become accessible. + */ +void odb_reprepare(struct object_database *o); + +/* + * Find source by its object directory path. Returns a `NULL` pointer in case + * the source could not be found. */ struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir); +/* Same as `odb_find_source()`, but dies in case the source doesn't exist. */ +struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir); + /* * Replace the current writable object directory with the specified temporary * object directory; returns the former primary source. @@ -257,8 +250,8 @@ void odb_add_to_alternates_file(struct object_database *odb, * recursive alternates it points to), but do not modify the on-disk alternates * file. */ -void odb_add_to_alternates_memory(struct object_database *odb, - const char *dir); +struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, + const char *dir); /* * Read an object from the database. Returns the object data and assigns object diff --git a/pack-bitmap.c b/pack-bitmap.c index d14421ee20441487f1c9b6a43df746edea15b894..ac71035d7715f7f9a86468fd2754bb5283a80e41 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -216,7 +216,7 @@ static uint32_t bitmap_num_objects(struct bitmap_index *index) static struct repository *bitmap_repo(struct bitmap_index *bitmap_git) { if (bitmap_is_midx(bitmap_git)) - return bitmap_git->midx->repo; + return bitmap_git->midx->source->odb->repo; return bitmap_git->pack->repo; } @@ -418,13 +418,12 @@ char *midx_bitmap_filename(struct multi_pack_index *midx) { struct strbuf buf = STRBUF_INIT; if (midx->has_chain) - get_split_midx_filename_ext(midx->repo->hash_algo, &buf, - midx->object_dir, + get_split_midx_filename_ext(midx->source, &buf, get_midx_checksum(midx), MIDX_EXT_BITMAP); else - get_midx_filename_ext(midx->repo->hash_algo, &buf, - midx->object_dir, get_midx_checksum(midx), + get_midx_filename_ext(midx->source, &buf, + get_midx_checksum(midx), MIDX_EXT_BITMAP); return strbuf_detach(&buf, NULL); @@ -463,7 +462,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, if (bitmap_git->pack || bitmap_git->midx) { struct strbuf buf = STRBUF_INIT; - get_midx_filename(midx->repo->hash_algo, &buf, midx->object_dir); + get_midx_filename(midx->source, &buf); trace2_data_string("bitmap", bitmap_repo(bitmap_git), "ignoring extra midx bitmap file", buf.buf); close(fd); @@ -493,7 +492,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, } for (i = 0; i < bitmap_git->midx->num_packs + bitmap_git->midx->num_packs_in_base; i++) { - if (prepare_midx_pack(bitmap_repo(bitmap_git), bitmap_git->midx, i)) { + if (prepare_midx_pack(bitmap_git->midx, i)) { warning(_("could not open pack %s"), bitmap_git->midx->pack_names[i]); goto cleanup; @@ -665,7 +664,7 @@ static int open_pack_bitmap(struct repository *r, struct packed_git *p; int ret = -1; - for (p = get_all_packs(r); p; p = p->next) { + for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { if (open_pack_bitmap_1(bitmap_git, p) == 0) { ret = 0; /* @@ -2466,7 +2465,7 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, struct multi_pack_index *m = bitmap_git->midx; for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) { struct bitmapped_pack pack; - if (nth_bitmapped_pack(r, bitmap_git->midx, &pack, i) < 0) { + if (nth_bitmapped_pack(bitmap_git->midx, &pack, i) < 0) { warning(_("unable to load pack: '%s', disabling pack-reuse"), bitmap_git->midx->pack_names[i]); free(packs); @@ -3363,7 +3362,7 @@ int verify_bitmap_files(struct repository *r) free(midx_bitmap_name); } - for (struct packed_git *p = get_all_packs(r); + for (struct packed_git *p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { char *pack_bitmap_name = pack_bitmap_filename(p); res |= verify_bitmap_file(r->hash_algo, pack_bitmap_name); diff --git a/pack-objects.c b/pack-objects.c index a9d9855063aea85f4b1b8f70f301c644c5d5e225..d8eb679735484a17196ba9d77ac6c8c7bfff2536 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -86,6 +86,7 @@ struct object_entry *packlist_find(struct packing_data *pdata, static void prepare_in_pack_by_idx(struct packing_data *pdata) { + struct packfile_store *packs = pdata->repo->objects->packfiles; struct packed_git **mapping, *p; int cnt = 0, nr = 1U << OE_IN_PACK_BITS; @@ -95,7 +96,7 @@ static void prepare_in_pack_by_idx(struct packing_data *pdata) * (i.e. in_pack_idx also zero) should return NULL. */ mapping[cnt++] = NULL; - for (p = get_all_packs(pdata->repo); p; p = p->next, cnt++) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next, cnt++) { if (cnt == nr) { free(mapping); return; diff --git a/pack-revindex.c b/pack-revindex.c index 0cc422a1e67bc84477daf3526edebcce3a05eb39..d0791cc4938fa2c784a1a585210552ee2c6d06fa 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -379,25 +379,25 @@ int load_midx_revindex(struct multi_pack_index *m) * not want to accidentally call munmap() in the middle of the * MIDX. */ - trace2_data_string("load_midx_revindex", m->repo, + trace2_data_string("load_midx_revindex", m->source->odb->repo, "source", "midx"); m->revindex_data = (const uint32_t *)m->chunk_revindex; return 0; } - trace2_data_string("load_midx_revindex", m->repo, + trace2_data_string("load_midx_revindex", m->source->odb->repo, "source", "rev"); if (m->has_chain) - get_split_midx_filename_ext(m->repo->hash_algo, &revindex_name, - m->object_dir, get_midx_checksum(m), + get_split_midx_filename_ext(m->source, &revindex_name, + get_midx_checksum(m), MIDX_EXT_REV); else - get_midx_filename_ext(m->repo->hash_algo, &revindex_name, - m->object_dir, get_midx_checksum(m), + get_midx_filename_ext(m->source, &revindex_name, + get_midx_checksum(m), MIDX_EXT_REV); - ret = load_revindex_from_disk(m->repo->hash_algo, + ret = load_revindex_from_disk(m->source->odb->repo->hash_algo, revindex_name.buf, m->num_objects, &m->revindex_map, diff --git a/packfile.c b/packfile.c index 5d73932f50ce6854d695e92d049c6a6c0a1421a7..5a7caec29259771d2f15e9ac4979c0eec964251a 100644 --- a/packfile.c +++ b/packfile.c @@ -278,7 +278,7 @@ static int unuse_one_window(struct packed_git *current) if (current) scan_windows(current, &lru_p, &lru_w, &lru_l); - for (p = current->repo->objects->packed_git; p; p = p->next) + for (p = current->repo->objects->packfiles->packs; p; p = p->next) scan_windows(p, &lru_p, &lru_w, &lru_l); if (lru_p) { munmap(lru_w->base, lru_w->len); @@ -362,13 +362,8 @@ void close_pack(struct packed_git *p) void close_object_store(struct object_database *o) { struct odb_source *source; - struct packed_git *p; - for (p = o->packed_git; p; p = p->next) - if (p->do_not_close) - BUG("want to close pack marked 'do-not-close'"); - else - close_pack(p); + packfile_store_close(o->packfiles); for (source = o->sources; source; source = source->next) { if (source->midx) @@ -468,7 +463,7 @@ static int close_one_pack(struct repository *r) struct pack_window *mru_w = NULL; int accept_windows_inuse = 1; - for (p = r->objects->packed_git; p; p = p->next) { + for (p = r->objects->packfiles->packs; p; p = p->next) { if (p->pack_fd == -1) continue; find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse); @@ -784,16 +779,44 @@ struct packed_git *add_packed_git(struct repository *r, const char *path, return p; } -void install_packed_git(struct repository *r, struct packed_git *pack) +void packfile_store_add_pack(struct packfile_store *store, + struct packed_git *pack) { if (pack->pack_fd != -1) pack_open_fds++; - pack->next = r->objects->packed_git; - r->objects->packed_git = pack; + pack->next = store->packs; + store->packs = pack; hashmap_entry_init(&pack->packmap_ent, strhash(pack->pack_name)); - hashmap_add(&r->objects->pack_map, &pack->packmap_ent); + hashmap_add(&store->map, &pack->packmap_ent); +} + +struct packed_git *packfile_store_load_pack(struct packfile_store *store, + const char *idx_path, int local) +{ + struct strbuf key = STRBUF_INIT; + struct packed_git *p; + + /* + * We're being called with the path to the index file, but `pack_map` + * holds the path to the packfile itself. + */ + strbuf_addstr(&key, idx_path); + strbuf_strip_suffix(&key, ".idx"); + strbuf_addstr(&key, ".pack"); + + p = hashmap_get_entry_from_hash(&store->map, strhash(key.buf), key.buf, + struct packed_git, packmap_ent); + if (!p) { + p = add_packed_git(store->odb->repo, idx_path, + strlen(idx_path), local); + if (p) + packfile_store_add_pack(store, p); + } + + strbuf_release(&key); + return p; } void (*report_garbage)(unsigned seen_bits, const char *path); @@ -895,23 +918,14 @@ static void prepare_pack(const char *full_name, size_t full_name_len, const char *file_name, void *_data) { struct prepare_pack_data *data = (struct prepare_pack_data *)_data; - struct packed_git *p; size_t base_len = full_name_len; if (strip_suffix_mem(full_name, &base_len, ".idx") && !(data->m && midx_contains_pack(data->m, file_name))) { - struct hashmap_entry hent; - char *pack_name = xstrfmt("%.*s.pack", (int)base_len, full_name); - unsigned int hash = strhash(pack_name); - hashmap_entry_init(&hent, hash); - - /* Don't reopen a pack we already have. */ - if (!hashmap_get(&data->r->objects->pack_map, &hent, pack_name)) { - p = add_packed_git(data->r, full_name, full_name_len, data->local); - if (p) - install_packed_git(data->r, p); - } - free(pack_name); + char *trimmed_path = xstrndup(full_name, full_name_len); + packfile_store_load_pack(data->r->objects->packfiles, + trimmed_path, data->local); + free(trimmed_path); } if (!report_garbage) @@ -935,14 +949,14 @@ static void prepare_pack(const char *full_name, size_t full_name_len, report_garbage(PACKDIR_FILE_GARBAGE, full_name); } -static void prepare_packed_git_one(struct odb_source *source, int local) +static void prepare_packed_git_one(struct odb_source *source) { struct string_list garbage = STRING_LIST_INIT_DUP; struct prepare_pack_data data = { .m = source->midx, .r = source->odb->repo, .garbage = &garbage, - .local = local, + .local = source->local, }; for_each_file_in_pack_dir(source->path, prepare_pack, &data); @@ -951,40 +965,6 @@ static void prepare_packed_git_one(struct odb_source *source, int local) string_list_clear(data.garbage, 0); } -static void prepare_packed_git(struct repository *r); -/* - * Give a fast, rough count of the number of objects in the repository. This - * ignores loose objects completely. If you have a lot of them, then either - * you should repack because your performance will be awful, or they are - * all unreachable objects about to be pruned, in which case they're not really - * interesting as a measure of repo size in the first place. - */ -unsigned long repo_approximate_object_count(struct repository *r) -{ - if (!r->objects->approximate_object_count_valid) { - struct odb_source *source; - unsigned long count = 0; - struct packed_git *p; - - prepare_packed_git(r); - - for (source = r->objects->sources; source; source = source->next) { - struct multi_pack_index *m = get_multi_pack_index(source); - if (m) - count += m->num_objects; - } - - for (p = r->objects->packed_git; p; p = p->next) { - if (open_pack_index(p)) - continue; - count += p->num_objects; - } - r->objects->approximate_object_count = count; - r->objects->approximate_object_count_valid = 1; - } - return r->objects->approximate_object_count; -} - DEFINE_LIST_SORT(static, sort_packs, struct packed_git, next); static int sort_pack(const struct packed_git *a, const struct packed_git *b) @@ -1013,95 +993,98 @@ static int sort_pack(const struct packed_git *a, const struct packed_git *b) return -1; } -static void rearrange_packed_git(struct repository *r) -{ - sort_packs(&r->objects->packed_git, sort_pack); -} - -static void prepare_packed_git_mru(struct repository *r) +static void packfile_store_prepare_mru(struct packfile_store *store) { struct packed_git *p; - INIT_LIST_HEAD(&r->objects->packed_git_mru); + INIT_LIST_HEAD(&store->mru); - for (p = r->objects->packed_git; p; p = p->next) - list_add_tail(&p->mru, &r->objects->packed_git_mru); + for (p = store->packs; p; p = p->next) + list_add_tail(&p->mru, &store->mru); } -static void prepare_packed_git(struct repository *r) +void packfile_store_prepare(struct packfile_store *store) { struct odb_source *source; - if (r->objects->packed_git_initialized) + if (store->initialized) return; - odb_prepare_alternates(r->objects); - for (source = r->objects->sources; source; source = source->next) { - int local = (source == r->objects->sources); - prepare_multi_pack_index_one(source, local); - prepare_packed_git_one(source, local); + odb_prepare_alternates(store->odb); + for (source = store->odb->sources; source; source = source->next) { + prepare_multi_pack_index_one(source); + prepare_packed_git_one(source); } - rearrange_packed_git(r); - - prepare_packed_git_mru(r); - r->objects->packed_git_initialized = 1; -} - -void reprepare_packed_git(struct repository *r) -{ - struct odb_source *source; + sort_packs(&store->packs, sort_pack); - obj_read_lock(); - - /* - * Reprepare alt odbs, in case the alternates file was modified - * during the course of this process. This only _adds_ odbs to - * the linked list, so existing odbs will continue to exist for - * the lifetime of the process. - */ - r->objects->loaded_alternates = 0; - odb_prepare_alternates(r->objects); - - for (source = r->objects->sources; source; source = source->next) - odb_clear_loose_cache(source); - - r->objects->approximate_object_count_valid = 0; - r->objects->packed_git_initialized = 0; - prepare_packed_git(r); - obj_read_unlock(); + packfile_store_prepare_mru(store); + store->initialized = true; } -struct packed_git *get_packed_git(struct repository *r) +void packfile_store_reprepare(struct packfile_store *store) { - prepare_packed_git(r); - return r->objects->packed_git; + store->initialized = false; + packfile_store_prepare(store); } -struct multi_pack_index *get_multi_pack_index(struct odb_source *source) +struct packed_git *packfile_store_get_packs(struct packfile_store *store) { - prepare_packed_git(source->odb->repo); - return source->midx; + packfile_store_prepare(store); + return store->packs; } -struct packed_git *get_all_packs(struct repository *r) +struct packed_git *packfile_store_get_all_packs(struct packfile_store *store) { - prepare_packed_git(r); + packfile_store_prepare(store); - for (struct odb_source *source = r->objects->sources; source; source = source->next) { + for (struct odb_source *source = store->odb->sources; source; source = source->next) { struct multi_pack_index *m = source->midx; if (!m) continue; for (uint32_t i = 0; i < m->num_packs + m->num_packs_in_base; i++) - prepare_midx_pack(r, m, i); + prepare_midx_pack(m, i); } - return r->objects->packed_git; + return store->packs; } -struct list_head *get_packed_git_mru(struct repository *r) +struct list_head *packfile_store_get_packs_mru(struct packfile_store *store) { - prepare_packed_git(r); - return &r->objects->packed_git_mru; + packfile_store_prepare(store); + return &store->mru; +} + +/* + * Give a fast, rough count of the number of objects in the repository. This + * ignores loose objects completely. If you have a lot of them, then either + * you should repack because your performance will be awful, or they are + * all unreachable objects about to be pruned, in which case they're not really + * interesting as a measure of repo size in the first place. + */ +unsigned long repo_approximate_object_count(struct repository *r) +{ + if (!r->objects->approximate_object_count_valid) { + struct odb_source *source; + unsigned long count = 0; + struct packed_git *p; + + packfile_store_prepare(r->objects->packfiles); + + for (source = r->objects->sources; source; source = source->next) { + struct multi_pack_index *m = get_multi_pack_index(source); + if (m) + count += m->num_objects; + } + + for (p = r->objects->packfiles->packs; p; p = p->next) { + if (open_pack_index(p)) + continue; + count += p->num_objects; + } + r->objects->approximate_object_count = count; + r->objects->approximate_object_count_valid = 1; + } + return r->objects->approximate_object_count; } unsigned long unpack_object_header_buffer(const unsigned char *buf, @@ -1156,7 +1139,7 @@ unsigned long get_size_from_delta(struct packed_git *p, * * Other worrying sections could be the call to close_pack_fd(), * which can close packs even with in-use windows, and to - * reprepare_packed_git(). Regarding the former, mmap doc says: + * odb_reprepare(). Regarding the former, mmap doc says: * "closing the file descriptor does not unmap the region". And * for the latter, it won't re-open already available packs. */ @@ -1220,7 +1203,7 @@ const struct packed_git *has_packed_and_bad(struct repository *r, { struct packed_git *p; - for (p = r->objects->packed_git; p; p = p->next) + for (p = r->objects->packfiles->packs; p; p = p->next) if (oidset_contains(&p->bad_objects, oid)) return p; return NULL; @@ -2075,19 +2058,19 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa { struct list_head *pos; - prepare_packed_git(r); + packfile_store_prepare(r->objects->packfiles); for (struct odb_source *source = r->objects->sources; source; source = source->next) - if (source->midx && fill_midx_entry(r, oid, e, source->midx)) + if (source->midx && fill_midx_entry(source->midx, oid, e)) return 1; - if (!r->objects->packed_git) + if (!r->objects->packfiles->packs) return 0; - list_for_each(pos, &r->objects->packed_git_mru) { + list_for_each(pos, &r->objects->packfiles->mru) { struct packed_git *p = list_entry(pos, struct packed_git, mru); if (!p->multi_pack_index && fill_pack_entry(oid, e, p)) { - list_move(&p->mru, &r->objects->packed_git_mru); + list_move(&p->mru, &r->objects->packfiles->mru); return 1; } } @@ -2097,19 +2080,19 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa static void maybe_invalidate_kept_pack_cache(struct repository *r, unsigned flags) { - if (!r->objects->kept_pack_cache.packs) + if (!r->objects->packfiles->kept_cache.packs) return; - if (r->objects->kept_pack_cache.flags == flags) + if (r->objects->packfiles->kept_cache.flags == flags) return; - FREE_AND_NULL(r->objects->kept_pack_cache.packs); - r->objects->kept_pack_cache.flags = 0; + FREE_AND_NULL(r->objects->packfiles->kept_cache.packs); + r->objects->packfiles->kept_cache.flags = 0; } struct packed_git **kept_pack_cache(struct repository *r, unsigned flags) { maybe_invalidate_kept_pack_cache(r, flags); - if (!r->objects->kept_pack_cache.packs) { + if (!r->objects->packfiles->kept_cache.packs) { struct packed_git **packs = NULL; size_t nr = 0, alloc = 0; struct packed_git *p; @@ -2122,7 +2105,7 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags) * covers, one kept and one not kept, but the midx returns only * the non-kept version. */ - for (p = get_all_packs(r); p; p = p->next) { + for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { if ((p->pack_keep && (flags & ON_DISK_KEEP_PACKS)) || (p->pack_keep_in_core && (flags & IN_CORE_KEEP_PACKS))) { ALLOC_GROW(packs, nr + 1, alloc); @@ -2132,11 +2115,11 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags) ALLOC_GROW(packs, nr + 1, alloc); packs[nr] = NULL; - r->objects->kept_pack_cache.packs = packs; - r->objects->kept_pack_cache.flags = flags; + r->objects->packfiles->kept_cache.packs = packs; + r->objects->packfiles->kept_cache.flags = flags; } - return r->objects->kept_pack_cache.packs; + return r->objects->packfiles->kept_cache.packs; } int find_kept_pack_entry(struct repository *r, @@ -2219,7 +2202,7 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, int r = 0; int pack_errors = 0; - for (p = get_all_packs(repo); p; p = p->next) { + for (p = packfile_store_get_all_packs(repo->objects->packfiles); p; p = p->next) { if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) continue; if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) && @@ -2333,3 +2316,46 @@ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *l *len = hdr - out; return 0; } + +static int pack_map_entry_cmp(const void *cmp_data UNUSED, + const struct hashmap_entry *entry, + const struct hashmap_entry *entry2, + const void *keydata) +{ + const char *key = keydata; + const struct packed_git *pg1, *pg2; + + pg1 = container_of(entry, const struct packed_git, packmap_ent); + pg2 = container_of(entry2, const struct packed_git, packmap_ent); + + return strcmp(pg1->pack_name, key ? key : pg2->pack_name); +} + +struct packfile_store *packfile_store_new(struct object_database *odb) +{ + struct packfile_store *store; + CALLOC_ARRAY(store, 1); + store->odb = odb; + INIT_LIST_HEAD(&store->mru); + hashmap_init(&store->map, pack_map_entry_cmp, NULL, 0); + return store; +} + +void packfile_store_free(struct packfile_store *store) +{ + for (struct packed_git *p = store->packs, *next; p; p = next) { + next = p->next; + free(p); + } + hashmap_clear(&store->map); + free(store); +} + +void packfile_store_close(struct packfile_store *store) +{ + for (struct packed_git *p = store->packs; p; p = p->next) { + if (p->do_not_close) + BUG("want to close pack marked 'do-not-close'"); + close_pack(p); + } +} diff --git a/packfile.h b/packfile.h index f16753f2a9bb4c18e832041a9145a3a5c79f492c..e7a5792b6cf691453e116ce541abf6e3280066f6 100644 --- a/packfile.h +++ b/packfile.h @@ -52,19 +52,114 @@ struct packed_git { char pack_name[FLEX_ARRAY]; /* more */ }; -static inline int pack_map_entry_cmp(const void *cmp_data UNUSED, - const struct hashmap_entry *entry, - const struct hashmap_entry *entry2, - const void *keydata) -{ - const char *key = keydata; - const struct packed_git *pg1, *pg2; +/* + * A store that manages packfiles for a given object database. + */ +struct packfile_store { + struct object_database *odb; - pg1 = container_of(entry, const struct packed_git, packmap_ent); - pg2 = container_of(entry2, const struct packed_git, packmap_ent); + /* + * The list of packfiles in the order in which they are being added to + * the store. + */ + struct packed_git *packs; - return strcmp(pg1->pack_name, key ? key : pg2->pack_name); -} + /* + * Cache of packfiles which are marked as "kept", either because there + * is an on-disk ".keep" file or because they are marked as "kept" in + * memory. + * + * Should not be accessed directly, but via `kept_pack_cache()`. The + * list of packs gets invalidated when the stored flags and the flags + * passed to `kept_pack_cache()` mismatch. + */ + struct { + struct packed_git **packs; + unsigned flags; + } kept_cache; + + /* A most-recently-used ordered version of the packs list. */ + struct list_head mru; + + /* + * A map of packfile names to packed_git structs for tracking which + * packs have been loaded already. + */ + struct hashmap map; + + /* + * Whether packfiles have already been populated with this store's + * packs. + */ + bool initialized; +}; + +/* + * Allocate and initialize a new empty packfile store for the given object + * database. + */ +struct packfile_store *packfile_store_new(struct object_database *odb); + +/* + * Free the packfile store and all its associated state. All packfiles + * tracked by the store will be closed. + */ +void packfile_store_free(struct packfile_store *store); + +/* + * Close all packfiles associated with this store. The packfiles won't be + * free'd, so they can be re-opened at a later point in time. + */ +void packfile_store_close(struct packfile_store *store); + +/* + * Prepare the packfile store by loading packfiles and multi-pack indices for + * all alternates. This becomes a no-op if the store is already prepared. + * + * It shouldn't typically be necessary to call this function directly, as + * functions that access the store know to prepare it. + */ +void packfile_store_prepare(struct packfile_store *store); + +/* + * Clear the packfile caches and try to look up any new packfiles that have + * appeared since last preparing the packfiles store. + * + * This function must be called under the `odb_read_lock()`. + */ +void packfile_store_reprepare(struct packfile_store *store); + +/* + * Add the pack to the store so that contained objects become accessible via + * the store. This moves ownership into the store. + */ +void packfile_store_add_pack(struct packfile_store *store, + struct packed_git *pack); + +/* + * Get packs managed by the given store. Does not load the MIDX or any packs + * referenced by it. + */ +struct packed_git *packfile_store_get_packs(struct packfile_store *store); + +/* + * Get all packs managed by the given store, including packfiles that are + * referenced by multi-pack indices. + */ +struct packed_git *packfile_store_get_all_packs(struct packfile_store *store); + +/* + * Get all packs in most-recently-used order. + */ +struct list_head *packfile_store_get_packs_mru(struct packfile_store *store); + +/* + * Open the packfile and add it to the store if it isn't yet known. Returns + * either the newly opened packfile or the preexisting packfile. Returns a + * `NULL` pointer in case the packfile could not be opened. + */ +struct packed_git *packfile_store_load_pack(struct packfile_store *store, + const char *idx_path, int local); struct pack_window { struct pack_window *next; @@ -142,14 +237,6 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, #define PACKDIR_FILE_GARBAGE 4 extern void (*report_garbage)(unsigned seen_bits, const char *path); -void reprepare_packed_git(struct repository *r); -void install_packed_git(struct repository *r, struct packed_git *pack); - -struct packed_git *get_packed_git(struct repository *r); -struct list_head *get_packed_git_mru(struct repository *r); -struct multi_pack_index *get_multi_pack_index(struct odb_source *source); -struct packed_git *get_all_packs(struct repository *r); - /* * Give a rough count of objects in the repository. This sacrifices accuracy * for speed. diff --git a/repository.c b/repository.c index ecd691181fc97d59241a53765d898300b1f9cf99..97f0578381d894f6bbd417d3ad025937f40d812f 100644 --- a/repository.c +++ b/repository.c @@ -168,6 +168,7 @@ void repo_set_gitdir(struct repository *repo, if (!repo->objects->sources) { CALLOC_ARRAY(repo->objects->sources, 1); repo->objects->sources->odb = repo->objects; + repo->objects->sources->local = true; repo->objects->sources_tail = &repo->objects->sources->next; } expand_base_dir(&repo->objects->sources->path, o->object_dir, diff --git a/server-info.c b/server-info.c index 9bb30d9ab71d2231905a031a7e8f173e31585a67..1d33de821e9f5e5b470dd618127b8354fb580009 100644 --- a/server-info.c +++ b/server-info.c @@ -287,12 +287,13 @@ static int compare_info(const void *a_, const void *b_) static void init_pack_info(struct repository *r, const char *infofile, int force) { + struct packfile_store *packs = r->objects->packfiles; struct packed_git *p; int stale; int i; size_t alloc = 0; - for (p = get_all_packs(r); p; p = p->next) { + for (p = packfile_store_get_all_packs(packs); p; p = p->next) { /* we ignore things on alternate path since they are * not available to the pullers in general. */ diff --git a/t/helper/test-find-pack.c b/t/helper/test-find-pack.c index 611a13a32610d2f291d89ef1aca12aab1383c335..e001dc3066db70ed08da78f1566eb9f764d93619 100644 --- a/t/helper/test-find-pack.c +++ b/t/helper/test-find-pack.c @@ -39,7 +39,7 @@ int cmd__find_pack(int argc, const char **argv) if (repo_get_oid(the_repository, argv[0], &oid)) die("cannot parse %s as an object name", argv[0]); - for (p = get_all_packs(the_repository); p; p = p->next) + for (p = packfile_store_get_all_packs(the_repository->objects->packfiles); p; p = p->next) if (find_pack_entry_one(&oid, p)) { printf("%s\n", p->pack_name); actual_count++; diff --git a/t/helper/test-pack-mtimes.c b/t/helper/test-pack-mtimes.c index d51aaa3dc40d1287c60d2058b94404df777e7ee4..7c428c16011a23d18855e87865e1d21d99bb27e0 100644 --- a/t/helper/test-pack-mtimes.c +++ b/t/helper/test-pack-mtimes.c @@ -37,7 +37,7 @@ int cmd__pack_mtimes(int argc, const char **argv) if (argc != 2) usage(pack_mtimes_usage); - for (p = get_all_packs(the_repository); p; p = p->next) { + for (p = packfile_store_get_all_packs(the_repository->objects->packfiles); p; p = p->next) { strbuf_addstr(&buf, basename(p->pack_name)); strbuf_strip_suffix(&buf, ".pack"); strbuf_addstr(&buf, ".mtimes"); diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index da2aa036b57ef6f0c96b1e15d711fac125c2d687..6de5d1665afbfc8a6bee29983107cd1c8e28a98b 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -11,14 +11,24 @@ #include "gettext.h" #include "pack-revindex.h" +static struct multi_pack_index *setup_midx(const char *object_dir) +{ + struct odb_source *source; + setup_git_directory(); + source = odb_find_source(the_repository->objects, object_dir); + if (!source) + source = odb_add_to_alternates_memory(the_repository->objects, + object_dir); + return load_multi_pack_index(source); +} + static int read_midx_file(const char *object_dir, const char *checksum, int show_objects) { uint32_t i; struct multi_pack_index *m; - setup_git_directory(); - m = load_multi_pack_index(the_repository, object_dir, 1); + m = setup_midx(object_dir); if (!m) return 1; @@ -56,7 +66,7 @@ static int read_midx_file(const char *object_dir, const char *checksum, for (i = 0; i < m->num_packs; i++) printf("%s\n", m->pack_names[i]); - printf("object-dir: %s\n", m->object_dir); + printf("object-dir: %s\n", m->source->path); if (show_objects) { struct object_id oid; @@ -65,7 +75,7 @@ static int read_midx_file(const char *object_dir, const char *checksum, for (i = 0; i < m->num_objects; i++) { nth_midxed_object_oid(&oid, m, i + m->num_objects_in_base); - fill_midx_entry(the_repository, &oid, &e, m); + fill_midx_entry(m, &oid, &e); printf("%s %"PRIu64"\t%s\n", oid_to_hex(&oid), e.offset, e.p->pack_name); @@ -81,8 +91,7 @@ static int read_midx_checksum(const char *object_dir) { struct multi_pack_index *m; - setup_git_directory(); - m = load_multi_pack_index(the_repository, object_dir, 1); + m = setup_midx(object_dir); if (!m) return 1; printf("%s\n", hash_to_hex(get_midx_checksum(m))); @@ -96,9 +105,7 @@ static int read_midx_preferred_pack(const char *object_dir) struct multi_pack_index *midx = NULL; uint32_t preferred_pack; - setup_git_directory(); - - midx = load_multi_pack_index(the_repository, object_dir, 1); + midx = setup_midx(object_dir); if (!midx) return 1; @@ -119,14 +126,12 @@ static int read_midx_bitmapped_packs(const char *object_dir) struct bitmapped_pack pack; uint32_t i; - setup_git_directory(); - - midx = load_multi_pack_index(the_repository, object_dir, 1); + midx = setup_midx(object_dir); if (!midx) return 1; for (i = 0; i < midx->num_packs + midx->num_packs_in_base; i++) { - if (nth_bitmapped_pack(the_repository, midx, &pack, i) < 0) { + if (nth_bitmapped_pack(midx, &pack, i) < 0) { close_midx(midx); return 1; } diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index bd75dea9501ed7ba98fd23ed24da8843022383b7..4e5e882989fc45529881d6365f3b78879d529dbd 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -28,11 +28,11 @@ midx_read_expect () { EOF if test $NUM_PACKS -ge 1 then - ls $OBJECT_DIR/pack/ | grep idx | sort + ls "$OBJECT_DIR"/pack/ | grep idx | sort fi && printf "object-dir: $OBJECT_DIR\n" } >expect && - test-tool read-midx $OBJECT_DIR >actual && + test-tool read-midx "$OBJECT_DIR" >actual && test_cmp expect actual } @@ -305,7 +305,7 @@ test_expect_success 'midx picks objects from preferred pack' ' ofs=$(git show-index expect && grep ^$b out >actual && @@ -639,7 +639,7 @@ test_expect_success 'force some 64-bit offsets with pack-objects' ' ( cd ../objects64 && pwd ) >.git/objects/info/alternates && midx64=$(git multi-pack-index --object-dir=../objects64 write) ) && - midx_read_expect 1 63 5 objects64 " large-offsets" + midx_read_expect 1 63 5 "$(pwd)/objects64" " large-offsets" ' test_expect_success 'verify multi-pack-index with 64-bit offsets' ' diff --git a/transport-helper.c b/transport-helper.c index 0789e5bca53282d29f30eea9a2eca129e115f40f..4d95d84f9e4d05db5117016bcdacadf3a4fe46b2 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -450,7 +450,7 @@ static int fetch_with_fetch(struct transport *transport, } strbuf_release(&buf); - reprepare_packed_git(the_repository); + odb_reprepare(the_repository->objects); return 0; }