From 2141a2a62b23f8f68e97837f1795100b09e9c652 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Tue, 23 Jan 2024 15:06:52 +0100 Subject: [PATCH 1/4] refs: introduce `is_pseudoref()` and `is_headref()` Introduce two new functions `is_pseudoref()` and `is_headref()`. This provides the necessary functionality for us to add pseudorefs and HEAD to the loose ref cache in the files backend, allowing us to build tooling to print these refs. The `is_pseudoref()` function internally calls `is_pseudoref_syntax()` but adds onto it by also checking to ensure that the pseudoref either ends with a "_HEAD" suffix or matches a list of exceptions. After which we also parse the contents of the pseudoref to ensure that it conforms to the ref format. We cannot directly add the new syntax checks to `is_pseudoref_syntax()` because the function is also used by `is_current_worktree_ref()` and making it stricter to match only known pseudorefs might have unintended consequences due to files like 'BISECT_START' which isn't a pseudoref but sometimes contains object ID. Keeping this in mind, we leave `is_pseudoref_syntax()` as is and create `is_pseudoref()` which is stricter. Ideally we'd want to move the new syntax checks to `is_pseudoref_syntax()` but a prerequisite for this would be to actually remove the exception list by converting those pseudorefs to also contain a '_HEAD' suffix and perhaps move bisect related files like 'BISECT_START' to a new directory similar to the 'rebase-merge' directory. Signed-off-by: Karthik Nayak --- refs.c | 39 +++++++++++++++++++++++++++++++++++++++ refs.h | 3 +++ 2 files changed, 42 insertions(+) diff --git a/refs.c b/refs.c index 20e8f1ff1f1..559f5aeea82 100644 --- a/refs.c +++ b/refs.c @@ -859,6 +859,45 @@ static int is_pseudoref_syntax(const char *refname) return 1; } +int is_pseudoref(struct ref_store *refs, const char *refname) +{ + static const char *const irregular_pseudorefs[] = { + "AUTO_MERGE", + "BISECT_EXPECTED_REV", + "NOTES_MERGE_PARTIAL", + "NOTES_MERGE_REF", + "MERGE_AUTOSTASH", + }; + struct object_id oid; + size_t i; + + if (!is_pseudoref_syntax(refname)) + return 0; + + if (ends_with(refname, "_HEAD")) { + read_ref_full(refname, RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, + &oid, NULL); + return !is_null_oid(&oid); + } + + for (i = 0; i < ARRAY_SIZE(irregular_pseudorefs); i++) + if (!strcmp(refname, irregular_pseudorefs[i])) { + read_ref_full(refname, RESOLVE_REF_READING | RESOLVE_REF_NO_RECURSE, + &oid, NULL); + return !is_null_oid(&oid); + } + + return 0; +} + +int is_headref(struct ref_store *refs, const char *refname) +{ + if (!strcmp(refname, "HEAD")) + return refs_ref_exists(refs, refname); + + return 0; +} + static int is_current_worktree_ref(const char *ref) { return is_pseudoref_syntax(ref) || is_per_worktree_ref(ref); } diff --git a/refs.h b/refs.h index 11b3b6cceaf..46b8085d632 100644 --- a/refs.h +++ b/refs.h @@ -1021,4 +1021,7 @@ extern struct ref_namespace_info ref_namespace[NAMESPACE__COUNT]; */ void update_ref_namespace(enum ref_namespace namespace, char *ref); +int is_pseudoref(struct ref_store *refs, const char *refname); +int is_headref(struct ref_store *refs, const char *refname); + #endif /* REFS_H */ -- GitLab From c96f0a9c8340ca8a7c43d43de8c67afc1659aaf1 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Fri, 5 Jan 2024 16:04:06 +0100 Subject: [PATCH 2/4] refs: extract out `loose_fill_ref_dir_regular_file()` Extract out the code for adding a single file to the loose ref dir as `loose_fill_ref_dir_regular_file()` from `loose_fill_ref_dir()` in `refs/files-backend.c`. This allows us to use this function independently in the following commits where we add code to also add pseudorefs to the ref dir. Signed-off-by: Karthik Nayak --- refs/files-backend.c | 62 +++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/refs/files-backend.c b/refs/files-backend.c index b288fc97dba..22495a48071 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -229,6 +229,38 @@ static void add_per_worktree_entries_to_dir(struct ref_dir *dir, const char *dir } } +static void loose_fill_ref_dir_regular_file(struct files_ref_store *refs, + const char *refname, + struct ref_dir *dir) +{ + struct object_id oid; + int flag; + + if (!refs_resolve_ref_unsafe(&refs->base, refname, RESOLVE_REF_READING, + &oid, &flag)) { + oidclr(&oid); + flag |= REF_ISBROKEN; + } else if (is_null_oid(&oid)) { + /* + * It is so astronomically unlikely + * that null_oid is the OID of an + * actual object that we consider its + * appearance in a loose reference + * file to be repo corruption + * (probably due to a software bug). + */ + flag |= REF_ISBROKEN; + } + + if (check_refname_format(refname, REFNAME_ALLOW_ONELEVEL)) { + if (!refname_is_safe(refname)) + die("loose refname is dangerous: %s", refname); + oidclr(&oid); + flag |= REF_BAD_NAME | REF_ISBROKEN; + } + add_entry_to_dir(dir, create_ref_entry(refname, &oid, flag)); +} + /* * Read the loose references from the namespace dirname into dir * (without recursing). dirname must end with '/'. dir must be the @@ -257,8 +289,6 @@ static void loose_fill_ref_dir(struct ref_store *ref_store, strbuf_add(&refname, dirname, dirnamelen); while ((de = readdir(d)) != NULL) { - struct object_id oid; - int flag; unsigned char dtype; if (de->d_name[0] == '.') @@ -274,33 +304,7 @@ static void loose_fill_ref_dir(struct ref_store *ref_store, create_dir_entry(dir->cache, refname.buf, refname.len)); } else if (dtype == DT_REG) { - if (!refs_resolve_ref_unsafe(&refs->base, - refname.buf, - RESOLVE_REF_READING, - &oid, &flag)) { - oidclr(&oid); - flag |= REF_ISBROKEN; - } else if (is_null_oid(&oid)) { - /* - * It is so astronomically unlikely - * that null_oid is the OID of an - * actual object that we consider its - * appearance in a loose reference - * file to be repo corruption - * (probably due to a software bug). - */ - flag |= REF_ISBROKEN; - } - - if (check_refname_format(refname.buf, - REFNAME_ALLOW_ONELEVEL)) { - if (!refname_is_safe(refname.buf)) - die("loose refname is dangerous: %s", refname.buf); - oidclr(&oid); - flag |= REF_BAD_NAME | REF_ISBROKEN; - } - add_entry_to_dir(dir, - create_ref_entry(refname.buf, &oid, flag)); + loose_fill_ref_dir_regular_file(refs, refname.buf, dir); } strbuf_setlen(&refname, dirnamelen); } -- GitLab From d165358b83c41620b3d6237a3de2be942c5b7ef1 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Fri, 5 Jan 2024 17:13:42 +0100 Subject: [PATCH 3/4] refs: introduce `refs_for_each_all_refs()` Introduce a new ref iteration flag `DO_FOR_EACH_INCLUDE_ALL_REFS`, which will be used to iterate over all refs. In the files backend this is limited to regular refs, pseudorefs and HEAD. For other backends like the reftable this is the universal set of all refs stored in the backend. Refs which fall outside the `refs/` and aren't either pseudorefs or HEAD are more of a grey area. This is because we don't block the users from creating such refs but they are not officially supported. In the files backend, we can isolate such files from other files. Introduce `refs_for_each_all_refs()` which calls `do_for_each_ref()` with this newly introduced flag. In `refs/files-backend.c`, introduce a new function `add_pseudoref_and_head_entries()` to add pseudorefs and HEAD to the `ref_dir`. We then finally call `add_pseudoref_and_head_entries()` whenever the `DO_FOR_EACH_INCLUDE_ALL_REFS` flag is set. Any new ref backend will also have to implement similar changes on its end. Signed-off-by: Karthik Nayak --- refs.c | 7 +++++ refs.h | 6 ++++ refs/files-backend.c | 65 ++++++++++++++++++++++++++++++++++++++++---- refs/refs-internal.h | 7 +++++ 4 files changed, 80 insertions(+), 5 deletions(-) diff --git a/refs.c b/refs.c index 559f5aeea82..89b925719fa 100644 --- a/refs.c +++ b/refs.c @@ -1762,6 +1762,13 @@ int for_each_rawref(each_ref_fn fn, void *cb_data) return refs_for_each_rawref(get_main_ref_store(the_repository), fn, cb_data); } +int refs_for_each_all_refs(struct ref_store *refs, each_ref_fn fn, + void *cb_data) +{ + return do_for_each_ref(refs, "", NULL, fn, 0, + DO_FOR_EACH_INCLUDE_ALL_REFS, cb_data); +} + static int qsort_strcmp(const void *va, const void *vb) { const char *a = *(const char **)va; diff --git a/refs.h b/refs.h index 46b8085d632..77ecb820f9c 100644 --- a/refs.h +++ b/refs.h @@ -396,6 +396,12 @@ int for_each_namespaced_ref(const char **exclude_patterns, int refs_for_each_rawref(struct ref_store *refs, each_ref_fn fn, void *cb_data); int for_each_rawref(each_ref_fn fn, void *cb_data); +/* + * Iterates over all ref types, regular, pseudorefs and HEAD. + */ +int refs_for_each_all_refs(struct ref_store *refs, each_ref_fn fn, + void *cb_data); + /* * Normalizes partial refs to their fully qualified form. * Will prepend to the if it doesn't start with 'refs/'. diff --git a/refs/files-backend.c b/refs/files-backend.c index 22495a48071..104f2e1ac79 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -315,9 +315,59 @@ static void loose_fill_ref_dir(struct ref_store *ref_store, add_per_worktree_entries_to_dir(dir, dirname); } -static struct ref_cache *get_loose_ref_cache(struct files_ref_store *refs) +/* + * Add pseudorefs to the ref dir by parsing the directory for any files + * which follow the pseudoref syntax. + */ +static void add_pseudoref_and_head_entries(struct ref_store *ref_store, + struct ref_dir *dir, + const char *dirname) +{ + struct files_ref_store *refs = + files_downcast(ref_store, REF_STORE_READ, "fill_ref_dir"); + struct strbuf path = STRBUF_INIT, refname = STRBUF_INIT; + struct dirent *de; + size_t dirnamelen; + DIR *d; + + files_ref_path(refs, &path, dirname); + + d = opendir(path.buf); + if (!d) { + strbuf_release(&path); + return; + } + + strbuf_addstr(&refname, dirname); + dirnamelen = refname.len; + + while ((de = readdir(d)) != NULL) { + unsigned char dtype; + + if (de->d_name[0] == '.') + continue; + if (ends_with(de->d_name, ".lock")) + continue; + strbuf_addstr(&refname, de->d_name); + + dtype = get_dtype(de, &path, 1); + if (dtype == DT_REG && (is_pseudoref(ref_store, de->d_name) || + is_headref(ref_store, de->d_name))) + loose_fill_ref_dir_regular_file(refs, refname.buf, dir); + + strbuf_setlen(&refname, dirnamelen); + } + strbuf_release(&refname); + strbuf_release(&path); + closedir(d); +} + +static struct ref_cache *get_loose_ref_cache(struct files_ref_store *refs, + unsigned int flags) { if (!refs->loose) { + struct ref_dir *dir; + /* * Mark the top-level directory complete because we * are about to read the only subdirectory that can @@ -328,12 +378,17 @@ static struct ref_cache *get_loose_ref_cache(struct files_ref_store *refs) /* We're going to fill the top level ourselves: */ refs->loose->root->flag &= ~REF_INCOMPLETE; + dir = get_ref_dir(refs->loose->root); + + if (flags & DO_FOR_EACH_INCLUDE_ALL_REFS) + add_pseudoref_and_head_entries(dir->cache->ref_store, dir, + refs->loose->root->name); + /* * Add an incomplete entry for "refs/" (to be filled * lazily): */ - add_entry_to_dir(get_ref_dir(refs->loose->root), - create_dir_entry(refs->loose, "refs/", 5)); + add_entry_to_dir(dir, create_dir_entry(refs->loose, "refs/", 5)); } return refs->loose; } @@ -861,7 +916,7 @@ static struct ref_iterator *files_ref_iterator_begin( * disk, and re-reads it if not. */ - loose_iter = cache_ref_iterator_begin(get_loose_ref_cache(refs), + loose_iter = cache_ref_iterator_begin(get_loose_ref_cache(refs, flags), prefix, ref_store->repo, 1); /* @@ -1222,7 +1277,7 @@ static int files_pack_refs(struct ref_store *ref_store, packed_refs_lock(refs->packed_ref_store, LOCK_DIE_ON_ERROR, &err); - iter = cache_ref_iterator_begin(get_loose_ref_cache(refs), NULL, + iter = cache_ref_iterator_begin(get_loose_ref_cache(refs, 0), NULL, the_repository, 0); while ((ok = ref_iterator_advance(iter)) == ITER_OK) { /* diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 8e9f04cc670..1cf75064350 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -260,6 +260,13 @@ enum do_for_each_ref_flags { * INCLUDE_BROKEN, since they are otherwise not included at all. */ DO_FOR_EACH_OMIT_DANGLING_SYMREFS = (1 << 2), + + /* + * Include all refs in the $GIT_DIR in contrast to generally only listing + * references having the "refs/" prefix. In the files-backend this is + * limited to regular refs, pseudorefs and HEAD. + */ + DO_FOR_EACH_INCLUDE_ALL_REFS = (1 << 3), }; /* -- GitLab From a17983d0baefa01acfa722ed2bd3b893cacd2ea8 Mon Sep 17 00:00:00 2001 From: Karthik Nayak Date: Tue, 16 Jan 2024 12:15:53 +0100 Subject: [PATCH 4/4] for-each-ref: avoid filtering on empty pattern When the user uses an empty string pattern (""), we don't match any refs in git-for-each-ref(1). This is because in git-for-each-ref(1), we use path based matching and an empty string doesn't match any path. In this commit we change this behavior by making empty string pattern match all references. This is done by introducing a new flag `FILTER_REFS_NO_FILTER` in `ref-filter.c`, which uses the newly introduced `refs_for_each_all_refs()` function to iterate over all the refs in the repository. Signed-off-by: Karthik Nayak --- Documentation/git-for-each-ref.txt | 3 ++- builtin/for-each-ref.c | 21 +++++++++++++++++- ref-filter.c | 13 ++++++++++-- ref-filter.h | 4 +++- t/t6302-for-each-ref-filter.sh | 34 ++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/Documentation/git-for-each-ref.txt b/Documentation/git-for-each-ref.txt index be9543f6840..b1cb482bf53 100644 --- a/Documentation/git-for-each-ref.txt +++ b/Documentation/git-for-each-ref.txt @@ -32,7 +32,8 @@ OPTIONS If one or more patterns are given, only refs are shown that match against at least one pattern, either using fnmatch(3) or literally, in the latter case matching completely or from the - beginning up to a slash. + beginning up to a slash. If an empty string is provided all refs + are printed, including HEAD and pseudorefs. --stdin:: If `--stdin` is supplied, then the list of patterns is read from diff --git a/builtin/for-each-ref.c b/builtin/for-each-ref.c index 3885a9c28e1..5aa879e8be6 100644 --- a/builtin/for-each-ref.c +++ b/builtin/for-each-ref.c @@ -25,6 +25,7 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) struct ref_format format = REF_FORMAT_INIT; int from_stdin = 0; struct strvec vec = STRVEC_INIT; + unsigned int flags = FILTER_REFS_ALL; struct option opts[] = { OPT_BIT('s', "shell", &format.quote_style, @@ -93,11 +94,29 @@ int cmd_for_each_ref(int argc, const char **argv, const char *prefix) /* vec.v is NULL-terminated, just like 'argv'. */ filter.name_patterns = vec.v; } else { + size_t i; + filter.name_patterns = argv; + + /* + * Search for any empty string pattern, if it exists then we + * print all refs without any filtering. + */ + i = 0; + while (argv[i]) { + if (!argv[i][0]) { + flags = FILTER_REFS_NO_FILTER; + /* doing this removes any pattern from being matched */ + filter.name_patterns[0] = NULL; + break; + } + + i++; + } } filter.match_as_path = 1; - filter_and_format_refs(&filter, FILTER_REFS_ALL, sorting, &format); + filter_and_format_refs(&filter, flags, sorting, &format); ref_filter_clear(&filter); ref_sorting_release(sorting); diff --git a/ref-filter.c b/ref-filter.c index 35b989e1dfe..6dac133b873 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -2622,6 +2622,11 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter, each_ref_fn cb, void *cb_data) { + if (filter->kind & FILTER_REFS_NO_FILTER) { + return refs_for_each_all_refs( + get_main_ref_store(the_repository), cb, cb_data); + } + if (!filter->match_as_path) { /* * in this case, the patterns are applied after @@ -2775,8 +2780,12 @@ static struct ref_array_item *apply_ref_filter(const char *refname, const struct /* Obtain the current ref kind from filter_ref_kind() and ignore unwanted refs. */ kind = filter_ref_kind(filter, refname); - if (!(kind & filter->kind)) + if (filter->kind & FILTER_REFS_NO_FILTER) { + if (kind == FILTER_REFS_DETACHED_HEAD) + kind = FILTER_REFS_OTHERS; + } else if (!(kind & filter->kind)) { return NULL; + } if (!filter_pattern_match(filter, refname)) return NULL; @@ -3041,7 +3050,7 @@ static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref ret = for_each_fullref_in("refs/remotes/", fn, cb_data); else if (filter->kind == FILTER_REFS_TAGS) ret = for_each_fullref_in("refs/tags/", fn, cb_data); - else if (filter->kind & FILTER_REFS_ALL) + else if (filter->kind & FILTER_REFS_ALL || filter->kind & FILTER_REFS_NO_FILTER) ret = for_each_fullref_in_pattern(filter, fn, cb_data); if (!ret && (filter->kind & FILTER_REFS_DETACHED_HEAD)) head_ref(fn, cb_data); diff --git a/ref-filter.h b/ref-filter.h index 07cd6f6da3d..1eab325ce04 100644 --- a/ref-filter.h +++ b/ref-filter.h @@ -22,7 +22,9 @@ #define FILTER_REFS_ALL (FILTER_REFS_TAGS | FILTER_REFS_BRANCHES | \ FILTER_REFS_REMOTES | FILTER_REFS_OTHERS) #define FILTER_REFS_DETACHED_HEAD 0x0020 -#define FILTER_REFS_KIND_MASK (FILTER_REFS_ALL | FILTER_REFS_DETACHED_HEAD) +#define FILTER_REFS_NO_FILTER 0x0040 +#define FILTER_REFS_KIND_MASK (FILTER_REFS_ALL | FILTER_REFS_DETACHED_HEAD | \ + FILTER_REFS_NO_FILTER) struct atom_value; struct ref_sorting; diff --git a/t/t6302-for-each-ref-filter.sh b/t/t6302-for-each-ref-filter.sh index 82f3d1ea0f2..3922326cab3 100755 --- a/t/t6302-for-each-ref-filter.sh +++ b/t/t6302-for-each-ref-filter.sh @@ -31,6 +31,40 @@ test_expect_success 'setup some history and refs' ' git update-ref refs/odd/spot main ' +cat >expect <<-\EOF + HEAD + ORIG_HEAD + refs/heads/main + refs/heads/side + refs/odd/spot + refs/tags/annotated-tag + refs/tags/doubly-annotated-tag + refs/tags/doubly-signed-tag + refs/tags/four + refs/tags/one + refs/tags/signed-tag + refs/tags/three + refs/tags/two +EOF + +test_expect_success 'empty pattern prints pseudorefs' ' + git update-ref ORIG_HEAD main && + git for-each-ref --format="%(refname)" "" >actual && + test_cmp expect actual +' + +test_expect_success 'empty pattern with other patterns' ' + git update-ref ORIG_HEAD main && + git for-each-ref --format="%(refname)" "" "refs/" >actual && + test_cmp expect actual +' + +test_expect_success 'empty pattern towards the end' ' + git update-ref ORIG_HEAD main && + git for-each-ref --format="%(refname)" "refs/" "" >actual && + test_cmp expect actual +' + test_expect_success 'filtering with --points-at' ' cat >expect <<-\EOF && refs/heads/main -- GitLab